ATLAS Offline Software
AtlCoolCopy.cxx
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
3 */
4 
5 // AtlCoolCopy.cxx
6 // COOL copying utility, C++ based on PyCoolCopy from Sven Schmidt
7 // Richard Hawkings, started 30/5/06
8 // compiles in offline cmt framework to binary executable, needs offline env
9 
10 #include <algorithm>
11 #include <cmath>
12 #include <ctime>
13 #include <fstream>
14 #include <iostream>
15 #include <sstream>
16 #include <string>
17 #include <vector>
18 #include <nlohmann/json.hpp>
19 #include <curl/curl.h>
20 
21 #include "CoolKernel/DatabaseId.h"
22 #include "CoolKernel/Exception.h"
23 #include "CoolKernel/IDatabaseSvc.h"
24 #include "CoolKernel/IDatabase.h"
25 #include "CoolKernel/IFolder.h"
26 #include "CoolKernel/FolderSpecification.h"
27 #include "CoolKernel/IFolderSet.h"
28 #include "CoolKernel/IObject.h"
29 #include "CoolKernel/IObjectIterator.h"
30 #include "CoolKernel/IRecordIterator.h"
31 #include "CoolApplication/Application.h"
32 #include "CoralBase/AttributeListException.h"
33 
39 #include "RelationalAccess/ConnectionService.h"
40 #include "RelationalAccess/IConnectionServiceConfiguration.h"
41 #include "RelationalAccess/ISessionProxy.h"
42 #include "RelationalAccess/ITransaction.h"
43 #include "RelationalAccess/ISchema.h"
44 #include "RelationalAccess/ITable.h"
45 #include "RelationalAccess/IQuery.h"
46 #include "RelationalAccess/ICursor.h"
47 
52 
54 
55 #include "PoolMapElement.h"
56 #include "ReplicaSorter.h"
57 #include "CoolTagInfo.h"
58 
59 #include "TFile.h"
60 #include "TTree.h"
61 #include "TH1.h"
62 #include "TObjString.h"
63 
64 size_t WriteCallback(void *contents, size_t size, size_t nmemb, std::string *s) {
65  size_t newLength = size * nmemb;
66 
67  try
68  {
69  s->append((char *)contents, newLength);
70  }
71  catch (std::bad_alloc &e)
72  {
73  // handle memory problem
74  return 0;
75  }
76  return newLength;
77 }
78 
79 
80 class AtlCoolCopy {
81  public:
82  AtlCoolCopy(const std::string& sourcedb, const std::string& destdb,
83  bool allowcreate=false);
84  bool isOpen() const;
85  bool addFolder(const std::string& folder,const bool onlyTags);
86  bool addExclude(const std::string& folder);
88  (const std::string& folder,const std::vector<std::string>& taglist);
89  int doCopy ATLAS_NOT_THREAD_SAFE ();
90  int setOpts(int argc, const char* argv[]);
91 
92  private:
93  // methods
94  bool openConnections(const std::string& sourcedb, const std::string& destdb,
95  bool allowcreate);
96  bool openCoraCool();
97  bool getLockedTags();
98  bool procOptVector(const int argc, const char* argv[],
99  std::vector<std::string>& folders);
100  static std::string transConn(const std::string& inconn);
101  void setChannelRange(const cool::IFolderPtr& sourcefl);
102  static cool::ChannelId channelID(const cool::IFolderPtr& folder,
103  const std::string& chanstring);
105  (const std::string& folder,const std::string& destfolder,
106  const cool::IFolderPtr& sourcefl,const CoraCoolFolderPtr& sourceflc,
107  const cool::IFolderPtr& destfl,const CoraCoolFolderPtr& destflc,
108  const std::string& sourcetag,const std::string& desttag,
109  const cool::ValidityKey since,const cool::ValidityKey until,
110  bool timestamp,bool checkrefs,bool iscora,
111  const cool::PayloadMode::Mode paymode, bool created);
112  void adjustIOVs(const cool::ValidityKey& since,
113  const cool::ValidityKey& until,
114  const cool::ValidityKey& qsince,
115  const cool::ValidityKey& quntil,
116  cool::ValidityKey& newsince,
117  cool::ValidityKey& newuntil,
118  const bool timestamp) const;
119  int nocopyIOVs(const std::string& folder,
120  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
121  const cool::ValidityKey since,const cool::ValidityKey until,
122  bool checkrefs);
123  int verifyIOVs(const std::string& folder,
124  const cool::IFolderPtr& sourcefl,const CoraCoolFolderPtr& sourceflc,
125  const cool::IFolderPtr& destfl,const CoraCoolFolderPtr& destflc,
126  const std::string& sourcetag,
127  const cool::ValidityKey since,const cool::ValidityKey until,
128  const bool checkrefs,const bool iscora,
129  const cool::PayloadMode::Mode paymode);
130  int rootIOVs(const std::string& folder,
131  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
132  const cool::ValidityKey since,const cool::ValidityKey until,
133  const bool timestamp);
134  std::string rootDirs(const std::string& folder, const std::string& toproot);
135  bool rootAllocate(const cool::IFieldSpecification& spec,
136  void*& sptr,char& rootID) const;
137  void rootWrite(void* sptr, const cool::IField& field) const;
138  static bool isNumeric(const char* input);
139  static bool equalRecord(const cool::IRecord& lhs,
140  const cool::IRecord& rhs);
141  int analyseIOVs(const std::string& folder,
142  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
143  const cool::ValidityKey since,const cool::ValidityKey until,
144  const bool timestamp);
145  static TH1F* bookOrFindTH1F(const std::string& hID, const std::string& htitle,
146  const int chan, const float xlow, const float xhigh);
147  static cool::ValidityKey timeVal(const char* input);
148  static std::string timeString(const cool::ValidityKey iovtime);
149  static cool::ValidityKey runLBVal(const char* input1, const char* input2);
150  bool getTimeFromRun();
151  bool getOnlineRun();
152  bool getBulkRun();
153  bool getRunList();
154 
155  static int getUpdateMode(std::string_view desc, std::string_view tag);
156 
157  bool checkChannels(const std::string& folder,
158  const cool::IFolderPtr& sourcefl,const cool::IFolderPtr& destfl,
159  bool newfolder);
160  void checkRef(const cool::IRecord& payload,
161  const std::string& folder,const std::string& tag);
162  int tagParents();
163  int writeTagInfo();
164  int listPoolRefs();
165  int resolvePoolRefs ATLAS_NOT_THREAD_SAFE ();
166  static std::string getCoolHistGUID(const std::string& file);
167  void filePoolRefs();
168  static pool::IFileCatalog* setupCatalog(const std::vector<std::string>& catvec);
169 
170  // input parameters
171  std::string m_sourcedb;
172  std::string m_destdb;
179  bool m_debug;
180  bool m_alliov;
181  bool m_verify;
182  bool m_root;
184  bool m_analyse;
186  bool m_listpfn;
188  bool m_poolall;
189  bool m_nocopy;
190  bool m_nodata;
192  bool m_chdesc;
193  bool m_hitag;
194  bool m_nohitag;
203  bool m_skipout;
204  bool m_skiprep;
205  bool m_applock;
208  bool m_gettime;
211  bool m_getbulk;
221  cool::ValidityKey m_runemin;
222  cool::ValidityKey m_runemax;
223  cool::ValidityKey m_timemin;
224  cool::ValidityKey m_timemax;
225  cool::ValidityKey m_newrunemin;
226  cool::ValidityKey m_newrunemax;
227  cool::ValidityKey m_newtimemin;
228  cool::ValidityKey m_newtimemax;
229  cool::ValidityKey m_srunemin;
230  cool::ValidityKey m_srunemax;
231  std::vector<std::string> m_channelRange;
232  std::string m_channel1;
233  std::string m_channel2;
236  long long m_anadelt;
237  std::string m_outfolder;
238  std::string m_outtag;
239  std::string m_newdataset;
240  std::string m_checkoutputfile;
241  std::string m_timedb;
242  std::string m_taglabel;
243  std::string m_runinfohost;
244  std::vector<std::string> m_addguid; // additional guids to be processsed
245  std::vector<std::string> m_addlfn; // additional LFNs to be processsed
246  std::vector<std::string> m_parfile; // list of additional files with params
247  std::vector<cool::ChannelId> m_excludechans; // list of channels to exclude
248  std::vector<std::string> m_runfile; // list of filenames with run numbers
249  std::vector<unsigned int> m_runlist; // list of runs open for UPD2 tags
250 
251  // internal variables
252  coral::ConnectionService m_coralsvc;
253  cool::Application m_coolapp;
254  cool::IDatabaseSvc* m_dbSvc;
256  cool::IDatabasePtr m_sourceDbPtr;
257  cool::IDatabasePtr m_destDbPtr;
260  std::vector<std::string> m_folderlist; // list of leaf folders to process
261  std::vector<std::string> m_folderexcl; // list of leaf folders to exclude
262  std::vector<std::string> m_tags; // list of tags
263  std::vector<std::string> m_magic; // list magic tags fragments to match
264  std::vector<std::string> m_poolcat; // list of POOL catalogues for input
265  std::vector<std::string> m_mergecat; // list of POOL catalogues for mergechk
266  bool m_open;
267  cool::ChannelSelection m_chansel;
268  typedef std::map<std::string,PoolMapElement> PoolMap;
269  PoolMap::iterator m_poollast; // pointer to last POOL ref updated
270  PoolMap m_poolrefs; // POOL refs and usage counts
271  // hierarchical tags - tag name and folder
272  using HiTagMap = std::map<std::string, std::string>;
274  // tags indirectly used in the hierarchy and have to be treated at end
275  std::vector<std::string> m_hiparent;
276  // map of CoolTagInfo objects - tags which have have their descriptions
277  // and lock status set in the destination DB at the end
278  using CoolTagMap = std::map<std::string, CoolTagInfo>;
280  // output root file for ROOT file export and IOV analysis
281  TFile* p_rootfile;
282 
283  // ROOT ntuple variables.
284  ULong64_t m_nt_since = 0;
285  ULong64_t m_nt_until = 0;
286  UInt_t m_nt_runsince = 0;
287  UInt_t m_nt_rununtil = 0;
288  UInt_t m_nt_lbsince = 0;
289  UInt_t m_nt_lbuntil = 0;
290  UInt_t m_nt_channel = 0;
291  char m_nt_tagid[256] = {0};
292  std::string m_nt_treename;
293  std::vector<void*> m_nt_bufferptr;
294 };
295 
296 inline bool AtlCoolCopy::isOpen() const { return m_open; }
297 
298 void printHelp();
299 
300 AtlCoolCopy::AtlCoolCopy(const std::string& sourcedb, const std::string& destdb,
301  bool allowcreate) :
302  m_sourcedb(sourcedb),m_destdb(destdb),m_allowcreate(allowcreate),
303  m_recreate(false),
304  m_includehead(false),m_excludehead(false),
305  m_usertags(true),m_userupdatehead(false),m_debug(false),m_alliov(false),
306  m_verify(false),m_root(false),m_zeronull(false),m_analyse(false),
307  m_checkrefs(false),m_listpfn(false),m_poolopen(false),m_poolall(false),
308  m_nocopy(false),m_nodata(false),m_nochannel(false),m_chdesc(false),
309  m_hitag(false),m_nohitag(false),m_forcesingle(false),m_forcemulti(false),
310  m_forcerune(false),m_forcetime(false),m_forcepay(false),m_forcenopay(false),
311  m_sourceread(true),m_truncate(false),m_skipout(false),m_skiprep(false),
312  m_applock(false),m_applocksv(false),
313  m_readoracle(false),m_gettime(false),m_getonline(false),m_onlinerun(false),
314  m_getbulk(false),
315  m_prunetags(false),m_lockedonly(false),m_copytaginfo(false),
316  m_copytaglock(false),m_coracool(true),m_ignoremode(false),
317  m_ignorespec(false),m_checkdesttag(false),m_noclobroot(false),
318  m_runemin(cool::ValidityKeyMin),m_runemax(cool::ValidityKeyMax),
319  m_timemin(cool::ValidityKeyMin),m_timemax(cool::ValidityKeyMax),
320  m_newrunemin(cool::ValidityKeyMin),m_newrunemax(cool::ValidityKeyMax),
321  m_newtimemin(cool::ValidityKeyMin),m_newtimemax(cool::ValidityKeyMax),
322  m_srunemin(cool::ValidityKeyMin),m_srunemax(cool::ValidityKeyMax),
323  m_channel1(""),m_channel2(""),m_bufsize(1000),m_sealmsg(5),
324  m_anadelt(-1),m_outfolder(""),m_outtag(""),m_newdataset(""),
325  m_checkoutputfile(""),m_timedb(""),m_taglabel(""),
326  m_runinfohost("http://atlas-run-info-api.web.cern.ch/api"),
327  m_coolapp(&m_coralsvc),
328  m_dbSvc(&(m_coolapp.databaseService())),m_repsort(nullptr),
329  m_open(false),m_chansel(cool::ChannelSelection::all()),p_rootfile(nullptr)
330 {
331  m_poolrefs.clear();
332  m_poollast=m_poolrefs.end();
333  // configure CORAL components
334  coral::IConnectionServiceConfiguration& csconfig=m_coralsvc.configuration();
335  csconfig.disablePoolAutomaticCleanUp();
336  csconfig.setConnectionTimeOut(0);
337 }
338 
339 bool AtlCoolCopy::openConnections(const std::string& sourcedb,
340  const std::string& destdb,bool allowcreate) {
341  // check connections not already open
342  if (m_open) return true;
343  // initialise replica sorter if it was requested
344  if (m_readoracle) {
345  m_repsort=new ReplicaSorter();
346  coral::IConnectionServiceConfiguration& csconfig=m_coralsvc.configuration();
347  csconfig.setReplicaSortingAlgorithm(*m_repsort);
348  }
349  // cool::IDatabaseSvc& dbSvc=cool::DatabaseSvcFactory::databaseService();
350  // open source database
351  std::cout << "Open source database: " << sourcedb << std::endl;
352  if (!m_sourceread) std::cout << "... in UPDATE mode" << std::endl;
353  try {
354  m_sourceDbPtr=m_dbSvc->openDatabase(transConn(sourcedb),m_sourceread);
355  }
356  catch (std::exception& e) {
357  std::cout << "Cool exception caught: " << e.what() << std::endl;
358  return false;
359  }
360  // open destination database
361  if (m_nocopy) {
362  m_open=true;
363  return true;
364  } else if (m_root || m_analyse) {
365  std::cout << "Open destination ROOT file: " << destdb << std::endl;
366  p_rootfile=new TFile(destdb.c_str(),"RECREATE");
367  if (p_rootfile==nullptr) std::cout << "ERROR: Could not open ROOT file" <<
368  std::endl;
369  m_open=(p_rootfile!=nullptr);
370  return m_open;
371  }
372  std::string tdestdb=transConn(destdb);
373  std::cout << "Open destination database: " << tdestdb << std::endl;
374  try {
375  m_destDbPtr=m_dbSvc->openDatabase(tdestdb,m_verify);
376  // if the open succeeds and we are using recreate mode, drop/delete
377  // the existing database first
378  if (m_recreate) {
379  std::cout <<
380  "Forcing recreation of destination database - deleting existing data!"
381  << std::endl;
382  m_destDbPtr.reset();
383  m_dbSvc->dropDatabase(tdestdb);
384  // go into catch to recrete database
385  throw cool::DatabaseDoesNotExist("old database deleted");
386  }
387  }
388  catch (std::exception& e) {
389  std::cout << "COOL exception caught: " << e.what() << std::endl;
390  // try to recover by creating new DB if possible and requested
391  if (allowcreate || m_recreate) {
392  std::cout << "Try to create new conditions DB" << std::endl;
393  try {
394  m_destDbPtr=m_dbSvc->createDatabase(tdestdb);
395  std::cout << "Creation succeeded" << std::endl;
396  }
397  catch (cool::Exception& e) {
398  std::cout << "Creation failed" << std::endl;
399  return false;
400  }
401  } else {
402  return false;
403  }
404  }
405  m_open=true;
406  return true;
407 }
408 
410  if (m_sourceCoraPtr.get()==nullptr) {
411  std::cout << "Attempt to open source CoraCool DB " << m_sourcedb <<
412  std::endl;
415  std::cout << "Opened CoraCool source DB" << std::endl;
416  }
417  // skip destination DB if not copying
418  if (m_nocopy) return true;
419  if (m_destCoraPtr.get()==nullptr) {
420  std::cout << "Attempt to open destination CoraCool DB " << m_destdb <<
421  std::endl;
424  std::cout << "Opened CoraCool dest DB" << std::endl;
425  }
426  return true;
427 }
428 
430  // set m_tags to list of top level tags which are locked
431  m_tags.clear();
432  cool::IFolderSetPtr topfolder=m_sourceDbPtr->getFolderSet("/");
433  const std::vector<std::string>& toptaglist=topfolder->listTags();
434  for (std::vector<std::string>::const_iterator toptag=toptaglist.begin();
435  toptag!=toptaglist.end();++toptag) {
436  cool::HvsTagLock::Status tstat=topfolder->tagLockStatus(*toptag);
437  if (tstat==cool::HvsTagLock::LOCKED ||
438  tstat==cool::HvsTagLock::PARTIALLYLOCKED) {
439  std::cout << "Top-level tag " << *toptag << " will be copied" <<
440  std::endl;
441  m_tags.push_back(*toptag);
442  }
443  }
444  std::cout << "Total of " << m_tags.size() << " top-level tags to be copied"
445  << std::endl;
446  return (!m_tags.empty());
447 }
448 
449 std::string AtlCoolCopy::transConn(const std::string& inconn) {
450  // translate simple connection string (no slash) to mycool.db with given
451  // instance name, all others are left alone
452  if (inconn.find('/')==std::string::npos) {
453  return "sqlite://X;schema=mycool.db;dbname="+inconn;
454  } else {
455  return inconn;
456  }
457 }
458 
459 
460 bool AtlCoolCopy::addFolder(const std::string& folder,const bool onlyTags) {
461  std::cout << "Add folders in path:" << folder << " [ ";
462  const std::vector<std::string> nodelist=m_sourceDbPtr->listAllNodes();
463  // find all matching leaf folders
464  for (std::vector<std::string>::const_iterator nodeitr=nodelist.begin();
465  nodeitr!=nodelist.end();++nodeitr) {
466  // match exact folder name or leading part of path
467  // for leading part matches, next char in folder name must be '/'
468  // so /CALO/SetA matches /CALO/SetA/X but not /CALO/SetAB
469  if (*nodeitr==folder || folder=="/" ||
470  (nodeitr->compare(0,folder.size(),folder)==0 &&
471  nodeitr->size()>folder.size() && nodeitr->compare(folder.size(),1,"/")==0)) {
472  // check if folder on exclude list
473  bool exclude=false;
474  for (std::vector<std::string>::const_iterator iexcl=m_folderexcl.begin();
475  iexcl!=m_folderexcl.end();++iexcl) {
476  if (iexcl->compare(0,1,"/")==0) {
477  // exclude pattern starting / matches folder path (/SCT or /SCT/DCS)
478  exclude=(exclude || nodeitr->compare(0,iexcl->size(),*iexcl)==0);
479  } else {
480  // exclude pattern without leading / matches anywhere in folder
481  exclude=(exclude || (nodeitr->find(*iexcl)!=std::string::npos));
482  }
483  }
484  // check folder exists (is a leaf folder), and not excluded
485  if (m_sourceDbPtr->existsFolder(*nodeitr) && !exclude && !onlyTags) {
486  // only add if folder not already on list
487  if (find(m_folderlist.begin(),m_folderlist.end(),*nodeitr)==
488  m_folderlist.end()) {
489  std::cout << *nodeitr << " ";
490  m_folderlist.push_back(*nodeitr);
491  }
492  }
493  // if its a folderset, check for any hierarchical tags
494  if (!m_nohitag && m_sourceDbPtr->existsFolderSet(*nodeitr) && !exclude) {
495  if (!m_prunetags) {
496  const std::vector<std::string> foldersettags=
497  m_sourceDbPtr->getFolderSet(*nodeitr)->listTags();
498  if (!foldersettags.empty()) {
499  for (std::vector<std::string>::const_iterator
500  itr=foldersettags.begin();itr!=foldersettags.end();++itr) {
501  m_hitagmap[*itr]=*nodeitr;
502  }
503  }
504  } else {
505  // only take tags in this folder which are referenced from one of
506  // the input tags (assumed to be top-level tags)
507  cool::IFolderSetPtr sfolder=m_sourceDbPtr->getFolderSet(*nodeitr);
508  for (std::vector<std::string>::const_iterator toptag=m_tags.begin();
509  toptag!=m_tags.end();++toptag) {
510  try {
511  std::string rtag=sfolder->resolveTag(*toptag);
512  m_hitagmap[rtag]=*nodeitr;
513  }
514  // catch exceptions indicating tag defines nothing here
515  // note std::exception rather than COOL exception to cover case
516  // when trying to resolve a leaf tag in the '/' folder, which
517  // throws a coral AttributeException
518  catch (std::exception& e) {}
519  }
520  }
521  }
522  }
523  }
524  std::cout << "]" << std::endl;
525  return true;
526 }
527 
528 bool AtlCoolCopy::addExclude(const std::string& folder) {
529  std::cout << "Adding folder to exclude list: " << folder << std::endl;
530  m_folderexcl.push_back(folder);
531  return true;
532 }
533 
535  (const std::string& folder,const std::vector<std::string>& taglist) {
536  // get source folder
537  cool::IFolderPtr sourcefl,destfl;
538  CoraCoolFolderPtr sourceflc,destflc;
539  try {
540  sourcefl=m_sourceDbPtr->getFolder(folder);
541  }
542  catch (cool::Exception& e) {
543  std::cout << "Could not get source folder: " << folder << std::endl;
544  return 20;
545  }
546  const std::string& sourcedesc=sourcefl->description();
547  // check if folder should be skipped from replication
548  if (m_skiprep && sourcedesc.find("<norep/>")!=std::string::npos) {
549  std::cout << "Folder " << folder <<
550  " skipped due to <norep/> metadata" << std::endl;
551  return 0;
552  }
553  const bool iscora=(m_coracool &&
554  sourcedesc.find("<coracool")!=std::string::npos);
555  if (iscora) {
556  // activate CoraCool and get source folder pointer
557  openCoraCool();
558  sourceflc=m_sourceCoraPtr->getFolder(folder);
559  }
560  // setup the channel-range
561  setChannelRange(sourcefl);
562 
563  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
564  cool::FolderVersioning::Mode dvermode=vermode;
565  if (m_forcesingle) {
566  dvermode=cool::FolderVersioning::SINGLE_VERSION;
567  std::cout << "Forcing destination folder to singleversion" << std::endl;
568  }
569  if (m_forcemulti) {
570  dvermode=cool::FolderVersioning::MULTI_VERSION;
571  std::cout << "Forcing destination folder to multiversion" << std::endl;
572  }
573  // setup type of folder (payload mode, inline, separate, or vector)
574  const cool::PayloadMode::Mode spaymode=
575  sourcefl->folderSpecification().payloadMode();
576  cool::PayloadMode::Mode dpaymode=spaymode;
577 
578  // set name for destination folder
579  std::string destfolder=folder;
580  if (!m_outfolder.empty()) {
581  destfolder=m_outfolder;
582  std::cout << "Destination folder will be renamed to " << destfolder <<
583  std::endl;
584  }
585 
586 // get destination folder, try to create if not there (and not verifying!)
587  bool created=false;
588  if (!m_nocopy && !m_root && !m_analyse) {
589  if (!m_destDbPtr->existsFolder(destfolder) && !m_verify) {
590  std::cout << "Creating folder " << destfolder << " payload-type " <<
591  dpaymode << " on destination" << std::endl;
592  created=true;
593  std::string metadata=sourcedesc;
594  if (m_forcerune || m_forcetime) {
595  std::string newmeta=m_forcerune ? "run-lumi" : "time";
596  std::string::size_type p1,p2;
597  p1=metadata.find("<timeStamp>");
598  p2=metadata.find("</timeStamp>");
599  if (p1!=std::string::npos && p2!=std::string::npos) {
600  metadata.replace(0,p2,"<timeStamp>"+newmeta);
601  std::cout << "Forced destination folder to " << newmeta << " : "
602  << metadata << std::endl;
603  } else {
604  std::cout <<
605  "ERROR: Could not parse metadata string to force timestamp type"
606  << std::endl;
607  }
608  }
609  // force separate or inline payload, but not if input is vector
610  if (m_forcepay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
611  dpaymode=cool::PayloadMode::SEPARATEPAYLOAD;
612  if (m_forcenopay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
613  dpaymode=cool::PayloadMode::INLINEPAYLOAD;
614 
615  try {
616  if (iscora) {
617  destflc=m_destCoraPtr->createFolder(destfolder,
618  sourceflc->coralTableName(),
619  sourceflc->fkSpecification(),
620  sourceflc->payloadSpecification(),
621  sourceflc->coralFKey(),
622  sourceflc->coralPKey(),
623  metadata,dvermode,true);
624  std::cout << "Created CoraCool folder" << std::endl;
625  destfl=m_destDbPtr->getFolder(destfolder);
626  } else {
627  destfl=m_destDbPtr->createFolder(destfolder,
628  cool::FolderSpecification(dvermode,
629  sourcefl->payloadSpecification(),dpaymode),
630  metadata,true);
631  }
632  }
633  catch (cool::Exception&e ) {
634  std::cout << "Create folder failed - aborting" << std::endl;
635  return 30;
636  }
637  }
638  // now get the pointer to destination folder if needed
639  // note both COOL and CoraCool pointers are set if needed
640  try {
641  if (iscora) destflc=m_destCoraPtr->getFolder(destfolder);
642  destfl=m_destDbPtr->getFolder(destfolder);
643  }
644  catch (cool::Exception& e) {
645  std::cout << "Could not get destination folder: " << destfolder
646  << std::endl;
647  return 21;
648  }
649  // check payload specifications of folders are the same
650  const cool::IRecordSpecification& sourcespec=
651  sourcefl->payloadSpecification();
652  const cool::IRecordSpecification& destspec=
653  destfl->payloadSpecification();
654  if (!(sourcespec==destspec)) {
655  bool badspec=false;
656  if (m_ignorespec) {
657  // specifications differ - check names are same
658  std::cout <<
659  "WARNING Source and destination folder specifications differ" <<
660  std::endl;
661  for (unsigned int i=0;i<sourcespec.size();++i) {
662  const std::string& sname=sourcespec[i].name();
663  if (!destspec.exists(sname)) {
664  std::cout << "ERROR: Field " << sname << " absent from destination"
665  << std::endl;
666  badspec=true;
667  }
668  }
669  } else {
670  badspec=true;
671  }
672  if (badspec) {
673  std::cout <<
674  "ERROR Source and destination folder specifications differ"
675  << std::endl;
676  return 22;
677  }
678  }
679  // check folder descriptions are the same - just print WARNING if not
680  const std::string& destdesc=destfl->description();
681  if (sourcedesc!=destdesc) {
682  std::cout << "WARNING: Source and destination folder descriptions (meta-data) differ" << std::endl;
683  std::cout << "Source folder: " << sourcedesc << std::endl;
684  std::cout << "Destn folder: " << destdesc << std::endl;
685  }
686  // check payload modes are same - print warning if not
687  const cool::PayloadMode::Mode dpaymode=
688  destfl->folderSpecification().payloadMode();
689  if (spaymode!=dpaymode) {
690  std::cout << "WARNING: Source (" << spaymode << ") and destination (" <<
691  dpaymode << " folder payload modes differ" << std::endl;
692  }
693 
694  // check/set channels table if requested
695  if (!m_nochannel) {
696  if (!checkChannels(folder,sourcefl,destfl,created)) return 23;
697  }
698  }
699  // if only copying structures, stop here
700  if (m_nodata) return 0;
701  // extract folder/range information
702  std::cout << "Start to process folder: " << folder;
703  // check for timestamp XML
704  bool timestamp=(sourcefl->description().find("<timeStamp>time")!=
705  std::string::npos);
706  cool::ValidityKey since,until;
707  if (timestamp) {
708  since=m_timemin;
709  until=m_timemax;
710  std::cout << " (timestamp)" << std::endl;
711  } else {
712  since=m_runemin;
713  until=m_runemax;
714  std::cout << " (run/lumi)" << std::endl;
715  }
716  if (m_alliov || m_truncate)
717  std::cout << "Output IOVs will be modified" << std::endl;
718  if (m_skipout)
719  std::cout << "IOVs extending outside selection range will be skipped"
720  << std::endl;
721 
722  bool checkrefs=false;
723  const std::string name0=sourcefl->payloadSpecification()[0].name();
724  if (m_checkrefs && (name0=="PoolRef" || name0=="fileGUID")) {
725  checkrefs=true;
726  std::cout << "Check POOL references in folder " << folder << std::endl;
727  }
728  // if nocopy, and checking POOL references, and folder is not a POOL ref one
729  // can skip the data access
730  if (m_checkrefs && m_nocopy && !checkrefs) return 0;
731 
732  // check for <fullrep/> metadata indicating copy all tags if -skiprep option
733  bool copyall=
734  (m_skiprep && (sourcedesc.find("<fullrep/>")!=std::string::npos));
735  if (copyall) std::cout <<
736  "All tags in folder will be copied due to <fullrep/> metadata" << std::endl;
737 
738  // set up true list of tags to copy
739  std::vector<std::string> tags;
740  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
741  std::cout << "Single version folder" << std::endl;
742  tags.emplace_back("HEAD");
743  } else {
744  std::cout << "Multi version folder: consider tags [ ";
745  // get list of tags in this node which are requested
746  const std::vector<std::string> foldertags=sourcefl->listTags();
747  for (std::vector<std::string>::const_iterator itag=foldertags.begin();
748  itag!=foldertags.end();++itag) {
749  bool copyit=copyall;
750  // if input taglist is empty, take all tags in folder
751  if (taglist.empty() ||
752  find(taglist.begin(),taglist.end(),*itag)!=taglist.end())
753  copyit=true;
754  for (std::vector<std::string>::const_iterator imtag=m_magic.begin();
755  imtag!=m_magic.end();++imtag) {
756  if (itag->find(*imtag)!=std::string::npos) copyit=true;
757  }
758  if (copyit) {
759  tags.push_back(*itag);
760  std::cout << *itag << " ";
761  }
762  }
763  // if no tags were found, or doing inclusive hierarchical tag copying
764  // try resolving input tags hierarchically
765  if (tags.empty() || m_hitag) {
766  for (std::vector<std::string>::const_iterator itag=taglist.begin();
767  itag!=taglist.end();++itag) {
768  try {
769  std::string htag=sourcefl->resolveTag(*itag);
770  if (find(tags.begin(),tags.end(),htag)==tags.end()) {
771  std::cout << *itag << "=" << htag << " ";
772  tags.push_back(std::move(htag));
773  }
774  }
775  // ignore exceptions indicating tag not defined here
776  // note std::exception rather than COOL exception to cover case
777  // when trying to resolve a leaf tag in the '/' folder, which
778  // throws a coral AttributeException
779  catch (std::exception& e) { }
780  }
781  }
782  std::cout << "]" << std::endl;
783  // if still no tags were found, or forced HEAD tag copying, or no
784  // tags in the folder at all (=MV folder being used as SV), add HEAD
785  if (((tags.empty() || m_includehead) && !m_excludehead) ||
786  foldertags.empty())
787  tags.emplace_back("HEAD");
788  }
789 
790  sourcefl->setPrefetchAll(false);
791  for (std::vector<std::string>::const_iterator itag=tags.begin();
792  itag!=tags.end();++itag) {
793  std::string outtag=*itag;
794  if (!m_outtag.empty()) outtag=m_outtag;
795  int retcode;
796  // verify or copy folder
797  if (m_verify) {
798  if (!destfl) {
799  retcode = 1;
800  }
801  else {
802  retcode=verifyIOVs(folder,sourcefl,sourceflc,destfl,destflc,
803  *itag,since,until,checkrefs,iscora,spaymode);
804  }
805  } else if (m_nocopy) {
806  retcode=nocopyIOVs(folder,sourcefl,*itag,since,until,checkrefs);
807  } else if (m_root) {
808  retcode=rootIOVs(folder,sourcefl,*itag,since,until,timestamp);
809  } else if (m_analyse) {
810  retcode=analyseIOVs(folder,sourcefl,*itag,since,until,timestamp);
811  } else {
812  retcode=copyIOVs(folder,destfolder,sourcefl,sourceflc,destfl,destflc,
813  *itag,outtag,since,until,timestamp,checkrefs,iscora,
814  spaymode,created);
815  }
816  if (retcode!=0) {
817  std::cout << "ERROR operation failed for folder " << folder << " tag " <<
818  *itag << std::endl;
819  return retcode;
820  }
821  }
822  return 0;
823 }
824 
825 void AtlCoolCopy::setChannelRange(const cool::IFolderPtr& sourcefl) {
826  // add range specified via -ch1 -ch2 if given
827  if (!m_channel1.empty() && !m_channel2.empty()) {
828  m_channelRange.clear();
829  m_channelRange.push_back(m_channel1+":"+m_channel2);
830  }
832  const size_t nChanRange=m_channelRange.size();
833  for (size_t i=0;i<nChanRange;i++) {
834  size_t cpos=m_channelRange[i].find(':');
835  if (cpos==std::string::npos || cpos > m_channelRange[i].size()-1) {
836  // single channel
837  std::cout << "Adding channel " << m_channelRange[i] <<
838  " to channel selection" << std::endl;
839  if (m_chansel.allChannels())
840  m_chansel=cool::ChannelSelection(channelID(sourcefl,
841  m_channelRange[i]));
842  else
843  m_chansel.addChannel(channelID(sourcefl,m_channelRange[i]));
844  }
845  else {
846  // Channel Range
847  std::string c1=m_channelRange[i].substr(0,cpos);
848  std::string c2=m_channelRange[i].substr(1+cpos);
849  std::cout << "Adding channel range " << c1 << " to " << c2 <<
850  " to channel selection" << std::endl;
851  if (m_chansel.allChannels())
852  m_chansel=cool::ChannelSelection(channelID(sourcefl,c1),
853  channelID(sourcefl,c2));
854  else
855  m_chansel.addRange(channelID(sourcefl,c1),channelID(sourcefl,c2));
856  }
857  }//end loop over channel ranges
858 }
859 
860 cool::ChannelId AtlCoolCopy::channelID(const cool::IFolderPtr& folder,
861  const std::string& chanstring) {
862  const char* cstr=chanstring.c_str();
863  if (isNumeric(cstr)) {
864  // channel is a number
865  return cool::ChannelId(strtoul(cstr,nullptr,10));
866  } else {
867  cool::ChannelId chan=0;
868  try {
869  chan=folder->channelId(chanstring);
870  std::cout << "Channel name " << chanstring << " maps to channelID "
871  << chan << std::endl;
872  }
873  catch (cool::Exception& e) {
874  std::cout << "ERROR: Channel name " << chanstring <<
875  " not defined in folder " << std::endl;
876  }
877  return chan;
878  }
879 }
880 
881 // Calls non-thread-safe functions of CoraCoolFolder.
882 int AtlCoolCopy::copyIOVs ATLAS_NOT_THREAD_SAFE
883  (const std::string& folder,
884  const std::string& destfolder,
885  const cool::IFolderPtr& sourcefl,const CoraCoolFolderPtr& sourceflc,
886  const cool::IFolderPtr& destfl,const CoraCoolFolderPtr& destflc,
887  const std::string& sourcetag,const std::string& desttag,
888  const cool::ValidityKey since,const cool::ValidityKey until,
889  bool timestamp,bool checkrefs,bool iscora,
890  const cool::PayloadMode::Mode paymode ,bool created) {
891 
892  std::cout << "Copying tag " << sourcetag << " of folder " << folder <<
893  " to destination tag " << desttag << std::endl;
894  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
895  cool::FolderVersioning::Mode dvermode=destfl->versioningMode();
896  // check for online mode
897  int updatemode=getUpdateMode(destfl->description(),desttag);
898  std::vector<std::string> dtaglist=destfl->listTags();
899  // if online mode, must have --getonline and either -truncate or -alliov
900  // only if destination DB is oracle
901  if (updatemode==1 && m_destdb.find("oracle")!=std::string::npos &&
902  (!m_getonline || (!m_truncate && !m_alliov))) {
903  if (m_ignoremode) {
904  std::cout << "Folder is online (UPD1) mode but IGNORING PROTECTION"
905  << std::endl;
906  } else {
907  if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
908  std::cout << "Folder is online mode (UPD1) and tag already exist - -getonline and -truncate or -alliov options MUST be used" << std::endl;
909  return 35;
910  }
911  else {
912  std::cout << "Folder is online mode (UPD1), new tag will be created" << std::endl;
913  }
914 
915  }
916  }
917  // if UPD4 mode, must have -getbulk and either -truncate or -alliov
918  if (updatemode==4 && m_destdb.find("oracle")!=std::string::npos &&
919  (!m_getbulk || (!m_truncate && !m_alliov))) {
920  if (m_ignoremode) {
921  std::cout << "Folder is bulkreco (UPD4) mode but IGNORING PROTECTION"
922  << std::endl;
923  } else {
924  if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
925  std::cout << "Folder is bulkreco mode (UPD4) - -getbulk and -truncate or -alliov options MUST be used" << std::endl;
926  return 35;
927  } else {
928  std::cout << "Folder is bulkreco mode (UPD4), new tag will be created" << std::endl;
929  }
930 
931  }
932  }
933 
934  // if destination tags are being checked, check if this tag already
935  // exists in destination folder, if so skip copy
936  // also skip if tag is HEAD, and folder was not newly created
937  if (m_checkdesttag) {
938  if (find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()
939  || (desttag=="HEAD" && !created)) {
940  std::cout << "Destination tag " << desttag <<
941  " already exists in folder " << folder << " - skip copy" << std::endl;
942  return 0;
943  }
944  }
945  bool relock=false; // tag must be relocked afterwards
946  bool prot=false; //update is protected, irrespective of if tag must be locked
947  if ((m_applock || (updatemode==1 && m_getonline) || updatemode==2 ||
948  (updatemode==4 && m_getbulk)) &&
949  dvermode==cool::FolderVersioning::MULTI_VERSION) {
950  // check if the destination tag exists and is locked
951  std::vector<std::string> dtaglist=destfl->listTags();
952  for (std::vector<std::string>::const_iterator itr=dtaglist.begin();
953  itr!=dtaglist.end();++itr) {
954  if (desttag==*itr) {
955  // tag exists - must protect it unless ignoremode
956  prot=!m_ignoremode;
957  // check lock status
958  if (destfl->tagLockStatus(desttag)==cool::HvsTagLock::LOCKED) {
959  std::cout << "Unlocking destination tag " << desttag <<
960  " for append or UPDx access" << std::endl;
961  if (m_applocksv) {
962  std::cout << "Appending according to SV folder rules" << std::endl;
963  if (updatemode!=3) {
964  std::cout <<
965  "ERROR: Only allowed for UPD3 mode tags" << std::endl;
966  return 36;
967  }
968  }
969  destfl->setTagLockStatus(desttag,cool::HvsTagLock::UNLOCKED);
970  relock=true;
971  }
972  }
973  }
974  }
975  int nobj=0;
976  int nbuf=0;
977  int nskip=0;
978  int nbad=0;
979  // set up tag to be used at point of insertion
980  std::string localtag="";
981  if (dvermode==cool::FolderVersioning::MULTI_VERSION &&
982  desttag!="HEAD" && m_usertags) localtag=desttag;
983  // now loop over IOVs - separate for CoraCool and COOL
984  try {
985  if (iscora) {
986  // branch for CoraCool
987  sourceflc->setPrefetchAll(false);
988  CoraCoolObjectIterPtr sourceitr;
989  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
990  sourceitr=sourceflc->browseObjects(since,until,m_chansel);
991  } else {
992  sourceitr=sourceflc->browseObjects(since,until,m_chansel,sourcetag);
993  }
994  // keep track of new FKs of inserted objects
995  std::map<int,int> insertkeymap;
996  unsigned int nref=0;
997  while (sourceitr->hasNext()) {
998  CoraCoolObjectPtr obj=sourceitr->next();
999  // check object should not be skipped as spreading outside select-IOV
1000  if (m_skipout &&
1001  (obj->since()<m_srunemin || obj->until()>m_srunemax)) {
1002  ++nskip;
1003  continue;
1004  }
1005  // check object should not be skipped as excluded channel
1006  if (!m_excludechans.empty()) {
1007  if (find(m_excludechans.begin(),m_excludechans.end(),
1008  obj->channelId())!=m_excludechans.end()) {
1009  ++nskip;
1010  continue;
1011  }
1012  }
1013  if (nbuf==0) {
1014  if (m_debug) std::cout <<
1015  "Setup new CoraCool storage buffer at object " << nobj << std::endl;
1016  destflc->setupStorageBuffer();
1017  }
1018  cool::ValidityKey newsince,newuntil;
1019  adjustIOVs(obj->since(),obj->until(),since,until,newsince,newuntil,
1020  timestamp);
1021  // skip negative/zero IOV lengths
1022  if (newsince>=newuntil) {
1023  std::cout << "WARNING: Skipping IOV with since " << newsince <<
1024  ">= until" << newuntil << std::endl;
1025  ++nbad;
1026  continue;
1027  }
1028  // find the old FK, check if we have already inserted this
1029  // in which case a reference can be added
1030  int oldfk=0;
1031  std::map<int,int>::const_iterator ikey=insertkeymap.end();
1032  bool foundkey=false;
1033  // ensure returned data is not of zero size - if so cannot extract
1034  // and remember the FK
1035  if (obj->size()>0) {
1036  oldfk=sourceflc->getAttrKey(
1037  (*obj->begin())[sourceflc->coralFKey()]);
1038  ikey=insertkeymap.find(oldfk);
1039  foundkey=true;
1040  }
1041  if (ikey==insertkeymap.end()) {
1042  int newfk=
1043  destflc->storeObject(newsince,newuntil,obj->begin(),obj->end(),
1044  obj->channelId(),localtag,
1045  (!m_userupdatehead && !localtag.empty()));
1046  if (foundkey) insertkeymap[oldfk]=newfk;
1047  } else {
1048  destflc->referenceObject(newsince,newuntil,ikey->second,
1049  obj->channelId(),localtag,
1050  (!m_userupdatehead && !localtag.empty()));
1051  ++nref;
1052  }
1053  ++nbuf;
1054  // flush buffer every m_bufsize objects
1055  if (nbuf==m_bufsize) {
1056  if (m_debug) std::cout << "Flush buffer after " << nobj << "," <<
1057  nbuf << " objects " << std::endl;
1058  // note CoraCool requires explicit re-setup of buffer after a flush
1059  destflc->flushStorageBuffer();
1060  destflc->setupStorageBuffer();
1061  nbuf=0;
1062  }
1063  ++nobj;
1064  }
1065  sourceitr->close();
1066  if (nref>0) std::cout << "Reference-to-existing used for " << nref
1067  << " payload objects" << std::endl;
1068  if (nbuf>0) {
1069  if (m_debug) std::cout << "Final buffer flush at " << nobj <<
1070  "," << nbuf << std::endl;
1071  destflc->flushStorageBuffer();
1072  }
1073  } else {
1074  // branch for pure COOL objects
1075  cool::IObjectIteratorPtr sourceitr;
1076  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1077  sourceitr=sourcefl->browseObjects(since,until,m_chansel);
1078  } else {
1079  sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
1080  }
1081  while (sourceitr->goToNext()) {
1082  const cool::IObject& obj=sourceitr->currentRef();
1083  // check object should not be skipped as spreading outside select-IOV
1084  if (m_skipout && (obj.since()<m_srunemin || obj.until()>m_srunemax)) {
1085  ++nskip;
1086  continue;
1087  }
1088  // check object should not be skipped as excluded channel
1089  if (!m_excludechans.empty()) {
1090  if (find(m_excludechans.begin(),m_excludechans.end(),
1091  obj.channelId())!=m_excludechans.end()) {
1092  ++nskip;
1093  continue;
1094  }
1095  }
1096  if (nbuf==0) {
1097  if (m_debug) std::cout << "Setup new storage buffer at object " <<
1098  nobj << std::endl;
1099  destfl->setupStorageBuffer();
1100  }
1101  cool::ValidityKey newsince;
1102  cool::ValidityKey newuntil;
1103  adjustIOVs(obj.since(),obj.until(),since,until,newsince,newuntil,
1104  timestamp);
1105  // skip negative/zero IOV lengths
1106  if (newsince>=newuntil) {
1107  std::cout << "WARNING: Skipping IOV with since " << newsince <<
1108  ">= until" << newuntil << std::endl;
1109  ++nbad;
1110  continue;
1111  }
1112  if (checkrefs) checkRef(obj.payload(),folder,sourcetag);
1113  if (prot) {
1114  // check existing data in destination DB to ensure no clash
1115  if (m_applocksv) {
1116  // appendlockedsv mode - all IOV ends must be infinite
1117  // and new since must be greater than old since
1118  if (newuntil!=cool::ValidityKeyMax) {
1119  std::cout << "New IOVs must have until=cool::ValidityKeyMax" << std::endl;
1120  throw cool::Exception("Illegal insert over locked IOV",
1121  "AtlCoolCopy");
1122  }
1123  cool::IObjectIteratorPtr checkitr=destfl->browseObjects(newsince,
1124  newuntil,obj.channelId(),desttag);
1125  while (checkitr->goToNext()) {
1126  const cool::IObject& checkobj=checkitr->currentRef();
1127  if (checkobj.since()>=newsince) {
1128  std::cout << "ERROR:: Attempt to insert SV overlapping IOV whilst appending to locked tag" << std::endl;
1129  throw cool::Exception("Illegal insert over locked IOV",
1130  "AtlCoolCopy");
1131  }
1132  if (checkobj.until()!=cool::ValidityKeyMax) {
1133  std::cout << "Existing IOVs must have until=cool::ValidityKeyMax" << std::endl;
1134  throw cool::Exception("Illegal insert over locked IOV",
1135  "AtlCoolCopy");
1136  }
1137  }
1138  checkitr->close();
1139  } else if (m_applock) {
1140  // appendlocked mode - just make sure no objects in this IOV
1141  const unsigned int nexist=
1142  destfl->countObjects(newsince,newuntil,obj.channelId(),desttag);
1143  if (nexist>0) {
1144  std::cout << "ERROR: Attempt to insert IOV over " << nexist <<
1145  " objects whilst appending to locked tag" << std::endl;
1146  throw cool::Exception("Illegal insert over locked IOV",
1147  "AtlCoolCopy");
1148  }
1149  } else if (updatemode==2) {
1150  // UPD2 tag - make sure run is on list of runs open for update
1151  // only for run/lumi based folders
1152  if (timestamp)
1153  throw cool::Exception(
1154  "Attempt to insert into locked UPD2 tag with timestamp format",
1155  "AtlCoolCopy");
1156  unsigned int run1=newsince >> 32;
1157  unsigned int run2=(newuntil-1) >> 32;
1158  for (unsigned int irun=run1;irun<=run2;++irun) {
1159  if (!std::binary_search(m_runlist.begin(),m_runlist.end(),irun))
1160  {
1161  std::cout << "Run " << irun << " from range [" << run1 << ","
1162  << run2 << "] not found in runfile list" << std::endl;
1163  throw cool::Exception("Illegal insert over locked IOV",
1164  "AtlCoolCopy");
1165  }
1166  }
1167  }
1168  // updatemode==1 will fall through, but no checks are done here
1169  }
1170  if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1171  // cool vector payload object copy
1172  cool::IRecordIterator& pitr=obj.payloadIterator();
1173  const cool::IRecordVectorPtr vptr=pitr.fetchAllAsVector();
1174  destfl->storeObject(newsince,newuntil,
1175  *vptr,obj.channelId(),localtag,
1176  (!m_userupdatehead && !localtag.empty()));
1177  nbuf+=vptr->size();
1178  pitr.close();
1179  } else {
1180  // standard COOL object copy
1181  destfl->storeObject(newsince,newuntil,
1182  obj.payload(),obj.channelId(),localtag,
1183  (!m_userupdatehead && !localtag.empty()));
1184  ++nbuf;
1185  }
1186  // flush buffer every m_bufsize objects
1187  if (nbuf>=m_bufsize) {
1188  if (m_debug) std::cout << "Flush buffer after " << nobj << "," <<
1189  nbuf << " objects " << std::endl;
1190  destfl->flushStorageBuffer();
1191  nbuf=0;
1192  }
1193  ++nobj;
1194  }
1195  // finished with the iterator
1196  sourceitr->close();
1197  if (nbuf>0) {
1198  if (m_debug) std::cout << "Final buffer flush at " << nobj <<
1199  "," << nbuf << std::endl;
1200  destfl->flushStorageBuffer();
1201  } // end of COOL-specific part
1202  }
1203  std::cout << "Folder copied with " << nobj << " objects" << std::endl;
1204  if (nskip>0) std::cout << nskip <<
1205  " objects were skipped extending outside IOV selection or excluded channel"
1206  << std::endl;
1207  if (nbad>0) std::cout << nbad <<
1208  " objects were skipped having zero or negative IOV lengths" << std::endl;
1209  }
1210  // exceptions thrown from insert loop (both COOL or CORACOOL)
1211  catch (cool::Exception& e) {
1212  std::cout << "Exception thrown from copy loop: " << e.what() <<
1213  std::endl;
1214  if (relock) {
1215  std::cout << "Relocking destination tag " << desttag << std::endl;
1216  destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1217  }
1218  return 31;
1219  }
1220  // check if tag needs to be relocked
1221  if (relock) {
1222  std::cout << "Relocking destination tag " << desttag << std::endl;
1223  destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1224  }
1225 
1226  if (dvermode==cool::FolderVersioning::MULTI_VERSION && desttag!="HEAD") {
1227  if (!m_usertags) {
1228  // apply HEAD style tagging
1229  std::cout << "Tag folder with HEAD-style tagging for tag: " << desttag
1230  << std::endl;
1231  try {
1232  destfl->tagCurrentHead(desttag,sourcefl->tagDescription(sourcetag));
1233  }
1234  catch (cool::Exception& e) {
1235  std::cout << "Exception thrown in HEAD-style folder-tag: " <<
1236  e.what() << std::endl;
1237  return 32;
1238  }
1239  }
1240  // record information about this tag for later
1241  if (m_copytaginfo) m_cooltagmap.insert(
1242  CoolTagMap::value_type(desttag,CoolTagInfo(m_sourceDbPtr,folder,
1243  destfolder,sourcetag,desttag,m_taglabel)));
1244  // check if any hierarchical tags reference the just-copied tag
1245  if (!m_hitagmap.empty()) {
1246  for (HiTagMap::const_iterator imap=m_hitagmap.begin();
1247  imap!=m_hitagmap.end();++imap) {
1248  try {
1249  if (sourcefl->findTagRelation(imap->first)==sourcetag) {
1250  std::cout << "Create hierarchical tag between " << desttag <<
1251  " and " << imap->first << " in folder " << imap->second <<
1252  std::endl;
1253  // first check this relation has not already been created
1254  try {
1255  std::string etag=destfl->resolveTag(imap->first);
1256  if (etag==desttag) {
1257  std::cout << "This relation has already been created" << std::endl;
1258  } else {
1259  std::cout << "ERROR: Tag in parent already related to " <<
1260  desttag << std::endl;
1261  }
1262  }
1263  catch (cool::Exception& e ) {
1264  // only do creation if does not already exist - via exception
1265  try {
1266  destfl->createTagRelation(imap->first,desttag);
1267  // check if this implicates a new parent folder
1268  // to be checked for further tags up the tree
1269  if (find(m_hiparent.begin(),m_hiparent.end(),imap->second)==
1270  m_hiparent.end()) m_hiparent.push_back(imap->second);
1271  // store the original tag information to transfer
1272  // properties later
1273  if (m_copytaginfo &&
1274  m_cooltagmap.find(imap->first)==m_cooltagmap.end())
1275  m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
1276  CoolTagInfo(m_sourceDbPtr,imap->second,imap->second,
1277  imap->first,imap->first)));
1278  }
1279  catch (cool::Exception& e) {
1280  std::cout << "Cool exception " << e.what() <<
1281  "thrown in hierarchical tag creation" << std::endl;
1282  return 34;
1283  }
1284  }
1285  }
1286  }
1287  catch (cool::Exception& e) {
1288  }
1289  }
1290  }
1291  }
1292  return 0;
1293 }
1294 
1295 void AtlCoolCopy::adjustIOVs(const cool::ValidityKey& since,
1296  const cool::ValidityKey& until,
1297  const cool::ValidityKey& qsince,
1298  const cool::ValidityKey& quntil,
1299  cool::ValidityKey& newsince,
1300  cool::ValidityKey& newuntil,
1301  bool timestamp) const {
1302  // set newsince/until to since/until
1303  // but doing any required truncation/adjustment
1304  newsince=since;
1305  newuntil=until;
1306  if (m_alliov) {
1307  // make IOV cover specified range
1308  if (timestamp) {
1309  newsince=m_newtimemin;
1310  newuntil=m_newtimemax;
1311  } else {
1312  newsince=m_newrunemin;
1313  newuntil=m_newrunemax;
1314  }
1315  }
1316  // check for IOV truncation to range of query
1317  if (m_truncate) {
1318  if (newsince<qsince) newsince=qsince;
1319  if (newuntil>quntil) newuntil=quntil;
1320  }
1321  // check not requesting an IOV with negative length
1322  if (newuntil<newsince) throw cool::Exception(
1323  "Attempt to insert IOV with -ve length","AtlCoolCopy::adjustIOVs");
1324 }
1325 
1326 int AtlCoolCopy::nocopyIOVs(const std::string& folder,
1327  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
1328  const cool::ValidityKey since,const cool::ValidityKey until,
1329  bool checkrefs) {
1330 
1331  std::cout << "Reading tag " << sourcetag << " of folder " << folder <<
1332  " (no copy)" << std::endl;
1333  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
1334  int nobj=0;
1335  cool::IObjectIteratorPtr sourceitr;
1336  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1337  sourceitr=sourcefl->browseObjects(since,until,m_chansel);
1338  } else {
1339  sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
1340  }
1341  try {
1342  while (sourceitr->goToNext()) {
1343  const cool::IObject& obj=sourceitr->currentRef();
1344  if (checkrefs) checkRef(obj.payload(),folder,sourcetag);
1345  ++nobj;
1346  }
1347  std::cout << "Folder scanned with " << nobj << " objects" << std::endl;
1348  }
1349  // exceptions thrown from read loop
1350  catch (cool::Exception& e) {
1351  std::cout << "Exception thrown from read loop: " << e.what() <<
1352  std::endl;
1353  return 33;
1354  }
1355  // finished with the iterator
1356  sourceitr->close();
1357  return 0;
1358 }
1359 
1360 int AtlCoolCopy::verifyIOVs(const std::string& folder,
1361  const cool::IFolderPtr& sourcefl,const CoraCoolFolderPtr& sourceflc,
1362  const cool::IFolderPtr& destfl,const CoraCoolFolderPtr& destflc,
1363  const std::string& sourcetag,
1364  const cool::ValidityKey since,const cool::ValidityKey until,
1365  const bool checkrefs,const bool iscora,
1366  const cool::PayloadMode::Mode paymode) {
1367 
1368  std::cout << "Verifying tag " << sourcetag << " of folder " << folder <<
1369  std::endl;
1370  destfl->setPrefetchAll(false);
1371  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
1372  int nobj=0;
1373  cool::IObjectIteratorPtr sourceitr,destitr;
1374  CoraCoolObjectIterPtr csourceitr,cdestitr;
1375  std::string tag=sourcetag;
1376  if (vermode==cool::FolderVersioning::SINGLE_VERSION || sourcetag=="HEAD")
1377  tag="";
1378  try {
1379  if (iscora) {
1380  sourceflc->setPrefetchAll(false);
1381  csourceitr=sourceflc->browseObjects(since,until,m_chansel,tag);
1382  cdestitr=destflc->browseObjects(since,until,m_chansel,tag);
1383  } else {
1384  sourceitr=sourcefl->browseObjects(since,until,m_chansel,tag);
1385  destitr=destfl->browseObjects(since,until,m_chansel,tag);
1386  }
1387  }
1388  catch (std::exception& e) {
1389  std::cout << "Exception thrown from verify iterator setup: " << e.what() <<
1390  std::endl;
1391  return 105;
1392  }
1393  try {
1394  // the check algorithm assumes the two databases will give back the
1395  // results of the query in the same order, which is currently the
1396  // case in COOL 1.3. If this changes, the verification will be more complex
1397  if (iscora) {
1398  const std::string& cfkey=sourceflc->coralFKey();
1399  const std::string& cpkey=sourceflc->coralPKey();
1400  while (csourceitr->hasNext()) {
1401  CoraCoolObjectPtr sobj=csourceitr->next();
1402  // check object should not be skipped as excluded channel
1403  if (!m_excludechans.empty()) {
1404  if (find(m_excludechans.begin(),m_excludechans.end(),
1405  sobj->channelId())!=m_excludechans.end()) {
1406  continue;
1407  }
1408  }
1409  if (cdestitr->hasNext()) {
1410  CoraCoolObjectPtr dobj=cdestitr->next();
1411  // check objects are the same
1412  int iret=0;
1413  // check IOV equality
1414  if (sobj->since()!=dobj->since() || sobj->until()!=dobj->until())
1415  iret=100;
1416  // check channel equality
1417  if (sobj->channelId()!=dobj->channelId())
1418  iret=101;
1419  // check size of payloads
1420  if (sobj->size()!=dobj->size()) {
1421  std::cout << "ERROR CoraCool object " << nobj <<
1422  " sizes do not match: " << sobj->size() << " " << dobj->size()
1423  << std::endl;
1424  iret=102;
1425  }
1426  // check payload equality (does not check attribute names)
1427  CoraCoolObject::const_iterator ditr=dobj->begin();
1428  for (CoraCoolObject::const_iterator sitr=sobj->begin();
1429  sitr!=sobj->end();++sitr,++ditr) {
1430  // loop over all the attributes, in order to skip PK and FK
1431  for (coral::AttributeList::const_iterator aitr=sitr->begin();
1432  aitr!=sitr->end();++aitr) {
1433  const std::string& aname=aitr->specification().name();
1434  if (aname!=cfkey && aname!=cpkey) {
1435  try {
1436  if (*aitr!=(*ditr)[aname]) {
1437  std::cout << "ERROR Values of attriute " << aname <<
1438  " differ" << std::endl;
1439  iret=102;
1440  }
1441  }
1442  catch (coral::AttributeListException& e) {
1443  std::cout << "ERROR: CoraCool attribute " << aname <<
1444  " not found in destination!" << std::endl;
1445  iret=102;
1446  }
1447  }
1448  }
1449  }
1450  if (iret!=0) {
1451  std::cout << "ERROR database entries do not match: since: " <<
1452  sobj->since() << "," << dobj->since() << " until: " <<
1453  sobj->until() << "," << dobj->until() << " channel: " <<
1454  sobj->channelId() << "," << dobj->channelId() << std::endl;
1455  return iret;
1456  }
1457  } else {
1458  std::cout <<
1459  "ERROR destination folder no matching CoraCool iterator for object "
1460  << nobj << std::endl;
1461  return 103;
1462  }
1463  ++ nobj;
1464  }
1465  } else {
1466  while (sourceitr->goToNext()) {
1467  const cool::IObject& sobj=sourceitr->currentRef();
1468  // check object should not be skipped as excluded channel
1469  if (!m_excludechans.empty()) {
1470  if (find(m_excludechans.begin(),m_excludechans.end(),
1471  sobj.channelId())!=m_excludechans.end()) {
1472  continue;
1473  }
1474  }
1475  if (checkrefs) checkRef(sobj.payload(),folder,sourcetag);
1476  if (destitr->goToNext()) {
1477  const cool::IObject& dobj=destitr->currentRef();
1478  // check objects are the same
1479  int iret=0;
1480  // check IOV equality
1481  if (sobj.since()!=dobj.since() || sobj.until()!=dobj.until())
1482  iret=100;
1483  // check channel equality
1484  if (sobj.channelId()!=dobj.channelId())
1485  iret=101;
1486  // check payload equality (does not check attribute names)
1487  // do not use cool::IRecord equality operator as does not correctly
1488  // handle null values
1489  if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1490  cool::IRecordIterator& spitr=sobj.payloadIterator();
1491  const cool::IRecordVectorPtr svptr=spitr.fetchAllAsVector();
1492  cool::IRecordIterator& dpitr=dobj.payloadIterator();
1493  const cool::IRecordVectorPtr dvptr=dpitr.fetchAllAsVector();
1494  if (svptr->size()!=dvptr->size()) {
1495  iret=102;
1496  } else {
1497  // loop through the payloads, checking equality - assumes
1498  // order is significant and same in source and destination DBs
1499  cool::IRecordVector::const_iterator svitr=svptr->begin();
1500  cool::IRecordVector::const_iterator svend=svptr->end();
1501  cool::IRecordVector::const_iterator dvitr=dvptr->begin();
1502  for (;svitr!=svend;++svitr,++dvitr) {
1503  if (!equalRecord(**svitr,**dvitr)) iret=102;
1504  }
1505  }
1506  if (iret!=0) {
1507  std::cout << "ERROR vector payloads do not match (size " << svptr->size() << "," << dvptr->size() << ")" << std::endl;
1508  }
1509 
1510  } else {
1511  // standard COOL folder - simple check of payloads
1512  if (!equalRecord(sobj.payload(),dobj.payload())) iret=102;
1513  }
1514 
1515  if (iret!=0) {
1516  std::cout << "ERROR database entries do not match: since: " <<
1517  sobj.since() << "," << dobj.since() << " until: " <<
1518  sobj.until() << "," << dobj.until() << " channel: " <<
1519  sobj.channelId() << "," << dobj.channelId() << std::endl;
1520  std::cout << "Source payload:" << std::endl;
1521  sobj.payload().attributeList().toOutputStream(std::cout);
1522  std::cout << std::endl << "Destination payload:" << std::endl;
1523  dobj.payload().attributeList().toOutputStream(std::cout);
1524  std::cout << std::endl;
1525  return iret;
1526  }
1527  } else {
1528  std::cout <<
1529  "ERROR destination folder no matching iterator for object "
1530  << nobj << std::endl;
1531  return 103;
1532  }
1533  ++nobj;
1534  }
1535  }
1536  }
1537  catch (cool::Exception& e) {
1538  std::cout << "Exception thrown from verify loop: " << e.what() <<
1539  std::endl;
1540  return 104;
1541  }
1542  // finished with the iterators
1543  if (iscora) {
1544  } else {
1545  sourceitr->close();
1546  destitr->close();
1547  }
1548  std::cout << "Verification of folder " << folder << " tag " <<
1549  sourcetag << " OK (" << nobj << " objects)" << std::endl;
1550  return 0;
1551 }
1552 
1553 
1554 
1555 int AtlCoolCopy::rootIOVs(const std::string& folder,
1556  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
1557  const cool::ValidityKey since,const cool::ValidityKey until,
1558  const bool timestamp) {
1559  // copy this selection to ROOT
1560  std::cout << "Write tag " << sourcetag << " of folder " << folder <<
1561  " to ROOT file" << std::endl;
1562  // create the directory structure - top directory COOL
1563  std::string treename=rootDirs(folder,"COOL");
1564  bool timestamp2=timestamp;
1565  if (m_forcetime) timestamp2=true;
1566  if (m_forcerune) timestamp2=false;
1567  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
1568  // create TTree with appropriate structure - only if it does not exist
1569  // could have been created from a previous tag in the same folder
1570  TTree* tree=static_cast<TTree*>(gDirectory->FindObject(treename.c_str()));
1571  if (tree==nullptr) {
1572  std::cout << "Book TTree " << treename << std::endl;
1573  tree=new TTree(treename.c_str(),"COOL datadump");
1574  if (timestamp2) {
1575  tree->Branch("IOVSince",&m_nt_since,"IOVSince/l");
1576  tree->Branch("IOVUntil",&m_nt_until,"IOVUntil/l");
1577  } else {
1578  tree->Branch("RunSince",&m_nt_runsince,"RunSince/i");
1579  tree->Branch("RunUntil",&m_nt_rununtil,"RunUntil/i");
1580  tree->Branch("LBSince",&m_nt_lbsince,"LBSince/i");
1581  tree->Branch("LBUntil",&m_nt_lbuntil,"LBUntil/i");
1582  }
1583  tree->Branch("ChannelID",&m_nt_channel,"ChannelID/i");
1584  if (vermode==cool::FolderVersioning::MULTI_VERSION)
1585  tree->Branch("TagID",m_nt_tagid,"TagID/C");
1586  // now loop through specification, creating payload buffer and tree
1587  const cool::IRecordSpecification& spec=
1588  (sourcefl->folderSpecification()).payloadSpecification();
1589  unsigned int ncolumns=spec.size();
1590  // clear the buffer of pointers - note this leaks the memory of the
1591  // previous buffers, but to do this properly would have to remember
1592  // the type of each object and delete appropriately
1593  m_nt_treename=std::move(treename);
1594  m_nt_bufferptr.clear();
1595  for (unsigned int icol=0;icol<ncolumns;++icol) {
1596  const cool::IFieldSpecification& fieldspec=spec[icol];
1597  void* ptr=nullptr;
1598  char rootID;
1599  if (rootAllocate(fieldspec,ptr,rootID)) {
1600  tree->Branch(fieldspec.name().c_str(),ptr,
1601  (fieldspec.name()+"/"+rootID).c_str());
1602  if (m_debug) std::cout << "Defining column for " << spec[icol].name()
1603  << " of type " << spec[icol].storageType().name() << std::endl;
1604  } else {
1605  std::cout << "Attribute " << spec[icol].name() << " of type " <<
1606  spec[icol].storageType().name() << " will be skipped" << std::endl;
1607  }
1608  m_nt_bufferptr.push_back(ptr);
1609  }
1610  } else {
1611  std::cout << "TTree " << treename << " already exists" << std::endl;
1612  // check we have seen and defined this tree
1613  unsigned int size=
1614  (sourcefl->folderSpecification()).payloadSpecification().size();
1615  if (treename!=m_nt_treename || size!=m_nt_bufferptr.size()) {
1616  std::cout << "ERROR in tree buffer definition: expect " << treename <<
1617  " size " << size << " but buffer has " << m_nt_treename << " size "
1618  << m_nt_bufferptr.size() << std::endl;
1619  return 123;
1620  }
1621  }
1622 
1623  int nobj=0;
1624  int nex=0;
1625  cool::IObjectIteratorPtr sourceitr;
1626  try {
1627  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1628  sourceitr=sourcefl->browseObjects(since,until,m_chansel);
1629  } else {
1630  sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
1631  }
1632  }
1633  catch (cool::Exception& e) {
1634  std::cout << "Exception thrown from ROOT copy iterator setup: " <<
1635  e.what() << std::endl;
1636  return 125;
1637  }
1638  // now loop over all IOVs to copy them
1639  try {
1640  while (sourceitr->goToNext()) {
1641  const cool::IObject& sobj=sourceitr->currentRef();
1642  if (timestamp2) {
1643  m_nt_since=sobj.since();
1644  m_nt_until=sobj.until();
1645  } else {
1646  m_nt_runsince=(sobj.since() >> 32);
1647  m_nt_rununtil=(sobj.until() >> 32);
1648  m_nt_lbsince=(sobj.since() & 0xFFFFFFFF);
1649  m_nt_lbuntil=(sobj.until() & 0xFFFFFFFF);
1650  }
1651  m_nt_channel=sobj.channelId();
1652  if (vermode==cool::FolderVersioning::MULTI_VERSION) {
1653  // truncate the string first to avoid coverity complaining about
1654  // potential buffer overruns
1655  std::string sourcetag2=sourcetag.substr(0,255);
1656  strncpy(m_nt_tagid,sourcetag2.c_str(),sizeof(m_nt_tagid)-1);
1657  }
1658  // loop over the payload elements and fill the ones for which buffers
1659  // are defined
1660  try {
1661  const cool::IRecord& record=sobj.payload();
1662  for (unsigned int icol=0;icol<record.size();++icol) {
1663  if (m_nt_bufferptr[icol]!=nullptr) rootWrite(m_nt_bufferptr[icol],record[icol]);
1664  }
1665  tree->Fill();
1666  ++nobj;
1667  }
1668  catch (cool::Exception& e) {
1669  // can get exceptions due to NULL values
1670  // catch them and continue...
1671  ++nex;
1672  }
1673  }
1674  sourceitr->close();
1675  std::cout << "Written " << nobj << " objects to ROOT TTree with " << nex
1676  << " nulls/exceptions" << std::endl;
1677  }
1678  catch (cool::Exception& e) {
1679  std::cout << "Exception thrown from ROOT file writing loop: " <<
1680  e.what() << std::endl;
1681  return 124;
1682  }
1683  return 0;
1684 }
1685 
1686 std::string AtlCoolCopy::rootDirs(const std::string& folder,
1687  const std::string& toproot) {
1688  // create a ROOT directory structure start with toproot (=COOL or COOLANA)
1689  // and cd to the directory where a tree should be created
1690  // return the tree name (=leaf of the COOL foldername)
1691  p_rootfile->cd();
1692  if (p_rootfile->FindObject(toproot.c_str())==nullptr) {
1693  p_rootfile->mkdir(toproot.c_str());
1694  std::cout << "Made top directory " << toproot << std::endl;
1695  }
1696  gDirectory->cd(toproot.c_str());
1697  // now parse the COOL folder name to create intermediate directories
1698  // assume folder names begin with '/', to be skipped
1699  std::string::size_type iofs1=1;
1700  std::string::size_type iofs2=1;
1701  std::string treename;
1702  while (iofs2!=std::string::npos) {
1703  iofs2=folder.find('/',iofs1);
1704  if (iofs2==std::string::npos) {
1705  // no more /, so current part of string is leaf tree name
1706  treename=folder.substr(iofs1);
1707  } else {
1708  std::string dirname=folder.substr(iofs1,iofs2-iofs1);
1709  iofs1=iofs2+1;
1710  if (gDirectory->FindObject(dirname.c_str())==nullptr) {
1711  std::cout << "Make directory " << dirname << std::endl;
1712  gDirectory->mkdir(dirname.c_str());
1713  }
1714  gDirectory->cd(dirname.c_str());
1715  }
1716  }
1717  return treename;
1718 }
1719 
1720 bool AtlCoolCopy::rootAllocate(const cool::IFieldSpecification& spec,
1721  void*& sptr,char& rootID) const {
1722  const cool::StorageType& stype=spec.storageType();
1723  rootID=' ';
1724  sptr=nullptr;
1725  if (stype==cool::StorageType::Bool) {
1726  // map bool to uint32 for now
1727  sptr=static_cast<void*>(new unsigned int(0));
1728  rootID='i';
1729  } else if (stype==cool::StorageType::UChar) {
1730  sptr=static_cast<void*>(new unsigned char(' '));
1731  rootID='C';
1732  } else if (stype==cool::StorageType::Int16) {
1733  sptr=static_cast<void*>(new short(0));
1734  rootID='S';
1735  } else if (stype==cool::StorageType::UInt16) {
1736  sptr=static_cast<void*>(new unsigned short(0));
1737  rootID='s';
1738  } else if (stype==cool::StorageType::Int32) {
1739  sptr=static_cast<void*>(new int(0));
1740  rootID='I';
1741  } else if (stype==cool::StorageType::UInt32) {
1742  sptr=static_cast<void*>(new unsigned int(0));
1743  rootID='i';
1744  } else if (stype==cool::StorageType::Int64) {
1745  sptr=static_cast<void*>(new long long int(0));
1746  rootID='L';
1747  } else if (stype==cool::StorageType::UInt63) {
1748  sptr=static_cast<void*>(new unsigned long long int(0));
1749  rootID='l';
1750  } else if (stype==cool::StorageType::Float) {
1751  sptr=static_cast<void*>(new float(0.));
1752  rootID='F';
1753  } else if (stype==cool::StorageType::Double) {
1754  sptr=static_cast<void*>(new double(0.));
1755  rootID='D';
1756  } else if (stype==cool::StorageType::String4k ||
1757  stype==cool::StorageType::String255) {
1758  sptr=static_cast<void*>(new char[4100]);
1759  rootID='C';
1760  } else if (stype==cool::StorageType::String64k && !m_noclobroot) {
1761  sptr=static_cast<void*>(new char[65536]);
1762  rootID='C';
1763  } else if (stype==cool::StorageType::String16M && !m_noclobroot) {
1764  sptr=static_cast<void*>(new char[67108864]);
1765  rootID='C';
1766  } else if (stype==cool::StorageType::Blob64k && !m_noclobroot) {
1767  sptr=static_cast<void*>(new char[65536]);
1768  rootID='C';
1769  } else {
1770  std::cout << "rootAllocate: Unsupported storage type for attribute: " <<
1771  spec.name() << std::endl;
1772  }
1773  return (sptr!=nullptr);
1774 }
1775 
1776 void AtlCoolCopy::rootWrite(void* sptr,const cool::IField& field) const {
1777  // check for NULL value
1778  bool recnull=(field.isNull() && m_zeronull);
1779  const cool::StorageType& stype=field.storageType();
1780  if (stype==cool::StorageType::Bool) {
1781  if (recnull) {
1782  *(static_cast<int*>(sptr))=0;
1783  } else {
1784  *(static_cast<int*>(sptr))=(field.data<bool>() ? 1 : 0);
1785  }
1786  } else if (stype==cool::StorageType::UChar) {
1787  if (recnull) {
1788  *(static_cast<unsigned char*>(sptr))=0;
1789  } else {
1790  *(static_cast<unsigned char*>(sptr))=field.data<unsigned char>();
1791  }
1792  } else if (stype==cool::StorageType::Int16) {
1793  if (recnull) {
1794  *(static_cast<short*>(sptr))=0;
1795  } else {
1796  *(static_cast<short*>(sptr))=field.data<short>();
1797  }
1798  } else if (stype==cool::StorageType::UInt16) {
1799  if (recnull) {
1800  *(static_cast<unsigned short*>(sptr))=0;
1801  } else {
1802  *(static_cast<unsigned short*>(sptr))=field.data<unsigned short>();
1803  }
1804  } else if (stype==cool::StorageType::Int32) {
1805  if (recnull) {
1806  *(static_cast<int*>(sptr))=0;
1807  } else {
1808  *(static_cast<int*>(sptr))=field.data<int>();
1809  }
1810  } else if (stype==cool::StorageType::UInt32) {
1811  if (recnull) {
1812  *(static_cast<unsigned int*>(sptr))=0;
1813  } else {
1814  *(static_cast<unsigned int*>(sptr))=field.data<unsigned int>();
1815  }
1816  } else if (stype==cool::StorageType::Int64) {
1817  if (recnull) {
1818  *(static_cast<long long*>(sptr))=0;
1819  } else {
1820  *(static_cast<long long*>(sptr))=field.data<long long>();
1821  }
1822  } else if (stype==cool::StorageType::UInt63) {
1823  if (recnull) {
1824  *(static_cast<unsigned long long*>(sptr))=0;
1825  } else {
1826  *(static_cast<unsigned long long*>(sptr))=field.data<unsigned long long>();
1827  }
1828  } else if (stype==cool::StorageType::Float) {
1829  if (recnull) {
1830  *(static_cast<float*>(sptr))=0.;
1831  } else {
1832  *(static_cast<float*>(sptr))=field.data<float>();
1833  }
1834  } else if (stype==cool::StorageType::Double) {
1835  if (recnull) {
1836  *(static_cast<double*>(sptr))=0.;
1837  } else {
1838  *(static_cast<double*>(sptr))=field.data<double>();
1839  }
1840  } else if (stype==cool::StorageType::String255 ||
1841  stype==cool::StorageType::String4k ||
1842  stype==cool::StorageType::String64k ||
1843  stype==cool::StorageType::String16M) {
1844  if (recnull) {
1845  strcpy(static_cast<char*>(sptr),"NULL");
1846  } else {
1847  strcpy(static_cast<char*>(sptr),field.data<std::string>().c_str());
1848  }
1849  } else if (stype==cool::StorageType::Blob64k && !m_noclobroot) {
1850  if (recnull) {
1851  strcpy(static_cast<char*>(sptr),"NULL");
1852  } else {
1853  auto blob = field.data<coral::Blob>();
1854  std::string blobStr((char*)blob.startingAddress(), blob.size());
1855  strcpy(static_cast<char*>(sptr), blobStr.c_str());
1856  }
1857  } else {
1858  std::cout << "ERROR: Unknown storage type in rootWrite!" << std::endl;
1859  }
1860 }
1861 
1862 int AtlCoolCopy::analyseIOVs(const std::string& folder,
1863  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
1864  const cool::ValidityKey since,const cool::ValidityKey until,
1865  const bool timestamp) {
1866  // analyse the IOV and channel structure of this folder/tag, make ROOT histos
1867  std::cout << "Analyse tag " << sourcetag << " of folder " << folder <<
1868  std::endl;
1869  bool anatime=(m_anadelt>=0);
1870  if (anatime) {
1871  std::cout << "Analyse time structures with tolerance of " <<
1872  m_anadelt/1.E9 << " seconds " << std::endl;
1873  }
1874  // create COOLANA/folder structure and create leaf directory for histograms
1875  std::string dirname=rootDirs(folder,"COOLANA");
1876  if (gDirectory->FindObject(dirname.c_str())==nullptr) {
1877  std::cout << "Make directory " << dirname << std::endl;
1878  gDirectory->mkdir(dirname.c_str());
1879  }
1880  gDirectory->cd(dirname.c_str());
1881  // get channels
1882  std::vector<cool::ChannelId> channels=sourcefl->listChannels();
1883  unsigned int nchan=channels.size();
1884  // end of last IOV seen on this channel
1885  long long* lastiov=new long long[nchan];
1886  long long* iovtotlen=new long long[nchan];
1887  long long* iovtotgap=new long long[nchan];
1888  int* iovn=new int[nchan];
1889  for (unsigned int i=0;i<nchan;++i) {
1890  lastiov[i]=-1; // no IOV seen on this channel yet
1891  iovtotlen[i]=0;
1892  iovtotgap[i]=0;
1893  iovn[i]=0;
1894  }
1895  cool::ValidityKey globsince=cool::ValidityKeyMax;
1896  cool::ValidityKey globuntil=cool::ValidityKeyMin;
1897 
1898  // book histograms - only if the objects do not already exist (previous tag)
1899  TH1F* h_iovchan=bookOrFindTH1F("IOVSperChannel","IOVs per channel",
1900  nchan,-0.5,nchan-0.5);
1901  TH1F* h_iovname=bookOrFindTH1F("IOVSperChannelName",
1902  "IOVs per channel-name (copy of IOVSperChannel)",
1903  nchan,-0.5,nchan-0.5);
1904  TH1F* h_iovlength=bookOrFindTH1F("IOVLogLength","IOV log10 length",
1905  100,0.,20.);
1906  TH1F* h_iovgap=bookOrFindTH1F("IOVLogGapLength","IOV log10 gap length",
1907  100,0.,20.);
1908  // histograms for time analysis if needed
1909  TH1F* h_anadelt=nullptr;
1910  TH1F* h_chandelt=nullptr;
1911  if (anatime) {
1912  h_anadelt=bookOrFindTH1F("IOVAlignLogDelT","IOV alignment log deltaT",
1913  100,0.,20.);
1914  // add factor 2 in number of channels, to catch those where a channel
1915  // changes many times within the m_anadelt window (i.e. noise/jitter)
1916  h_chandelt=bookOrFindTH1F("AlignedChanPerIOV","Aligned channels per IOV",
1917  nchan*2,0.5,nchan*2-0.5);
1918  }
1919  std::cout << "Booked histograms for folder with " << nchan << " channels"
1920  << std::endl;
1921  // setup storage for IOV time analysis
1922  using IOVTimeMap = std::map<cool::ValidityKey, int>;
1923  IOVTimeMap iov_time_map;
1924 
1925  // setup iterator to loop over objects
1926  int nobj=0;
1927  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
1928  cool::IObjectIteratorPtr sourceitr;
1929  try {
1930  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1931  sourceitr=sourcefl->browseObjects(since,until,m_chansel);
1932  } else {
1933  sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
1934  }
1935  }
1936  catch (cool::Exception& e) {
1937  std::cout << "Exception thrown from analysis copy iterator setup: " <<
1938  e.what() << std::endl;
1939  return 135;
1940  }
1941  // look up names of all the channels in one go
1942  std::map<cool::ChannelId,std::string>
1943  chanmap=sourcefl->listChannelsWithNames();
1944  // now loop over all IOVs and write them
1945  try {
1946  while (sourceitr->goToNext()) {
1947  const cool::IObject& sobj=sourceitr->currentRef();
1948  // get channel and find its index
1949  cool::ChannelId ichanid=sobj.channelId();
1950  const std::string& cname=chanmap[ichanid];
1953  std::equal_range(channels.begin(),channels.end(),ichanid);
1954  if (chanitr.first!=chanitr.second) {
1955  cool::ChannelId ichan=std::distance(channels.begin(),chanitr.first);
1956  Int_t bin=h_iovchan->Fill(ichan);
1957  if (ichan && !h_iovname->GetBinContent(bin)) {
1958  h_iovname->GetXaxis()->SetBinLabel(bin,cname.c_str());
1959  }
1960  h_iovname->Fill(ichan);
1961  // analyse IOV start/stop
1962  cool::ValidityKey since2=sobj.since();
1963  cool::ValidityKey until2=sobj.until();
1964  if (since2<globsince) globsince=since2;
1965  // dont count IOV is open-ended, so length statistics will make sense
1966  if (until2!=cool::ValidityKeyMax) {
1967  if (until2>globuntil) globuntil=until2;
1968  long long len=until2-since2;
1969  h_iovlength->Fill(log10(len));
1970  iovn[ichan]+=1;
1971  iovtotlen[ichan]+=len;
1972  }
1973  if (lastiov[ichan]<static_cast<long long>(since2) &&
1974  lastiov[ichan]>=0) {
1975  // have a gap in the IOV structure for this channel
1976  long long gap=since2-lastiov[ichan];
1977  iovtotgap[ichan]+=gap;
1978  h_iovgap->Fill(log10(gap));
1979  std::cout << "Gap of " << gap << std::endl;
1980  }
1981  if (until2!=cool::ValidityKeyMax) {
1982  lastiov[ichan]=until2;
1983  } else {
1984  lastiov[ichan]=since2;
1985  }
1986  // analyse IOV alignment
1987  if (anatime) {
1988  long long del1=-1;
1989  long long del2=-1;
1990  // find the nearest time to this one
1991  // this gives the first value exceeding since2
1992  IOVTimeMap::iterator hiitr=iov_time_map.lower_bound(since2);
1993  IOVTimeMap::iterator lowitr=hiitr;
1994  if (hiitr!=iov_time_map.end()) {
1995  // del1 is +ve time interval to next one
1996  del1=hiitr->first-since2;
1997  }
1998  if (lowitr!=iov_time_map.begin()) {
1999  // del2 is +ve time interval to previous one
2000  --lowitr;
2001  del2=since2-lowitr->first;
2002  }
2003  long long del=-1;
2004  IOVTimeMap::iterator moditr;
2005  bool domod=false;
2006  if (del1<=m_anadelt && del1>-1) {
2007  moditr=hiitr;
2008  del=del1;
2009  domod=true;
2010  }
2011  if (del2<=m_anadelt && del2>-1 && (del2<del || del==-1)) {
2012  moditr=lowitr;
2013  del=del2;
2014  domod=true;
2015  }
2016  if (domod) {
2017  ++(moditr->second);
2018  if (del>0) {
2019  h_anadelt->Fill(log10(del));
2020  } else {
2021  h_anadelt->Fill(0.);
2022  }
2023  } else {
2024  iov_time_map[since2]=0;
2025  }
2026  }
2027  } else {
2028  // channel ID not found
2029  std::cout << "ERROR :Channel " << ichanid <<
2030  " not found in channel list - ignored" << std::endl;
2031  }
2032  ++nobj;
2033  }
2034  std::cout << "Finished analysis with " << nobj << " objects" << std::endl;
2035  if (timestamp) {
2036  std::cout << "IOV timestamp range: " << globsince << " " <<
2037  timeString(globsince) << "to " << globuntil << " " <<
2038  timeString(globuntil) << std::endl;
2039  } else {
2040  std::cout << "IOV run/LB range [" << (globsince >> 32) << "," <<
2041  (globsince & 0xFFFFFFFF) << "] to [" << (globuntil >> 32) << "," <<
2042  (globuntil & 0xFFFFFFFF) << "]" << std::endl;
2043  }
2044  // calculate statistics for each channel
2045  TH1F* h_iovoccchan=bookOrFindTH1F("ChannelOcc","Channel Occupancy",
2046  nchan,-0.5,nchan-0.5);
2047  TH1F* h_iovlenchan=bookOrFindTH1F("IOVLenChan",
2048  "Mean IOV length per channel",nchan,-0.5,nchan-0.5);
2049  for (unsigned int i=0;i<nchan;++i) {
2050  float occ=0.;
2051  float avlen=0.;
2052  if (iovn[i]>0) {
2053  occ=1.-float(iovtotgap[i])/float(iovtotlen[i]+iovtotgap[i]);
2054  avlen=float(iovtotlen[i])/iovn[i];
2055  }
2056  h_iovoccchan->Fill(i,occ);
2057  h_iovlenchan->Fill(i,avlen);
2058  }
2059  if (anatime) {
2060  std::cout << "Alignment analysis: " << iov_time_map.size()
2061  << " seperate IOV starts (tolerance " << m_anadelt/1.E9 <<
2062  " seconds)" << std::endl;
2063  // fill histogram of number of channels incremented per IOV
2064  for (IOVTimeMap::const_iterator itr=iov_time_map.begin();
2065  itr!=iov_time_map.end();++itr) {
2066  h_chandelt->Fill(itr->second);
2067  }
2068  }
2069  }
2070  catch (cool::Exception& e) {
2071  std::cout << "Exception thrown from folder analysis reading loop: " <<
2072  e.what() << std::endl;
2073  return 134;
2074  }
2075  sourceitr->close();
2076  delete [] lastiov;
2077  delete [] iovtotlen;
2078  delete [] iovtotgap;
2079  delete[] iovn; iovn=nullptr;
2080  return 0;
2081 }
2082 
2083 TH1F* AtlCoolCopy::bookOrFindTH1F(const std::string& hID,
2084  const std::string& htitle,
2085  const int nchan, const float xlow, const float xhigh) {
2086  // check histogram does not exist in current directory
2087  // if yes, return pointer to it, else book and return pointer
2088  TObject* obj=gDirectory->FindObject(hID.c_str());
2089  if (obj==nullptr) {
2090  return new TH1F(hID.c_str(),htitle.c_str(),nchan,xlow,xhigh);
2091  } else {
2092  return static_cast<TH1F*>(obj);
2093  }
2094 }
2095 
2096 
2097 int AtlCoolCopy::doCopy ATLAS_NOT_THREAD_SAFE () {
2098  if (isOpen()) {
2099  int retcode=0;
2100  // execute copy for all defined folders
2101  for (std::vector<std::string>::const_iterator ifolder=
2102  m_folderlist.begin();ifolder!=m_folderlist.end();++ifolder) {
2103  int code=copyFolder(*ifolder,m_tags);
2104  if (code>retcode) retcode=code;
2105  }
2106  if (!m_hiparent.empty()) {
2107  // fill in any hierarchical tags for parent folders
2108  int code=tagParents();
2109  if (code>retcode) retcode=code;
2110  }
2111  if (m_copytaginfo) {
2112  // write information (tag description, lock status) to dest tags
2113  int code=writeTagInfo();
2114  if (code>retcode) retcode=code;
2115  }
2116  if (m_checkrefs) {
2117  int code=0;
2118  if (m_poolcat.empty()) {
2119  code=listPoolRefs();
2120  if (!m_checkoutputfile.empty()) filePoolRefs();
2121  } else {
2122  code=resolvePoolRefs();
2123  }
2124  if (code>retcode) retcode=code;
2125  }
2126  if (p_rootfile) {
2127  p_rootfile->Write();
2128  delete p_rootfile;
2129  }
2130  return retcode;
2131  } else {
2132  return 10;
2133  }
2134 }
2135 
2136 bool AtlCoolCopy::procOptVector(const int argc, const char* argv[],
2137  std::vector<std::string>& folders) {
2138  // process an array of options, return True if OK, False if not
2139  int ic=0;
2140  bool error=false;
2141  while (ic<argc) {
2142  int ir=argc-ic;
2143  std::string_view par0=argv[ic];
2144  // strip double "--" to achieve compatability with python-style options
2145  if (par0.compare(0,2,"--")==0) par0=par0.substr(1);
2146  if ((par0=="-f" || par0=="-folder") && ir>1) {
2147  folders.emplace_back(argv[ic+1]);
2148  ++ic;
2149  } else if ((par0=="-e" || par0=="-exclude") && ir>1) {
2150  addExclude(argv[ic+1]);
2151  ++ic;
2152  } else if ((par0=="-t" || par0=="-tag") && ir>1) {
2153  m_tags.emplace_back(argv[ic+1]);
2154  ++ic;
2155  } else if ((par0=="-mt" || par0=="-magic") && ir>1) {
2156  m_magic.push_back("_"+std::string(argv[ic+1]));
2157  ++ic;
2158  } else if ((par0=="-of" || par0=="-outfolder") && ir>1) {
2159  m_outfolder=argv[ic+1];
2160  ++ic;
2161  } else if ((par0=="-ot" || par0=="-outtag") && ir>1) {
2162  m_outtag=argv[ic+1];
2163  ++ic;
2164  } else if ((par0=="-bs" || par0=="-buffersize") && ir>1) {
2165  m_bufsize=atoi(argv[ic+1]);
2166  ++ic;
2167  } else if ((par0=="-sl" || par0=="-seal") && ir>1) {
2168  m_sealmsg=atoi(argv[ic+1]);
2169  ++ic;
2170  } else if ((par0=="-rls" || par0=="-runlumisince") && ir>2) {
2171  m_runemin=runLBVal(argv[ic+1],argv[ic+2]);
2172  ic+=2;
2173  } else if ((par0=="-rlu" || par0=="-runlumiuntil") && ir>2) {
2174  m_runemax=runLBVal(argv[ic+1],argv[ic+2]);
2175  ic+=2;
2176  } else if ((par0=="-nrls" || par0=="-newrunlumisince") && ir>2) {
2178  ic+=2;
2179  } else if ((par0=="-nrlu" || par0=="-newrunlumiuntil") && ir>2) {
2181  ic+=2;
2182  } else if ((par0=="-srls" || par0=="-skiprunlumisince") && ir>2) {
2183  m_srunemin=runLBVal(argv[ic+1],argv[ic+2]);
2184  m_skipout=true;
2185  ic+=2;
2186  } else if ((par0=="-srlu" || par0=="-skiprunlumiuntil") && ir>2) {
2187  m_srunemax=runLBVal(argv[ic+1],argv[ic+2]);
2188  m_skipout=true;
2189  ic+=2;
2190  } else if (par0=="-ro" || par0=="-root") {
2191  m_root=true;
2192  } else if (par0=="-zn" || par0=="-zeronull") {
2193  m_zeronull=true;
2194  } else if (par0=="-ana" || par0=="-analyse") {
2195  m_analyse=true;
2196  float ttime=atof(argv[ic+1])*1.E9;
2197  m_anadelt=static_cast<long long>(ttime);
2198  ++ic;
2199  } else if ((par0=="-rs" || par0=="-runsince") && ir>1) {
2200  m_runemin=(static_cast<long long>(atol(argv[ic+1])) << 32);
2201  ++ic;
2202  } else if ((par0=="-ru" || par0=="-rununtil") && ir>1) {
2203  m_runemax=(static_cast<long long>(1+atol(argv[ic+1])) << 32)-1;
2204  ++ic;
2205  } else if ((par0=="-r" || par0=="-run") && ir>1) {
2206  m_runemin=(static_cast<long long>(atol(argv[ic+1])) << 32);
2207  m_runemax=(static_cast<long long>(1+atol(argv[ic+1])) << 32)-1;
2208  ++ic;
2209  } else if ((par0=="-ts" || par0=="-timesince") && ir>1) {
2210  m_timemin=timeVal(argv[ic+1]);
2211  ++ic;
2212  } else if ((par0=="-tu" || par0=="-timeuntil") && ir>1) {
2213  m_timemax=timeVal(argv[ic+1]);
2214  ++ic;
2215  } else if ((par0=="-nts" || par0=="-newtimesince") && ir>1) {
2217  ++ic;
2218  } else if ((par0=="-ntu" || par0=="-newtimeuntil") && ir>1) {
2220  ++ic;
2221  } else if (par0=="-c" || par0=="-create") {
2222  m_allowcreate=true;
2223  } else if ((par0=="-ch" || par0=="-channel") && ir>1) {
2224  m_channelRange.emplace_back(argv[ic+1]);
2225  ++ic;
2226  } else if ((par0=="-ch1" || par0=="-channel1") && ir>1) {
2227  m_channel1=argv[ic+1];
2228  ++ic;
2229  } else if ((par0=="-ch2" || par0=="-channel2") && ir>1) {
2230  m_channel2=argv[ic+1];
2231  ++ic;
2232  } else if (par0=="-chd" || par0=="-channeldesc") {
2233  m_chdesc=true;
2234  // no abbreviaton for this one - dangerous option
2235  } else if (par0=="-forcerecreate") {
2236  m_recreate=true;
2237  } else if (par0=="-d" || par0=="-debug") {
2238  m_debug=true;
2239  } else if (par0=="-a" || par0=="-alliov") {
2240  m_alliov=true;
2241  } else if (par0=="-ih" || par0=="-includehead") {
2242  m_includehead=true;
2243  } else if (par0=="-eh" || par0=="-excludehead") {
2244  m_excludehead=true;
2245  } else if (par0=="-ht" || par0=="-headtag") {
2246  m_usertags=false;
2247  } else if (par0=="-uht" || par0=="-userheadtag") {
2248  m_userupdatehead=true;
2249  } else if (par0=="-v" || par0=="-verify") {
2250  m_verify=true;
2251  } else if (par0=="-nc" || par0=="-nocopy") {
2252  m_nocopy=true;
2253  } else if (par0=="-noc" || par0=="-nocoracool") {
2254  m_coracool=false;
2255  } else if (par0=="-nch" || par0=="-nochannel") {
2256  m_nochannel=true;
2257  } else if (par0=="-ncr" || par0=="-noclobroot") {
2258  m_noclobroot=true;
2259  } else if (par0=="-nd" || par0=="-nodata") {
2260  m_nodata=true;
2261  } else if (par0=="-nh" || par0=="-nohitag") {
2262  m_nohitag=true;
2263  } else if (par0=="-hi" || par0=="-hitag") {
2264  m_hitag=true;
2265  } else if ((par0=="-ec" || par0=="-excludechannel") && ir>1) {
2266  m_excludechans.push_back(strtoul(argv[ic+1],nullptr,10));
2267  ++ic;
2268  } else if (par0=="-fs" || par0=="-forcesingle") {
2269  m_forcesingle=true;
2270  } else if (par0=="-fm" || par0=="-forcemulti") {
2271  m_forcemulti=true;
2272  } else if (par0=="-frl" || par0=="-forcerunlumi") {
2273  m_forcerune=true;
2274  } else if (par0=="-ftm" || par0=="-forcetime") {
2275  m_forcetime=true;
2276  } else if (par0=="-fp" || par0=="-forcepayload") {
2277  m_forcepay=true;
2278  } else if (par0=="-fnp" || par0=="-forcenopayload") {
2279  m_forcenopay=true;
2280  } else if (par0=="-cr" || par0=="-checkrefs") {
2281  m_checkrefs=true;
2282  } else if (par0=="-lp" || par0=="-listpfn") {
2283  m_listpfn=true;
2284  m_checkrefs=true;
2285  } else if (par0=="-cf" || par0=="-checkfiles") {
2286  m_poolopen=true;
2287  m_checkrefs=true;
2288  } else if (par0=="-pa" || par0=="-poolall") {
2289  m_poolall=true;
2290  } else if ((par0=="-co" || par0=="-checkoutput") && ir>1) {
2292  ++ic;
2293  } else if ((par0=="-pc" || par0=="-poolcat") && ir>1) {
2294  m_poolcat.emplace_back(argv[ic+1]);
2295  ++ic;
2296  } else if ((par0=="-mc" || par0=="-mergecat") && ir>1) {
2297  m_mergecat.emplace_back(argv[ic+1]);
2298  ++ic;
2299  } else if ((par0=="-ds" || par0=="-dataset") && ir>1) {
2300  m_newdataset=argv[ic+1];
2301  ++ic;
2302  } else if (par0=="-us" || par0=="-updatesource") {
2303  m_sourceread=false;
2304  } else if (par0=="-cd" || par0=="-checkdest") {
2305  m_checkdesttag=true;
2306  } else if (par0=="-tr" || par0=="-truncate") {
2307  m_truncate=true;
2308  } else if (par0=="-al" || par0=="-appendlocked") {
2309  m_applock=true;
2310  } else if (par0=="-alsv" || par0=="-appendlockedsv") {
2311  m_applock=true;
2312  m_applocksv=true;
2313  } else if (par0=="-rdo" || par0=="-readoracle") {
2314  m_readoracle=true;
2315  } else if (par0=="-skiprep" || par0=="-sr") {
2316  m_skiprep=true;
2317  } else if (par0=="-go" || par0=="-getonline") {
2318  m_getonline=true;
2319  } else if (par0=="-onr" || par0=="-onlinerun") {
2320  m_onlinerun=true;
2321  } else if (par0=="-gb" || par0=="-getbulk") {
2322  m_getbulk=true;
2323  } else if (par0=="-gt" || par0=="-gettime") {
2324  m_gettime=true;
2325  } else if ((par0=="-tdb" || par0=="-timedb") && ir>1) {
2326  m_timedb=argv[ic+1];
2327  ++ic;
2328  } else if (par0=="-ignoremode" && ir>1) {
2329  std::cout << "Ignoremode password is " << argv[ic+1] << ":end" << std::endl;
2330  if (strcmp(argv[ic+1],"BackDoor")==0) {
2331  m_ignoremode=true;
2332  } else {
2333  std::cout << "ERROR: Incorrect password for -ignoremode" << std::endl;
2334  error=true;
2335  }
2336  ++ic;
2337  } else if (par0=="-is" || par0=="-ignorespec") {
2338  m_ignorespec=true;
2339  } else if (par0=="-pt" || par0=="-prunetags") {
2340  m_prunetags=true;
2341  m_excludehead=true;
2342  } else if (par0=="-lo" || par0=="-lockedonly") {
2343  m_lockedonly=true;
2344  m_prunetags=true;
2345  m_excludehead=true;
2346  } else if (par0=="-cti" || par0=="-copytaginfo") {
2347  m_copytaginfo=true;
2348  } else if (par0=="-ctl" || par0=="-copytaglock") {
2349  m_copytaginfo=true;
2350  m_copytaglock=true;
2351  } else if ((par0=="-tl" || par0=="-taglabel") && ir>1) {
2352  m_taglabel=argv[ic+1];
2353  ++ic;
2354  m_copytaginfo=true;
2355  } else if ((par0=="-ag" || par0=="-addguid") && ir>1) {
2356  m_addguid.emplace_back(argv[ic+1]);
2357  ++ic;
2358  } else if ((par0=="-alf" || par0=="-addlfn") && ir>1) {
2359  m_addlfn.emplace_back(argv[ic+1]);
2360  ++ic;
2361  } else if ((par0=="-pf" || par0=="-parfile") && ir>1) {
2362  m_parfile.emplace_back(argv[ic+1]);
2363  ++ic;
2364  } else if ((par0=="-rf" || par0=="-runfile") && ir>1) {
2365  m_runfile.emplace_back(argv[ic+1]);
2366  ++ic;
2367  } else if ((par0=="-ws" || par0=="-runinfohost") && ir>1) {
2368  m_runinfohost=argv[ic+1];
2369  ++ic;
2370  } else if (par0=="-h" || par0=="-help") {
2371  // help printout triggered by -999 return code
2372  return 999;
2373  } else {
2374  std::cout << "Parameter error for argument: " << par0 << std::endl;
2375  error=true;
2376  }
2377  ++ic;
2378  }
2379  return !error;
2380 }
2381 
2382 
2383 int AtlCoolCopy::setOpts(int argc, const char* argv[]) {
2384  // accumulate raw folder list
2385  std::vector<std::string> folders;
2386  // process options given on command-line
2387  if (!procOptVector(argc,argv,folders)) return 2;
2388 
2389  // parse any parameters in files via parfile option
2390  for (std::vector<std::string>::const_iterator ifile=m_parfile.begin();
2391  ifile!=m_parfile.end();++ifile) {
2392  std::cout << "Reading parameters from file " << *ifile << std::endl;
2393  FILE* p_inp=fopen(ifile->c_str(),"r");
2394  if (p_inp==nullptr) {
2395  std::cout << "File not found" << std::endl;
2396  return 3;
2397  }
2398  std::vector<char> p_buf (999);
2399  while (!feof(p_inp)) {
2400  char* p_line=fgets(p_buf.data(),p_buf.size(),p_inp);
2401  if (p_line!=nullptr) {
2402  int fargc=0;
2403  const char* fargv[99];
2404  // split this line into tokens
2405  char* p_start=nullptr;
2406  while (*p_line!='\0') {
2407  if (*p_line==' ' || *p_line=='\n') {
2408  // pointing at a space/newline, marking the end of a parameter
2409  if (p_start!=nullptr) {
2410  // if we have a parameter, mark the end and store it
2411  *p_line='\0';
2412  fargv[fargc]=p_start;
2413  fargc++;
2414  p_start=nullptr;
2415  }
2416  } else {
2417  // mark the start of a parameter
2418  if (p_start==nullptr) p_start=p_line;
2419  }
2420  ++p_line;
2421  }
2422  if (fargc>0) {
2423  if (!procOptVector(fargc,&fargv[0],folders)) {
2424  fclose(p_inp);
2425  return 3;
2426  }
2427  }
2428  }
2429  }
2430  std::cout << "Close file" << std::endl;
2431  fclose(p_inp);
2432  }
2433 
2434  // now open the database so folder lookup will work
2436  std::cout << "Problem opening connections" << std::endl;
2437  return 10;
2438  } else {
2439  // having assembled the raw list of folders and exclude list, construct
2440  // real list of folders (allows exclude to be specified after folders on
2441  // command line), if none given, add '/'
2442  if (m_lockedonly) {
2443  if (getLockedTags()==0) return 11;
2444  }
2445  if (folders.empty()) {
2446  // no folders specified, take all with tags
2447  addFolder("/",false);
2448  } else {
2449  // explicit list of folders
2450  for (std::vector<std::string>::const_iterator ifold=folders.begin();
2451  ifold!=folders.end();++ifold) addFolder(*ifold,false);
2452  // need to process only tags in root folder, if inclusive hiearchicals
2453  if (m_hitag) addFolder("/",true);
2454  }
2455  }
2456  if (m_getonline) {
2457  if (!getOnlineRun()) return 6;
2458  }
2459  if (m_getbulk) {
2460  if (!getBulkRun()) return 6;
2461  }
2462  if (m_gettime) {
2463  if (!getTimeFromRun()) return 5;
2464  }
2465  if (!m_runfile.empty()) {
2466  if (!getRunList()) return 7;
2467  }
2468  // list out parameter changes
2469  if (m_runemin!=cool::ValidityKeyMin || m_runemax!=cool::ValidityKeyMax)
2470  std::cout << "Source run/LB range [" << (m_runemin >> 32) << "," <<
2471  (m_runemin & 0xFFFFFFFF) << "] to [" << (m_runemax >> 32) << "," <<
2472  (m_runemax & 0xFFFFFFFF) << "]" << std::endl;
2473  if (m_timemin!=cool::ValidityKeyMin || m_timemax!=cool::ValidityKeyMax)
2474  std::cout << "Source timestamp range " << m_timemin << " " <<
2475  timeString(m_timemin) << "to " << m_timemax << " " <<
2476  timeString(m_timemax) << std::endl;
2477  if (m_alliov)
2478  std::cout << "Change o/p run/LB range [" << (m_newrunemin >> 32) << "," <<
2479  (m_newrunemin & 0xFFFFFFFF) << "] to [" << (m_newrunemax >> 32) << "," <<
2480  (m_newrunemax & 0xFFFFFFFF) << "]" << std::endl;
2481  if (m_alliov)
2482  std::cout << "Change o/p timestamp range " << m_newtimemin << " " <<
2483  timeString(m_newtimemin) << "to " << m_newtimemax << " " <<
2484  timeString(m_newtimemax) << std::endl;
2485  if (m_skipout) {
2486  std::cout << "Skip IOVs extending outside run/LB range [" <<
2487  (m_srunemin >> 32) << "," << (m_srunemin & 0xFFFFFFFF) << "] to [" <<
2488  (m_srunemax >> 32) << "," << (m_srunemax & 0xFFFFFFFF) << "]" <<
2489  std::endl;
2490  }
2491  if (!m_excludechans.empty()) {
2492  for (std::vector<cool::ChannelId>::const_iterator itr=
2493  m_excludechans.begin();itr!=m_excludechans.end();++itr)
2494  std::cout << "Channel " << *itr << " will be excluded" << std::endl;
2495  }
2496  if (m_hitag) std::cout << "All hierarchical tags connecting to referenced tags will be copied" << std::endl;
2497  if (m_nohitag) std::cout << "Hierarchical tag relations will not be copied"
2498  << std::endl;
2499  return 0;
2500 }
2501 
2502 bool AtlCoolCopy::isNumeric(const char* input) {
2503  // determine if input string is numeric or string
2504  bool isnum=true;
2505  const char* cptr=input;
2506  while (*cptr!='\0') {
2507  if (!isdigit(*cptr)) { isnum=false; break;}
2508  ++cptr;
2509  }
2510  return isnum;
2511 }
2512 
2513 bool AtlCoolCopy::equalRecord(const cool::IRecord& lhs,
2514  const cool::IRecord& rhs) {
2515  // equality test for COOL IRecords, handling NULL string attributes correctly
2516  // tests values and types of Attributes, not names
2517  if (lhs.size()!=rhs.size()) return false;
2518  for (size_t i=0;i<lhs.size();++i) {
2519  // types must match - have to test explicitly first
2520  if (lhs[i].specification().storageType()!=
2521  rhs[i].specification().storageType())
2522  return false;
2523  // now use IField equality to for final test (spec again and data)
2524  if (lhs[i]!=rhs[i]) {
2525  // if not equal, check for special case of strings and compare directly
2526  // types are known to be equal so only test LHS
2527  const cool::StorageType& stype=lhs[i].specification().storageType();
2528  if (stype==cool::StorageType::String255 ||
2529  stype==cool::StorageType::String4k ||
2530  stype==cool::StorageType::String64k ||
2531  stype==cool::StorageType::String16M) {
2532  // check if string data payloads are really equal or not
2533  if (lhs[i].data<std::string>()!=rhs[i].data<std::string>())
2534  return false;
2535  } else {
2536  // if not string, trust the result of IField !=operator
2537  return false;
2538  }
2539  }
2540  }
2541  return true;
2542 }
2543 
2544 
2545 cool::ValidityKey AtlCoolCopy::timeVal(const char* input) {
2546  // convert input char* string to 64bit COOL validityKey
2547  // input either represents a string in seconds, or a date in the form
2548  // yyyy/mm/dd:hh:mm:ss
2549  // first determine if input is a number
2550  if (isNumeric(input)) {
2551  return static_cast<long long>(atol(input))*
2552  static_cast<long long>(1.E9);
2553  } else {
2554  struct tm mytm{},mytm2{};
2555  char* result=strptime(input,"%Y-%m-%d:%T",&mytm);
2556  if (result!=nullptr) {
2557  // make the DST field zero, so the time is interpreted without daylight
2558  // savings time
2559  mytm.tm_isdst=0;
2560  // now have to correct for the local time zone - do this by also
2561  // calculating the time since epoch for 2/1/1970 midnight (no DST)
2562  cool::ValidityKey itime=static_cast<cool::ValidityKey>(mktime(&mytm));
2563  strptime("1970-01-02:00:00:00","%Y-%m-%d:%T",&mytm2);
2564  cool::ValidityKey caltime=static_cast<cool::ValidityKey>(mktime(&mytm2));
2565  if (caltime == static_cast<cool::ValidityKey>(-1)) {
2566  std::cout <<
2567  "ERROR in mktime" << std::endl;
2568  return 0;
2569  }
2570  itime+=24*60*60-caltime;
2571  return itime*static_cast<cool::ValidityKey>(1.E9);
2572  } else {
2573  std::cout <<
2574  "ERROR in format of time value, use e.g. 2007-05-25:14:01:00" <<
2575  std::endl;
2576  return 0;
2577  }
2578  }
2579 }
2580 
2581 std::string AtlCoolCopy::timeString(const cool::ValidityKey iovtime) {
2582  if (iovtime==cool::ValidityKeyMin) {
2583  return "ValidityKeyMin ";
2584  } else if (iovtime==cool::ValidityKeyMax) {
2585  return "ValidityKeyMax ";
2586  } else {
2587  time_t time=static_cast<time_t>(iovtime/1E9);
2588  struct tm result;
2589  char buf[32];
2590  return "UTC "+std::string(asctime_r(gmtime_r(&time, &result), buf));
2591  }
2592 }
2593 
2594 cool::ValidityKey AtlCoolCopy::runLBVal(const char* input1,const char* input2) {
2595  // parse the inputs as run/LB numbers to generate a validitykey
2596  // used in processing -rls-type options
2597  cool::ValidityKey val;
2598  if (input1[0]=='M' || input1[0]=='m') {
2599  val=((1LL << 31)-1) << 32;
2600  } else {
2601  val=static_cast<long long>(atol(input1)) << 32;
2602  }
2603  if (input2[0]=='M' || input2[0]=='m') {
2604  val+=(1LL << 32);
2605  } else {
2606  val+=atoll(input2);
2607  }
2608  if (val>cool::ValidityKeyMax) val=cool::ValidityKeyMax;
2609  return val;
2610 }
2611 
2612 
2614  // extract time limits for a run range using the expected information
2615  // in the /TDAQ/RunCtrl/SOR_Params and EOR_Params on the given DB connection
2616 
2617  // choose database instance based on run number switchover if none is given
2618  if (m_timedb.empty()) {
2619  if ((m_runemin >> 32)>=236107) {
2620  m_timedb="COOLONL_TDAQ/CONDBR2";
2621  } else {
2622  m_timedb="COOLONL_TDAQ/COMP200";
2623  }
2624  }
2625  // select foldername based on database instance
2626  std::string sorfolder="/TDAQ/RunCtrl/SOR_Params";
2627  std::string eorfolder="/TDAQ/RunCtrl/EOR_Params";
2628  if (m_timedb.find ("CONDBR2")!=std::string::npos) {
2629  sorfolder="/TDAQ/RunCtrl/SOR";
2630  eorfolder="/TDAQ/RunCtrl/EOR";
2631  }
2632  std::cout << "Extracting times for run range [" << (m_runemin >> 32) <<
2633  "," << (m_runemax >> 32) << "] from database " << m_timedb <<
2634  " folder " << sorfolder << std::endl;
2635  if (m_runemin==cool::ValidityKeyMin || m_runemax==cool::ValidityKeyMax) {
2636  std::cout << "ERROR: Run range not correctly specified" << std::endl;
2637  return false;
2638  }
2639  // open database connection
2640  cool::IDatabasePtr tdaqdb;
2641  try {
2642  tdaqdb=m_dbSvc->openDatabase(transConn(m_timedb),m_sourceread);
2643  std::cout << "Opened database connection" << std::endl;
2644  }
2645  catch (std::exception& e) {
2646  std::cout << "COOL exception caught: " << e.what() << std::endl;
2647  return false;
2648  }
2649  // start of run information
2650  try {
2651  cool::IFolderPtr folder=tdaqdb->getFolder(sorfolder);
2652  // channel number is arbitrary - have to loop
2653  cool::IObjectIteratorPtr itr=folder->browseObjects(m_runemin,m_runemin,
2655  int nobj=0;
2656  while (itr->goToNext()) {
2657  const cool::IRecord& payload=itr->currentRef().payload();
2658  m_timemin=payload["SORTime"].data<unsigned long long>();
2659  ++nobj;
2660  }
2661  itr->close();
2662  if (nobj!=1) {
2663  std::cout << "ERROR: Found " << nobj << " SOR records" << std::endl;
2664  return false;
2665  }
2666  }
2667  catch (std::exception& e) {
2668  std::cout << "Exception accessing SOR information: " << e.what() <<
2669  std::endl;
2670  return false;
2671  }
2672  // end of run information
2673  try {
2674  cool::IFolderPtr folder=tdaqdb->getFolder(eorfolder);
2675  // channel number is arbitrary - have to loop
2676  cool::IObjectIteratorPtr itr=folder->browseObjects(((m_runemax >> 32) << 32),((m_runemax >> 32) << 32),
2678  int nobj=0;
2679  while (itr->goToNext()) {
2680  const cool::IRecord& payload=itr->currentRef().payload();
2681  m_timemax=payload["EORTime"].data<unsigned long long>();
2682  ++nobj;
2683  }
2684  itr->close();
2685  if (nobj!=1) {
2686  std::cout << "ERROR: Found " << nobj << " EOR records" << std::endl;
2687  return false;
2688  }
2689  }
2690  catch (std::exception& e) {
2691  std::cout << "Exception accessing SOR information: " << e.what() <<
2692  std::endl;
2693  return false;
2694  }
2695  tdaqdb->closeDatabase();
2696  std::cout << "Timestamp range set to " << m_timemin << " " <<
2697  timeString(m_timemin) << "to " << m_timemax << " " <<
2698  timeString(m_timemax) << std::endl;
2699  return true;
2700 }
2701 
2703  // open using the ATLAS_COOLONL_GLOBAL schema, since we have authentication
2704  // information for this one by default
2705  // get minimum run-number/timestamp for bulk reco update
2706  std::cout << "Extracting current run-number from ATLAS_RUN_NUMBER @ ATONR_ADG ... " <<
2707  // Initialize libcurl
2708  curl_global_init(CURL_GLOBAL_ALL);
2709  using uniqueCurl_t = std::unique_ptr<CURL,decltype(&curl_easy_cleanup)>;
2710  uniqueCurl_t curl(curl_easy_init(), curl_easy_cleanup);
2711  CURLcode res = CURLE_OK;
2712  if (curl) {
2713  std::string url = m_runinfohost + "/runs?sort=runnumber:DESC&size=1";
2714  // Set the URL
2715  res = curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
2716  if (res != CURLE_OK) {
2717  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2718  return false;
2719  }
2720  // Follow HTTP redirections
2721  res = curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L);
2722  if (res != CURLE_OK) {
2723  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2724  return false;
2725  }
2726  // Response data buffer
2727  std::string response;
2728 
2729  // Set the callback function to receive response data
2730  res = curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, WriteCallback);
2731  if (res != CURLE_OK) {
2732  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2733  return false;
2734  }
2735  res = curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response);
2736  if (res != CURLE_OK) {
2737  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2738  return false;
2739  }
2740 
2741  // Perform the request
2742  res = curl_easy_perform(curl.get());
2743  if (res != CURLE_OK) {
2744  std::cerr << "Failed to perform request: " << curl_easy_strerror(res) << ":" << url.c_str() << std::endl;
2745  return false;
2746  } else {
2747  // Print the received response
2748  std::cout << "Response: " << std::endl;
2749  std::cout << response << std::endl;
2750  // Parse the JSON string
2751  try {
2753 
2754  // Extract the runnumber field
2755  int runNumber = jsonData["resources"][0]["runnumber"];
2756  int nextrun=runNumber+1;
2757  std::cout << "Next run started will be " << nextrun << std::endl;
2758  const long long rtime=time(nullptr);
2759  std::cout << "Epoch time extracted " << rtime << std::endl;
2760  if (m_alliov) {
2761  // if overwriting IOVs, set the new IOV lower limit
2762  m_newrunemin=(static_cast<long long>(nextrun)) << 32;
2763  m_newtimemin=rtime*static_cast<long long>(1E9);
2764  } else {
2765  // set the query lower limit - for use with truncate option
2766  m_runemin=(static_cast<long long>(nextrun)) << 32;
2767  m_timemin=rtime*static_cast<long long>(1E9);
2768  m_truncate=true;
2769  }
2770  } catch (nlohmann::json::parse_error& e) {
2771  std::cerr << "Failed to parse JSON response: " << e.what() << std::endl;
2772  return false;
2773  } catch (nlohmann::json::type_error& e) {
2774  std::cerr << "Failed to extract data from JSON response: " << e.what() << std::endl;
2775  return false;
2776  } catch (std::exception& e) {
2777  std::cerr << "Failed to extract run and timestamp from JSON response: " << e.what() << std::endl;
2778  return false;
2779  }
2780  }
2781  // Clean up done by unique_ptr d'tor
2782  } else {
2783  std::cerr << "Failed to initialize libcurl." << std::endl;
2784  return false;
2785  }
2786  // Cleanup libcurl
2787  curl_global_cleanup();
2788  return true;
2789 }
2791  // get minimum run-number/timestamp for bulk reco update
2792  if (m_getonline) {
2793  std::cout << "ERROR: -getonline and -getbulk cannot be used simultaneously"
2794  << std::endl;
2795  return false;
2796  }
2797  std::cout << "Call getbulk using URL" << std::endl;
2798  // Initialize libcurl
2799  curl_global_init(CURL_GLOBAL_ALL);
2800  using uniqueCurl_t = std::unique_ptr<CURL,decltype(&curl_easy_cleanup)>;
2801  uniqueCurl_t curl(curl_easy_init(), curl_easy_cleanup);
2802  //CURL *curl = curl_easy_init();
2803  CURLcode res = CURLE_OK;
2804 
2805  if (curl) {
2806  std::string url = m_runinfohost + "/runs/nemop/sync";
2807  // Set the URL
2808  res = curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
2809  if (res != CURLE_OK) {
2810  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2811  return false;
2812  }
2813  // Follow HTTP redirections
2814  res = curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L);
2815  if (res != CURLE_OK) {
2816  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2817  return false;
2818  }
2819  // Response data buffer
2820  std::string response;
2821 
2822  // Set the callback function to receive response data
2823  res = curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, WriteCallback);
2824  if (res != CURLE_OK) {
2825  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2826  return false;
2827  }
2828  res = curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response);
2829  if (res != CURLE_OK) {
2830  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2831  return false;
2832  }
2833  // Perform the request
2834  res = curl_easy_perform(curl.get());
2835  if (res != CURLE_OK) {
2836  std::cerr << "Failed to perform request: " << curl_easy_strerror(res) << ":" << url.c_str() << std::endl;
2837  return false;
2838  } else {
2839  // Print the received response
2840  std::cout << "Response: " << std::endl;
2841  std::cout << response << std::endl;
2842  try {
2843  // Split the response into two fields
2845  // Extract the run and timestamp fields
2846  int nextrun = jsonData["run"];
2847  long long rtime = jsonData["timestamp"];
2848  std::cout << "Next run started will be " << nextrun << std::endl;
2849  std::cout << "Epoch time extracted " << rtime << std::endl;
2850  if (m_alliov) {
2851  // if overwriting IOVs, set the new IOV lower limit
2852  m_newrunemin=(static_cast<long long>(nextrun)) << 32;
2853  m_newtimemin=rtime*static_cast<long long>(1E9);
2854  } else {
2855  // set the query lower limit - for use with truncate option
2856  m_runemin=(static_cast<long long>(nextrun)) << 32;
2857  m_timemin=rtime*static_cast<long long>(1E9);
2858  m_truncate=true;
2859  }
2860  } catch (nlohmann::json::parse_error& e) {
2861  std::cerr << "Failed to parse JSON response: " << e.what() << std::endl;
2862  return false;
2863  } catch (nlohmann::json::type_error& e) {
2864  std::cerr << "Failed to extract data from JSON response: " << e.what() << std::endl;
2865  return false;
2866  } catch (std::exception& e) {
2867  std::cerr << "Failed to extract run and timestamp from JSON response: " << e.what() << std::endl;
2868  return false;
2869  }
2870  }
2871  // Clean up done by unique_ptr
2872  } else {
2873  std::cerr << "Failed to initialize libcurl." << std::endl;
2874  return false;
2875  }
2876  // Cleanup libcurl
2877  curl_global_cleanup();
2878 
2879  return true;
2880 }
2881 
2883  // loop over all given filenames
2884  for (std::vector<std::string>::const_iterator itr=m_runfile.begin();
2885  itr!=m_runfile.end();++itr) {
2886  std::cout << "Reading allowed run numbers from file " << *itr << std::endl;
2887  FILE* p_inp=fopen(itr->c_str(),"r");
2888  if (p_inp==nullptr) {
2889  std::cout << "File not found" << std::endl;
2890  return false;
2891  }
2892  std::vector<char> p_buf (999);
2893  while (!feof(p_inp)) {
2894  char* p_line=fgets(p_buf.data(),p_buf.size(),p_inp);
2895  if (p_line!=nullptr) {
2896  unsigned int run=atoi(p_line);
2897  m_runlist.push_back(run);
2898  }
2899  }
2900  fclose(p_inp);
2901  }
2902  std::sort(m_runlist.begin(),m_runlist.end());
2903  std::cout << "Read list of " << m_runlist.size() << " runs from " <<
2904  m_runfile.size() << " input runlist files" << std::endl;
2905  for (std::vector<unsigned int>::const_iterator itr=m_runlist.begin();
2906  itr!=m_runlist.end();++itr) std::cout <<
2907  "Update allowed for run " << *itr << std::endl;
2908  return true;
2909 }
2910 
2911 int AtlCoolCopy::getUpdateMode(std::string_view desc,
2912  std::string_view tag) {
2913  // analyse the folder description and tag name for updateMode flags
2914  // return 1 for online mode, 0 otherwise
2915  std::string_view modestr="";
2916  int mode=0;
2917  std::string::size_type iofs1=desc.find("<updateMode>");
2918  std::string::size_type iofs2=desc.find("</updateMode>");
2919  if (iofs1!=std::string::npos && iofs2!=std::string::npos && iofs2>iofs1)
2920  modestr=desc.substr(iofs1+12,iofs2-iofs1-12);
2921  if (modestr=="UPD1" || tag.find("UPD1")!=std::string::npos) mode=1;
2922  if (modestr=="UPD2" || tag.find("UPD2")!=std::string::npos) mode=2;
2923  if (modestr=="UPD3" || tag.find("UPD3")!=std::string::npos) mode=3;
2924  if (modestr=="UPD4" || tag.find("UPD4")!=std::string::npos) mode=4;
2925  return mode;
2926 }
2927 
2928 
2929 bool AtlCoolCopy::checkChannels(const std::string& folder,
2930  const cool::IFolderPtr& sourcefl,const cool::IFolderPtr& destfl,
2931  bool newfolder) {
2932  // Channel manipulation is slow due to lack of bulk API for creating them
2933  // this routine therefore does the minimum work, not checking destination
2934  // if folder is newly created, and not copying description unless requested
2935  // get map of all channels
2936  const std::map<cool::ChannelId,std::string> chanmap=
2937  sourcefl->listChannelsWithNames();
2938  if (m_debug) std::cout << "Checking channels table for folder " << folder
2939  << " with " << chanmap.size() << " channels" << std::endl;
2940  int ncreate=0;
2941  int nmodify=0;
2942  // iterate over all channels
2943  for (std::map<cool::ChannelId,std::string>::const_iterator
2944  itr=chanmap.begin();itr!=chanmap.end();++itr) {
2945  // assume now ALL channels are in channels table (COOL 2.4 and above)
2946  const cool::ChannelId chan=itr->first;
2947  // check if channel should be skipped
2948  if (!m_excludechans.empty()) {
2949  if (find(m_excludechans.begin(),m_excludechans.end(),
2950  chan)!=m_excludechans.end()) continue;
2951  }
2952  if (!m_chansel.inSelection(chan)) continue;
2953 
2954  const std::string& sourcename=itr->second;
2955  // only look up channel description if we are copying it
2956  std::string sourcedesc="";
2957  if (m_chdesc) sourcedesc=sourcefl->channelDescription(chan);
2958  // only check destination if folder alreay existed
2959  if (!newfolder && destfl->existsChannel(chan)) {
2960  const std::string & destname=destfl->channelName(chan);
2961  std::string destdesc="";
2962  if (m_chdesc) destdesc=destfl->channelDescription(chan);
2963  // if sourcename is not set, and destination is, dont worry
2964  if (sourcename!=destname && !sourcename.empty()) {
2965  // channel names differ - error if verifying, set if copying
2966  if (m_verify) {
2967  std::cout << "ERROR: Channel " << chan << " names differ: " <<
2968  sourcename << " " << destname << std::endl;
2969  return false;
2970  }
2971  destfl->setChannelName(chan,sourcename);
2972  std::cout << "Modify channel " << chan << " name from "
2973  << destname << " to " << sourcename << std::endl;
2974  ++nmodify;
2975  }
2976  if (sourcedesc!=destdesc && !sourcedesc.empty()) {
2977  // channel descriptions differ - error if verifying, set if copying
2978  if (m_verify) {
2979  std::cout << "ERROR: Channel " << chan << " descriptions differ: "
2980  << sourcedesc << " " << destdesc << std::endl;
2981  return false;
2982  }
2983  std::cout << "Modify channel " << chan << " description from "
2984  << sourcedesc << " to " << destdesc << std::endl;
2985  destfl->setChannelDescription(chan,sourcedesc);
2986  ++nmodify;
2987  }
2988  } else {
2989  // channel does not exist on destination - if verifying, this is error
2990  if (m_verify) return false;
2991  // otherwise create it
2992  if (m_debug) std::cout << "Create new channel " << chan << " name "
2993  << sourcename << " description" << sourcedesc << std::endl;
2994  destfl->createChannel(chan,sourcename,sourcedesc);
2995  ++ncreate;
2996  }
2997  }
2998  if (ncreate>0) std::cout << "Created " << ncreate << " new channels for "
2999  << folder << std::endl;
3000  if (nmodify>0) std::cout << "Modified " << nmodify << " channel info for "
3001  << folder << std::endl;
3002  return true;
3003 }
3004 
3005 
3006 void AtlCoolCopy::checkRef(const cool::IRecord& payload,
3007  const std::string& folder, const std::string& tag) {
3008  const std::string name=payload[0].name();
3009  if (name=="PoolRef" || name=="fileGUID") {
3010  std::string poolref;
3011  std::string addr=payload[0].data<std::string>();
3012  if (name=="PoolRef") {
3013  std::string::size_type iofs1=addr.find("[DB=");
3014  std::string::size_type iofs2=addr.find(']',iofs1);
3015  if (iofs1!=std::string::npos && iofs2!=std::string::npos && iofs2>iofs1) {
3016  poolref=addr.substr(iofs1+4,iofs2-iofs1-4);
3017  } else {
3018  std::cout << "ERROR: Badly formed POOL object reference " <<
3019  addr << std::endl;
3020  }
3021  } else {
3022  // for fileGUID, POOL GUID is just the string
3023  poolref=std::move(addr);
3024  }
3025  std::string foldertag=folder+":"+tag;
3026  // insert into list, first check if same as before
3027  if (m_poollast!=m_poolrefs.end() && m_poollast->first==poolref) {
3028  m_poollast->second.inccount();
3029  m_poollast->second.addfoldertag(foldertag);
3030  } else {
3031  m_poollast=m_poolrefs.find(poolref);
3032  if (m_poollast!=m_poolrefs.end()) {
3033  m_poollast->second.inccount();
3034  m_poollast->second.addfoldertag(foldertag);
3035  } else {
3036  m_poolrefs[poolref]=PoolMapElement(1,foldertag);
3037  m_poollast=m_poolrefs.find(poolref);
3038  }
3039  }
3040  }
3041 }
3042 
3044  std::cout << "Copying additional hierarchical tags of parent folders"
3045  << std::endl;
3046  while (!m_hiparent.empty()) {
3047  // take first element and erase it
3048  std::string folderset=*m_hiparent.begin();
3049  m_hiparent.erase(m_hiparent.begin());
3050  // process any tags defined for this folderset
3051  if (m_sourceDbPtr->existsFolderSet(folderset)) {
3052  cool::IFolderSetPtr sourcefl=m_sourceDbPtr->getFolderSet(folderset);
3053  cool::IFolderSetPtr destfl=m_destDbPtr->getFolderSet(folderset);
3054  std::cout << "Processing folderset " << folderset << std::endl;
3055  for (HiTagMap::const_iterator imap=m_hitagmap.begin();
3056  imap!=m_hitagmap.end();++imap) {
3057  try {
3058  // will throw exception if this tag is not pointing to the folder
3059  std::string sourcetag=sourcefl->findTagRelation(imap->first);
3060  std::cout << "Create hierarchical tag between " << sourcetag <<
3061  " and " << imap->first << " in folder " << imap->second <<
3062  std::endl;
3063  try {
3064  // first check this relation has not already been created
3065  std::string etag=destfl->resolveTag(imap->first);
3066  if (etag==sourcetag) {
3067  std::cout << "This relation has already been created" << std::endl;
3068  } else {
3069  std::cout << "ERROR: Tag in parent already related to " <<
3070  etag << std::endl;
3071  return 34;
3072  }
3073  }
3074  catch (cool::Exception& e ) {
3075  // only do creation if does not already exist - via exception
3076  try {
3077  destfl->createTagRelation(imap->first,sourcetag);
3078  // check if this implicates yet another parent folder to be checked
3079  if (find(m_hiparent.begin(),m_hiparent.end(),imap->second)==
3080  m_hiparent.end()) m_hiparent.push_back(imap->second);
3081  if (m_copytaginfo &&
3082  m_cooltagmap.find(imap->first)==m_cooltagmap.end())
3083  m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
3084  CoolTagInfo(m_sourceDbPtr,imap->second,imap->second,
3085  imap->first,imap->first)));
3086  }
3087  catch (cool::Exception& e) {
3088  std::cout << "Cool exception " << e.what() <<
3089  "thrown in hierarchical tag creation" << std::endl;
3090  return 34;
3091  }
3092  }
3093  }
3094  // exceptions from findTag - do nothing
3095  catch (cool::Exception& e) { }
3096  }
3097  } else {
3098  std::cout << "ERROR Folderset " << folderset << " not found" <<
3099  std::endl;
3100  }
3101  }
3102  return 0;
3103 }
3104 
3106  std::cout << "Write tag description ";
3107  if (m_copytaglock) std::cout << "and lock info ";
3108  std::cout << "for " << m_cooltagmap.size() << " tags" << std::endl;
3109  for (CoolTagMap::const_iterator itr=m_cooltagmap.begin();
3110  itr!=m_cooltagmap.end();++itr) {
3111  itr->second.write(m_destDbPtr,m_copytaglock);
3112  }
3113  return 0;
3114 }
3115 
3117  std::cout << "Total of " << m_poolrefs.size() << " POOL files referenced"
3118  << std::endl;
3119  for (PoolMap::const_iterator ipool=m_poolrefs.begin();
3120  ipool!=m_poolrefs.end();++ipool) {
3121  std::cout << "Ref " << ipool->first << " (" << ipool->second.count()
3122  << ")" << std::endl;
3123  }
3124  return 0;
3125 }
3126 
3127 int AtlCoolCopy::resolvePoolRefs ATLAS_NOT_THREAD_SAFE () {
3128  std::cout << "Total of " << m_poolrefs.size() << " POOL Files referenced"
3129  << std::endl;
3130  pool::IFileCatalog* catalog=setupCatalog(m_poolcat);
3131  if (catalog==nullptr) return 110;
3132  pool::SimpleUtilityBase pool_utility;
3133  if (m_poolopen) {
3134  // prepare POOL session
3135  pool_utility.startSession();
3136  }
3137  // inject additional GUIDs/LFNs if needed
3138  if (!m_addguid.empty()) {
3139  for (std::vector<std::string>::const_iterator itr=m_addguid.begin();
3140  itr!=m_addguid.end();++itr) {
3141  m_poolrefs[*itr]=PoolMapElement(1,"ADDGUID");
3142  std::cout << "Added POOL file GUID: " << *itr << std::endl;
3143  }
3144  }
3145  if (!m_addlfn.empty()) {
3146  for (std::vector<std::string>::const_iterator itr=m_addlfn.begin();
3147  itr!=m_addlfn.end();++itr) {
3148  std::string guid;
3149  catalog->lookupFileByLFN(*itr,guid);
3150  std::cout << "Add POOL file GUID: " << guid << " from LFN " << *itr
3151  << std::endl;
3152  m_poolrefs[guid]=PoolMapElement(1,"ADDLFN");
3153  }
3154  }
3155 
3156  // set up for making new dataset if needed
3157  using LFNGVec = std::vector<std::pair<std::string, std::string> >;
3158  LFNGVec dsfound;
3159  bool dscopy=!m_newdataset.empty();
3160 
3161  // loop through GUIDs and attempt to resolve
3162  int nbad=0;
3163  for (PoolMap::iterator ipool=m_poolrefs.begin();
3164  ipool!=m_poolrefs.end();++ipool) {
3165  pool::FileCatalog::FileID guid=ipool->first;
3167  catalog->getLFNs( guid, lfns );
3168  if( !lfns.empty() ) {
3169  // file found in cataloge - print LFN and usage count
3170  const std::string lfn = lfns[0].first;
3171  ipool->second.setlfn(lfn);
3172  std::cout << "LFN: " << lfn << " (" << ipool->second.count()
3173  << ")" << std::endl;
3174  if (dscopy)
3175  dsfound.push_back(std::pair<std::string,std::string>(lfn,guid));
3176  } else {
3177  // error bit 0 - no LFN
3178  if (!m_listpfn) ipool->second.setErrorBit(0);
3179  }
3180  if (m_listpfn || m_poolopen) {
3181  std::string pfn, tech;
3182  catalog->getFirstPFN( guid, pfn, tech );
3183  if( !pfn.empty() ) {
3184  ipool->second.setpfn(pfn);
3185  std::cout << "PFN: " << pfn << " (" << ipool->second.count()
3186  << ")" << std::endl;
3187  if (m_poolopen) {
3188  // first try the file as a CoolHist file
3189  std::string hguid=getCoolHistGUID(pfn);
3190  if (!hguid.empty()) {
3191  // successful get of CoolHist GUID
3192  if (hguid!=ipool->first) {
3193  std::cout << "ERROR File CoolHist GUID " << hguid <<
3194  " inconsistent with catalogue " << ipool->first << std::endl;
3195  ipool->second.setErrorBit(3);
3196  }
3197  } else {
3198  // try the file as a genuine POOL file
3199  try {
3200  const std::string fid = pool_utility.readFileGUID( pfn );
3201  if( fid != ipool->first ) {
3202  std::cout << "ERROR File GUID " << fid <<
3203  " inconsistent with catalogue " << ipool->first << std::endl;
3204  // file GUID inconsistent with catalogue - error bit 3
3205  ipool->second.setErrorBit(3);
3206  }
3207  }
3208  catch( std::runtime_error& e ) {
3209  std::cout << "Cannot open file for reading!" << std::endl;
3210  std::cout << e.what() << std::endl;
3211  // File cannot be opened: set error bit 2
3212  ipool->second.setErrorBit(2);
3213  }
3214  }
3215  } // end of actions opening POOL file
3216  } else {
3217  // PFN not found - set bit 1
3218  ipool->second.setErrorBit(1);
3219  }
3220  }
3221  // check file error code
3222  if (ipool->second.errcode()>0) ++nbad;
3223  }
3224  catalog->commit();
3225  catalog->disconnect();
3226  delete catalog;
3227 
3228  // produce definition of new dataset if needed
3229  if (dscopy) {
3230  // start up a new catalogue instance with the merge catalogues
3231  // which indicate which files are already available and don't need to
3232  // put in the output dataset definition
3233  catalog=setupCatalog(m_mergecat);
3234  if (catalog==nullptr) return 110;
3235  const std::string dssname="register.sh";
3236  std::cout << "Write DQ2 registerFileInDataset commands to " << dssname
3237  << " for registration in dataset " << m_newdataset << std::endl;
3238  std::ofstream dsstream(dssname.c_str());
3239  for (LFNGVec::const_iterator itr=dsfound.begin();
3240  itr!=dsfound.end();++itr) {
3241  const std::string& lfn=itr->first;
3242  const std::string& guid=itr->second;
3244  catalog->getLFNs( guid, lfns );
3245  if( !lfns.empty() ) {
3246  // file is already registered - check logical names are consistent
3247  const std::string lfn2 = lfns[0].first;
3248  if (lfn2!=lfn) std::cout << "WARNING: LFNs for GUID " << guid <<
3249  " differ in input/merge datasets: " << lfn << " vs "
3250  << lfn2 << std::endl;
3251  } else {
3252  // file is not registered - add to output
3253  dsstream << "dq2-register-files " << m_newdataset << " "
3254  << lfn << " " << guid << std::endl;
3255  }
3256  }
3257  catalog->commit();
3258  catalog->disconnect();
3259  delete catalog;
3260  }
3261 
3262  int retcode=0;
3263  if (nbad==0) {
3264  std::cout << "All POOL references were resolved by catalogues"
3265  << std::endl;
3266  } else {
3267  std::cout << "ERROR Could not resolve " << nbad << " files" << std::endl;
3268  retcode=111;
3269  }
3270  if (nbad>0 || m_poolall) filePoolRefs();
3271  return retcode;
3272 }
3273 
3274 std::string AtlCoolCopy::getCoolHistGUID(const std::string& file) {
3275  // attempt to extract COOL Hist GUID from file
3276  std::string hguid="";
3277  TFile* myfile=TFile::Open(file.c_str(),"READ");
3278  if (myfile!=nullptr) {
3279  TObjString* oguid;
3280  myfile->GetObject("fileGUID",oguid);
3281  if (oguid!=nullptr) {
3282  hguid=oguid->GetString();
3283  std::cout << "CoolHist GUID found to be " << hguid << std::endl;
3284  }
3285  myfile->Close();
3286  }
3287  return hguid;
3288 }
3289 
3291  // list POOL refs and error codes, optionally to file
3292  if (!m_checkoutputfile.empty()) {
3293  std::cout << "Write POOL file checkoutput on " << m_checkoutputfile <<
3294  std::endl;
3295  } else {
3296  m_checkoutputfile="/dev/null";
3297  }
3298  std::ofstream chkostream(m_checkoutputfile.c_str());
3299  std::cout << "ErrCode GUID Count LFN PFN Folders ..." << std::endl;
3300  for (PoolMap::const_iterator ipool=m_poolrefs.begin();
3301  ipool!=m_poolrefs.end();++ipool) {
3302  if (m_poolall || ipool->second.errcode()>0) {
3303  std::ostringstream line;
3304  // write error code, GUID, usage count
3305  line << ipool->second.errcode() << " " << ipool->first << " " <<
3306  ipool->second.count() << " ";
3307  if (!ipool->second.lfn().empty()) {
3308  line << ipool->second.lfn() << " ";
3309  } else {
3310  line << "noLFN ";
3311  }
3312  if (!ipool->second.pfn().empty()) {
3313  line << ipool->second.pfn() << " ";
3314  } else {
3315  line << "noPFN ";
3316  }
3317  for (std::vector<std::string>::const_iterator
3318  itr=ipool->second.foldertag().begin();
3319  itr!=ipool->second.foldertag().end();++itr)
3320  line << *itr << " ";
3321  std::cout << line.str() << std::endl;
3322  chkostream << line.str() << std::endl;
3323  }
3324  }
3325 }
3326 
3328  const std::vector<std::string>& catvec) {
3330  try {
3331  catalog->setWriteCatalog("file:PoolFileCatalog.xml");
3332  for (std::vector<std::string>::const_iterator icat=catvec.begin();
3333  icat!=catvec.end();++icat) {
3334  std::cout << "Add catalogue: " << *icat << std::endl;
3335  // if catalogue contains no ":" specifier, assume plain file
3336  if (icat->find(':')==std::string::npos) {
3337  catalog->addReadCatalog("file:"+(*icat));
3338  } else {
3339  catalog->addReadCatalog(*icat);
3340  }
3341  }
3342  catalog->connect();
3343  catalog->start();
3344  return catalog;
3345  }
3346  catch (std::exception& e) {
3347  std::cout << "Could not setup POOL catalogues, exception:" << e.what()
3348  << std::endl;
3349  return nullptr;
3350  }
3351 }
3352 
3353 void printHelp() {
3354  std::cout << "usage: AtlCoolCopy.exe <sourceCoolDB> <destinationCoolDB> { <options> }" << std::endl;
3355  std::cout << "Options are (see doc/mainpage.h for more):" << std::endl <<
3356  "-a, -alliov : set IOVs on destination to [ValidityKeyMin, ValidityKeyMax]" << std::endl <<" or following settings of -nrls,-nrlu,-nts,-ntu" <<
3357  std::endl <<
3358  "-ag, -addguid <guid> : Add GUID to list accessed for POOL checks" <<
3359  std::endl <<
3360  "-alf, -addlfn <LFN> : Add LFN to list accessed for POOL checks" <<
3361  std::endl <<
3362  "-al, -appendlocked : Allow locked tags to be updated if no overlap" <<
3363  std::endl <<
3364  "-alsv, -appendlockedsv : Allow locked tag update for openended IOVs" <<
3365  std::endl <<
3366  "-ana, -analyse <delta_t>: produce analysis ROOT file (filename = dest DB argument)" << std::endl <<
3367 "-bs, -buffersize <size> : set size of bulkstorage output buf to <size> objs"
3368  << std::endl <<
3369  "-c, -create : create destination DB if not already existing"
3370  << std::endl <<
3371  "-cd, -checkdest : Check destination DB and skip already existing tags"
3372  << std::endl <<
3373  "-ch, -channel : restrict selection to given channel or range specified as c1:c2" << std::endl <<
3374  "-ch1, -channel1 : specify start of a range of channels" << std::endl <<
3375  "-ch2, -channel2 : specify end of a range of channels" << std::endl <<
3376  "-cf, -checkfiles : check POOL files can be opened and have correct GUID"
3377  << std::endl <<
3378  "-co, -checkoutput <file> : write POOL file check output on file"
3379  << std::endl << "-cti, -copytaginfo : Copy tag descriptions"
3380  << std::endl << "-ctl, -copytaglock : Copy tag locked status and descriptions"
3381  << std::endl << "-cr, -checkrefs : check POOL references"
3382  << std::endl << "-d, -debug : produce debug output" << std::endl;
3383  std::cout <<
3384  "-ds, -dataset : output register.sh for creating DQ2 datasets "
3385  << std::endl <<
3386  "-ec, -excludechannel : exclude given channel from copy" << std::endl <<
3387  "-eh, -excludehead : exclude HEAD tag even if no tags found in MV folders"
3388  << std::endl <<
3389  "-ih, -includehead : include HEAD as well as any tags in MV folders" <<
3390  std::endl <<
3391  "-e, -exclude <pattern> : exclude folders" << std::endl <<
3392  "-f, -folder <pattern> : include folders" << std::endl <<
3393  "-fs, -forcesingle : force destination folder to be singleversion"
3394  << std::endl <<
3395  "-fm, -forcemulti : force destination folder to be multiversion"
3396  << std::endl <<
3397  "-frl, -forcerunlumi : force destination folder to be run/LB indexed"
3398  << std::endl <<
3399  "-ftm, -forcetime : force destination folder to be timestamp indexed"
3400  << std::endl <<
3401  "-fp, -forcepayload : Force destn folders to be created with payload tbl" << std::endl <<
3402  "-fnp, -forcenopayload : Force destn folders to be created without payload tbl" << std::endl <<
3403  "-forcerecreate : delete and recreate destination database" << std::endl;
3404  std::cout << "-ht, -headtag : Use HEAD-style tagging"
3405  << std::endl <<
3406  "-go, -getonline : Set minimum run number (-rls setting) to next ONLINE run"
3407  << std::endl <<
3408  "-gb, -getbulk : Set minimum run number (-rls setting) to next bulk reco run"
3409  << std::endl <<
3410  "-gt, -gettime : Extract timestamp information for given run number range"
3411  << std::endl <<
3412  "-h, -help : print help and exit" << std::endl <<
3413  "-hi, -hitag : Copy hierrchical tags inclusively" << std::endl <<
3414  "-ignoremode <pwd> : Ignore UPDx mode protection (experts only)"
3415  << std::endl <<
3416  "-is, -ignorespec : Ignore differences in folder spec if names are all equal" << std::endl <<
3417  "-lo, -lockedonly : Only copy locked/partially-locked top-level tags"
3418  << std::endl <<
3419  "-nc, -nocopy : Do not actually copy, just read source DB" << std::endl <<
3420  "-nch, -nochannel : Do not check or copy channel information" << std::endl
3421 << "-ncr, -noclobroot: Do not copy CLOB data into ROOT files" << std::endl
3422 << "-noc, -nocoracool : Do not copy Coracool structure payloads " << std::endl <<
3423  "-nd, -nodata : Copy only folder structures, not data" << std::endl <<
3424  "-nh, -nohitag : Do not follow hierarchical tag relations" << std::endl <<
3425 "-mc, -mergecat <catfile> : specify POOL file catalogue for new dataset making"
3426  << std::endl;
3427  std::cout << "-of, -outfolder <folder> : rename folder on output"
3428  << std::endl <<
3429  "-onr, -onlinerun : Retrieve run number from online server (not replica)" <<std::endl <<
3430  "-ot, -outtag : " << "Rename tag on output" << std::endl;
3431  std::cout <<
3432  "-pa, -poolall : Output all POOL files (incl good) when checking files"
3433  << std::endl <<
3434 "-pc, -poolcat <catfile> : specify POOL file catalogue for ref checking"
3435  << std::endl <<
3436  "-pf, -parfile <file> : Read additional options/parameters from file" <<
3437  std::endl <<
3438 "-pt, -prunetags : Copy only hierarchical tags descending from specified toptags"
3439  << std::endl <<
3440 "-rdo, -readoracle : force data to be read from Oracle, using dbreplica.config"
3441  << std::endl <<
3442  "-sl, -seal <val>: Set SEAL (hence COOL/POOL) output level to <val>"
3443  << std::endl <<
3444  "-sr, -skiprep : Skip folders having <norep/> folder metadata tag" <<
3445  std::endl <<
3446  "-t, -tag <tag> : Copy multiversion data with tag <tag>" << std::endl <<
3447  "-mt, -magic <tag> : Include magic tags involving the given pattern " << std::endl <<
3448 "-tr, -truncate : Set destination IOVs outside query interval to query interval"
3449  << std::endl <<
3450 "-tl, -taglabel : Specify tag description to enter in destination DB"
3451  << std::endl <<
3452  "-uht, -userheadtag : Also copy user tag data to the HEAD" << std::endl <<
3453  "-v, -verify : Verify data present on destination rather than copying" <<
3454  std::endl <<
3455  "-ro, -root : Produce ROOT output file instead of copying to COOL"
3456  << std::endl <<
3457  "-zn, -zeronull : Zero NULLs in ROOT file instead of skipping them" <<
3458  std::endl <<
3459  "-rls, -runlumisince <run> <LB> : Set minimum IOV interval" << std::endl <<
3460  "-rlu, -runlumiuntil <run> <LB> : Set maximum IOV interval" << std::endl <<
3461  "-rs, -runsince <run> : Set minimum IOV interval" << std::endl <<
3462  "-ru, -rununtil <run> : Set maximum IOV interval" << std::endl <<
3463  "-r, -run <run> : Copy only run <run>" << std::endl <<
3464 "-srls, -skiprunlumisince <run> <LB> : Set minimum IOV for skipping IOV in copy"
3465  << std::endl <<
3466 "-srlu, -skiprunlumiuntil <run> <LB> : Set maximum IOV for skipping IOV in copy"
3467  << std::endl <<
3468  "-tdb, -timedb <dbconn> : Set database connection for time information" <<
3469  "-ts, -timesince <time> : Set minimum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3470  std::endl <<
3471  "-tu, -timeuntil <time> : Set maximum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3472  std::endl;
3473  std::cout <<
3474  "-nrls, -newrunlumisince <run> <LB> : Set minimum of output IOV interval (use with -alliov)" << std::endl <<
3475  "-nrlu, -newrunlumiuntil <run> <LB> : Set maximum of output IOV interval (use with --aliov)" << std::endl <<
3476  "-nts, -newtimesince <time> : Set minimum of outputIOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3477  std::endl <<
3478  "-ntu, -newtimeuntil <time> : Set maximum of output IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3479  std::endl;
3480  std::cout << "See http://twiki.cern.ch/twiki/bin/view/Atlas/AtlCoolCopy for more details" << std::endl;
3481 }
3482 
3483 int main ATLAS_NOT_THREAD_SAFE (int argc, const char* argv[]) {
3484  int retcode=0;
3485  if (argc<3) {
3486  printHelp();
3487  retcode=1;
3488  } else {
3489  AtlCoolCopy mycopy(argv[1],argv[2]);
3490  retcode=mycopy.setOpts(argc-3,&argv[3]);
3491  if (retcode==999) {
3492  printHelp();
3493  return 0;
3494  }
3495  if (retcode==0) retcode=mycopy.doCopy();
3496  }
3497  if (retcode>0) std::cout << "ERROR AtlCoolCopy.exe fails with exit code " <<
3498  retcode << std::endl;
3499  return retcode;
3500 }
SimpleUtilityBase.h
AtlCoolCopy::analyseIOVs
int analyseIOVs(const std::string &folder, const cool::IFolderPtr &sourcefl, const std::string &sourcetag, const cool::ValidityKey since, const cool::ValidityKey until, const bool timestamp)
Definition: AtlCoolCopy.cxx:1862
xAOD::iterator
JetConstituentVector::iterator iterator
Definition: JetConstituentVector.cxx:68
AtlCoolCopy::runLBVal
static cool::ValidityKey runLBVal(const char *input1, const char *input2)
Definition: AtlCoolCopy.cxx:2594
AtlCoolCopy::channelID
static cool::ChannelId channelID(const cool::IFolderPtr &folder, const std::string &chanstring)
Definition: AtlCoolCopy.cxx:860
AtlCoolCopy::m_skipout
bool m_skipout
Definition: AtlCoolCopy.cxx:203
AllowedVariables::e
e
Definition: AsgElectronSelectorTool.cxx:37
AtlCoolCopy::rootWrite
void rootWrite(void *sptr, const cool::IField &field) const
Definition: AtlCoolCopy.cxx:1776
AtlCoolCopy::m_magic
std::vector< std::string > m_magic
Definition: AtlCoolCopy.cxx:263
AtlCoolCopy::m_getbulk
bool m_getbulk
Definition: AtlCoolCopy.cxx:211
pool::SimpleUtilityBase::readFileGUID
virtual std::string readFileGUID(const std::string &pfn)
AtlCoolCopy::verifyIOVs
int verifyIOVs(const std::string &folder, const cool::IFolderPtr &sourcefl, const CoraCoolFolderPtr &sourceflc, const cool::IFolderPtr &destfl, const CoraCoolFolderPtr &destflc, const std::string &sourcetag, const cool::ValidityKey since, const cool::ValidityKey until, const bool checkrefs, const bool iscora, const cool::PayloadMode::Mode paymode)
Definition: AtlCoolCopy.cxx:1360
AtlCoolCopy::m_nt_rununtil
UInt_t m_nt_rununtil
Definition: AtlCoolCopy.cxx:287
AtlCoolCopy
Definition: AtlCoolCopy.cxx:80
AtlCoolCopy::m_outtag
std::string m_outtag
Definition: AtlCoolCopy.cxx:238
get_generator_info.result
result
Definition: get_generator_info.py:21
pool::IFileCatalog::Files
Gaudi::IFileCatalog::Files Files
Definition: IFileCatalog.h:26
AtlCoolCopy::m_channel1
std::string m_channel1
Definition: AtlCoolCopy.cxx:232
AtlCoolCopy::ATLAS_NOT_THREAD_SAFE
int doCopy ATLAS_NOT_THREAD_SAFE()
python.rtime.rtime
def rtime(fn, *args, **kw)
Definition: rtime.py:14
CheckTagAssociation.localtag
localtag
Definition: CheckTagAssociation.py:19
AtlCoolCopy::m_verify
bool m_verify
Definition: AtlCoolCopy.cxx:181
AtlCoolCopy::m_nt_lbuntil
UInt_t m_nt_lbuntil
Definition: AtlCoolCopy.cxx:289
AtlCoolCopy::m_srunemax
cool::ValidityKey m_srunemax
Definition: AtlCoolCopy.cxx:230
find
std::string find(const std::string &s)
return a remapped string
Definition: hcg.cxx:135
CoraCoolDatabase
Definition: CoraCoolDatabase.h:24
AtlCoolCopy::m_coralsvc
coral::ConnectionService m_coralsvc
Definition: AtlCoolCopy.cxx:252
json
nlohmann::json json
Definition: HistogramDef.cxx:9
AtlCoolCopy::m_copytaglock
bool m_copytaglock
Definition: AtlCoolCopy.cxx:215
AtlCoolCopy::getTimeFromRun
bool getTimeFromRun()
Definition: AtlCoolCopy.cxx:2613
response
MDT_Response response
Definition: MDT_ResponseTest.cxx:28
CaloCondBlobAlgs_fillNoiseFromASCII.spec
spec
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:46
AtlCoolCopy::m_readoracle
bool m_readoracle
Definition: AtlCoolCopy.cxx:207
xAOD::short
short
Definition: Vertex_v1.cxx:165
AtlCoolCopy::m_destdb
std::string m_destdb
Definition: AtlCoolCopy.cxx:172
AtlCoolCopy::m_repsort
ReplicaSorter * m_repsort
Definition: AtlCoolCopy.cxx:255
AtlCoolCopy::m_getonline
bool m_getonline
Definition: AtlCoolCopy.cxx:209
AtlCoolCopy::getUpdateMode
static int getUpdateMode(std::string_view desc, std::string_view tag)
Definition: AtlCoolCopy.cxx:2911
mergePhysValFiles.newfolder
newfolder
Definition: DataQuality/DataQualityUtils/scripts/mergePhysValFiles.py:82
AtlCoolCopy::m_checkoutputfile
std::string m_checkoutputfile
Definition: AtlCoolCopy.cxx:240
pool::FileCatalog::FileID
std::string FileID
Definition: IFileCatalog.h:20
defineDB.ichan
ichan
Definition: JetTagCalibration/share/defineDB.py:28
AtlCoolCopy::setChannelRange
void setChannelRange(const cool::IFolderPtr &sourcefl)
Definition: AtlCoolCopy.cxx:825
xAOD::char
char
Definition: TrigDecision_v1.cxx:38
parse
std::map< std::string, std::string > parse(const std::string &list)
Definition: egammaLayerRecalibTool.cxx:1113
extractSporadic.c1
c1
Definition: extractSporadic.py:133
tree
TChain * tree
Definition: tile_monitor.h:30
IDatabase.h
AtlCoolCopy::m_excludehead
bool m_excludehead
Definition: AtlCoolCopy.cxx:176
TRTCalib_cfilter.p1
p1
Definition: TRTCalib_cfilter.py:130
IFileCatalog.h
dirname
std::string dirname(std::string name)
Definition: utils.cxx:200
AtlCoolCopy::rootDirs
std::string rootDirs(const std::string &folder, const std::string &toproot)
Definition: AtlCoolCopy.cxx:1686
AtlCoolCopy::m_truncate
bool m_truncate
Definition: AtlCoolCopy.cxx:202
AtlCoolCopy::ATLAS_NOT_THREAD_SAFE
int copyFolder ATLAS_NOT_THREAD_SAFE(const std::string &folder, const std::vector< std::string > &taglist)
AtlCoolCopy::m_timemax
cool::ValidityKey m_timemax
Definition: AtlCoolCopy.cxx:224
bin
Definition: BinsDiffFromStripMedian.h:43
AtlCoolCopy::m_hitag
bool m_hitag
Definition: AtlCoolCopy.cxx:193
AtlCoolCopy::m_chansel
cool::ChannelSelection m_chansel
Definition: AtlCoolCopy.cxx:267
PlotCalibFromCool.nchan
nchan
Definition: PlotCalibFromCool.py:564
AtlCoolCopy::openConnections
bool openConnections(const std::string &sourcedb, const std::string &destdb, bool allowcreate)
Definition: AtlCoolCopy.cxx:339
AtlCoolCopy::m_userupdatehead
bool m_userupdatehead
Definition: AtlCoolCopy.cxx:178
python.subdetectors.tile.Blob
Blob
Definition: tile.py:17
CheckTagAssociation.taglist
taglist
Definition: CheckTagAssociation.py:103
AtlCoolCopy::m_parfile
std::vector< std::string > m_parfile
Definition: AtlCoolCopy.cxx:246
pool::IFileCatalog::disconnect
void disconnect()
Definition: IFileCatalog.h:40
AtlCoolCopy::m_hitagmap
HiTagMap m_hitagmap
Definition: AtlCoolCopy.cxx:273
pool::IFileCatalog::commit
void commit()
Save catalog to file.
Definition: IFileCatalog.h:49
defaults::Float
constexpr float Float
Definition: MSVtxValidationAlg.h:45
AtlCoolCopy::m_dbSvc
cool::IDatabaseSvc * m_dbSvc
Definition: AtlCoolCopy.cxx:254
exclude
std::set< std::string > exclude
list of directories to be excluded
Definition: hcg.cxx:95
ReadOfcFromCool.field
field
Definition: ReadOfcFromCool.py:48
AtlCoolCopy::m_newrunemin
cool::ValidityKey m_newrunemin
Definition: AtlCoolCopy.cxx:225
dbg::ptr
void * ptr(T *p)
Definition: SGImplSvc.cxx:74
AtlCoolCopy::m_usertags
bool m_usertags
Definition: AtlCoolCopy.cxx:177
AtlCoolCopy::m_srunemin
cool::ValidityKey m_srunemin
Definition: AtlCoolCopy.cxx:229
AtlCoolCopy::m_sourcedb
std::string m_sourcedb
Definition: AtlCoolCopy.cxx:171
CaloSwCorrections.gap
def gap(flags, cells_name, *args, **kw)
Definition: CaloSwCorrections.py:212
AtlCoolCopy::m_forcemulti
bool m_forcemulti
Definition: AtlCoolCopy.cxx:196
ITransaction.h
physics_parameters.url
string url
Definition: physics_parameters.py:27
ATLAS_NOT_THREAD_SAFE
int AtlCoolCopy::copyFolder ATLAS_NOT_THREAD_SAFE(const std::string &folder, const std::vector< std::string > &taglist)
Definition: AtlCoolCopy.cxx:535
AtlCoolCopy::equalRecord
static bool equalRecord(const cool::IRecord &lhs, const cool::IRecord &rhs)
Definition: AtlCoolCopy.cxx:2513
dq_defect_bulk_create_defects.line
line
Definition: dq_defect_bulk_create_defects.py:27
AtlCoolCopy::m_ignoremode
bool m_ignoremode
Definition: AtlCoolCopy.cxx:217
atlasStyleMacro.icol
int icol
Definition: atlasStyleMacro.py:13
AtlCoolCopy::m_newtimemax
cool::ValidityKey m_newtimemax
Definition: AtlCoolCopy.cxx:228
CoraCoolObject.h
cool
Definition: CoolTagInfo.h:12
dq_defect_copy_defect_database.channels
def channels
Definition: dq_defect_copy_defect_database.py:56
AtlCoolCopy::timeVal
static cool::ValidityKey timeVal(const char *input)
Definition: AtlCoolCopy.cxx:2545
AtlCoolCopy::m_coracool
bool m_coracool
Definition: AtlCoolCopy.cxx:216
AtlCoolCopy::m_applocksv
bool m_applocksv
Definition: AtlCoolCopy.cxx:206
pool::IFileCatalog::lookupFileByLFN
void lookupFileByLFN(const std::string &lfn, std::string &fid) const
Return the status of a LFName.
Definition: IFileCatalog.h:82
python.Dumpers.aname
string aname
Definition: Dumpers.py:5577
AtlCoolCopy::getCoolHistGUID
static std::string getCoolHistGUID(const std::string &file)
Definition: AtlCoolCopy.cxx:3274
dq_defect_copy_defect_database.since
def since
Definition: dq_defect_copy_defect_database.py:54
AtlCoolCopy::m_forcesingle
bool m_forcesingle
Definition: AtlCoolCopy.cxx:195
AtlCoolCopy::tagParents
int tagParents()
Definition: AtlCoolCopy.cxx:3043
python.checkMetadata.metadata
metadata
Definition: checkMetadata.py:175
AtlCoolCopy::m_gettime
bool m_gettime
Definition: AtlCoolCopy.cxx:208
AtlCoolCopy::checkRef
void checkRef(const cool::IRecord &payload, const std::string &folder, const std::string &tag)
Definition: AtlCoolCopy.cxx:3006
dq_defect_copy_defect_database.until
def until
Definition: dq_defect_copy_defect_database.py:55
AtlCoolCopy::m_checkrefs
bool m_checkrefs
Definition: AtlCoolCopy.cxx:185
ReplicaSorter.h
python.TileCalibTools.copyFolder
def copyFolder(dbr, dbw, folder, tagr, tagw, chanNum, pointInTime1, pointInTime2)
Definition: TileCalibTools.py:446
AtlCoolCopy::setupCatalog
static pool::IFileCatalog * setupCatalog(const std::vector< std::string > &catvec)
Definition: AtlCoolCopy.cxx:3327
python.setupRTTAlg.size
int size
Definition: setupRTTAlg.py:39
AtlCoolCopy::m_poolrefs
PoolMap m_poolrefs
Definition: AtlCoolCopy.cxx:270
skel.input1
tuple input1
Definition: skel.GENtoEVGEN.py:775
main
int main(int, char **)
Main class for all the CppUnit test classes
Definition: CppUnit_SGtestdriver.cxx:141
AtlCoolCopy::m_nt_lbsince
UInt_t m_nt_lbsince
Definition: AtlCoolCopy.cxx:288
TRTCalib_cfilter.p2
p2
Definition: TRTCalib_cfilter.py:131
AtlCoolCopy::m_newrunemax
cool::ValidityKey m_newrunemax
Definition: AtlCoolCopy.cxx:226
CoraCoolObject::const_iterator
AttrListVec::const_iterator const_iterator
Definition: CoraCoolObject.h:23
AtlCoolCopy::m_nt_since
ULong64_t m_nt_since
Definition: AtlCoolCopy.cxx:284
histSizes.code
code
Definition: histSizes.py:129
tags
std::vector< std::string > tags
Definition: hcg.cxx:102
AtlCoolCopy::m_nohitag
bool m_nohitag
Definition: AtlCoolCopy.cxx:194
pool::IFileCatalog
Definition: IFileCatalog.h:23
rerun_display.lfn
lfn
Definition: rerun_display.py:60
AtlCoolCopy::m_open
bool m_open
Definition: AtlCoolCopy.cxx:266
CaloCondBlobAlgs_fillNoiseFromASCII.desc
desc
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:53
AtlCoolCopy::m_poolall
bool m_poolall
Definition: AtlCoolCopy.cxx:188
AtlCoolCopy::m_nt_runsince
UInt_t m_nt_runsince
Definition: AtlCoolCopy.cxx:286
ParseInputs.gDirectory
gDirectory
Definition: Final2012/ParseInputs.py:133
AtlCoolCopy::m_skiprep
bool m_skiprep
Definition: AtlCoolCopy.cxx:204
lumiFormat.i
int i
Definition: lumiFormat.py:85
AtlCoolCopy::CoolTagMap
std::map< std::string, CoolTagInfo > CoolTagMap
Definition: AtlCoolCopy.cxx:278
ReadCellNoiseFromCool.chan
chan
Definition: ReadCellNoiseFromCool.py:52
LArCellNtuple.argv
argv
Definition: LArCellNtuple.py:152
python.LArBadChannelDBAlg.xFFFFFFFF
xFFFFFFFF
Definition: LArBadChannelDBAlg.py:73
AtlCoolCopy::m_alliov
bool m_alliov
Definition: AtlCoolCopy.cxx:180
AtlCoolCopy::m_nt_until
ULong64_t m_nt_until
Definition: AtlCoolCopy.cxx:285
maskDeadModules.run1
run1
Definition: maskDeadModules.py:40
AtlCoolCopy::m_channelRange
std::vector< std::string > m_channelRange
Definition: AtlCoolCopy.cxx:231
AtlCoolCopy::m_channel2
std::string m_channel2
Definition: AtlCoolCopy.cxx:233
AtlCoolCopy::m_ignorespec
bool m_ignorespec
Definition: AtlCoolCopy.cxx:218
AtlCoolCopy::m_checkdesttag
bool m_checkdesttag
Definition: AtlCoolCopy.cxx:219
AtlCoolCopy::m_poolcat
std::vector< std::string > m_poolcat
Definition: AtlCoolCopy.cxx:264
pool::IFileCatalog::start
void start()
redirect to init() for Gaudi FC
Definition: IFileCatalog.h:45
PlotPulseshapeFromCool.input
input
Definition: PlotPulseshapeFromCool.py:106
calibdata.exception
exception
Definition: calibdata.py:495
res
std::pair< std::vector< unsigned int >, bool > res
Definition: JetGroupProductTest.cxx:11
AtlCoolCopy::m_sourceread
bool m_sourceread
Definition: AtlCoolCopy.cxx:201
AtlCoolCopy::m_debug
bool m_debug
Definition: AtlCoolCopy.cxx:179
file
TFile * file
Definition: tile_monitor.h:29
AtlCoolCopy::m_excludechans
std::vector< cool::ChannelId > m_excludechans
Definition: AtlCoolCopy.cxx:247
ISession.h
AtlCoolCopy::adjustIOVs
void adjustIOVs(const cool::ValidityKey &since, const cool::ValidityKey &until, const cool::ValidityKey &qsince, const cool::ValidityKey &quntil, cool::ValidityKey &newsince, cool::ValidityKey &newuntil, const bool timestamp) const
Definition: AtlCoolCopy.cxx:1295
AtlCoolCopy::m_outfolder
std::string m_outfolder
Definition: AtlCoolCopy.cxx:237
Preparation.mode
mode
Definition: Preparation.py:107
AtlCoolCopy::ATLAS_NOT_THREAD_SAFE
int copyIOVs ATLAS_NOT_THREAD_SAFE(const std::string &folder, const std::string &destfolder, const cool::IFolderPtr &sourcefl, const CoraCoolFolderPtr &sourceflc, const cool::IFolderPtr &destfl, const CoraCoolFolderPtr &destflc, const std::string &sourcetag, const std::string &desttag, const cool::ValidityKey since, const cool::ValidityKey until, bool timestamp, bool checkrefs, bool iscora, const cool::PayloadMode::Mode paymode, bool created)
pool::IFileCatalog::setWriteCatalog
void setWriteCatalog(const std::string &connect)
Access to the (first) writable file catalog.
run
Definition: run.py:1
AtlCoolCopy::m_lockedonly
bool m_lockedonly
Definition: AtlCoolCopy.cxx:213
DQHistogramMergeRegExp.argc
argc
Definition: DQHistogramMergeRegExp.py:19
xAOD::double
double
Definition: CompositeParticle_v1.cxx:159
PoolMapElement.h
AtlCoolCopy::m_coolapp
cool::Application m_coolapp
Definition: AtlCoolCopy.cxx:253
AtlCoolCopy::m_listpfn
bool m_listpfn
Definition: AtlCoolCopy.cxx:186
AtlCoolCopy::procOptVector
bool procOptVector(const int argc, const char *argv[], std::vector< std::string > &folders)
Definition: AtlCoolCopy.cxx:2136
AtlCoolCopy::m_forcerune
bool m_forcerune
Definition: AtlCoolCopy.cxx:197
AtlCoolCopy::m_sourceDbPtr
cool::IDatabasePtr m_sourceDbPtr
Definition: AtlCoolCopy.cxx:256
contents
void contents(std::vector< std::string > &keys, TDirectory *td, const std::string &directory, const std::string &pattern, const std::string &path)
Definition: computils.cxx:321
pool::IFileCatalog::connect
void connect()
Definition: IFileCatalog.h:39
CoraCoolFolderPtr
boost::shared_ptr< CoraCoolFolder > CoraCoolFolderPtr
Definition: CoraCoolTypes.h:15
AtlCoolCopy::m_runemax
cool::ValidityKey m_runemax
Definition: AtlCoolCopy.cxx:222
AtlCoolCopy::m_folderlist
std::vector< std::string > m_folderlist
Definition: AtlCoolCopy.cxx:260
AtlCoolCopy::m_poolopen
bool m_poolopen
Definition: AtlCoolCopy.cxx:187
AtlCoolCopy::m_runlist
std::vector< unsigned int > m_runlist
Definition: AtlCoolCopy.cxx:249
CxxUtils::atof
double atof(std::string_view str)
Converts a string into a double / float.
Definition: Control/CxxUtils/Root/StringUtils.cxx:91
pool_uuid.guid
guid
Definition: pool_uuid.py:112
grepfile.ic
int ic
Definition: grepfile.py:33
ReadCellNoiseFromCoolCompare.run2
run2
Definition: ReadCellNoiseFromCoolCompare.py:53
AtlCoolCopy::m_onlinerun
bool m_onlinerun
Definition: AtlCoolCopy.cxx:210
CoraCoolDatabasePtr
boost::shared_ptr< CoraCoolDatabase > CoraCoolDatabasePtr
Definition: CoraCoolTypes.h:12
CoraCoolFolder.h
AtlCoolCopy::m_sealmsg
int m_sealmsg
Definition: AtlCoolCopy.cxx:235
AtlCoolCopy::m_sourceCoraPtr
CoraCoolDatabasePtr m_sourceCoraPtr
Definition: AtlCoolCopy.cxx:258
AtlCoolCopy::m_applock
bool m_applock
Definition: AtlCoolCopy.cxx:205
AtlCoolCopy::ATLAS_NOT_THREAD_SAFE
int resolvePoolRefs ATLAS_NOT_THREAD_SAFE()
AtlCoolCopy::listPoolRefs
int listPoolRefs()
Definition: AtlCoolCopy.cxx:3116
AtlCoolCopy::PoolMap
std::map< std::string, PoolMapElement > PoolMap
Definition: AtlCoolCopy.cxx:268
AtlCoolCopy::checkChannels
bool checkChannels(const std::string &folder, const cool::IFolderPtr &sourcefl, const cool::IFolderPtr &destfl, bool newfolder)
Definition: AtlCoolCopy.cxx:2929
AtlCoolCopy::m_taglabel
std::string m_taglabel
Definition: AtlCoolCopy.cxx:242
AtlCoolCopy::m_timemin
cool::ValidityKey m_timemin
Definition: AtlCoolCopy.cxx:223
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:240
printHelp
void printHelp()
Definition: AtlCoolCopy.cxx:3353
AtlCoolCopy::m_destDbPtr
cool::IDatabasePtr m_destDbPtr
Definition: AtlCoolCopy.cxx:257
AtlCoolCopy::nocopyIOVs
int nocopyIOVs(const std::string &folder, const cool::IFolderPtr &sourcefl, const std::string &sourcetag, const cool::ValidityKey since, const cool::ValidityKey until, bool checkrefs)
Definition: AtlCoolCopy.cxx:1326
AtlCoolCopy::filePoolRefs
void filePoolRefs()
Definition: AtlCoolCopy.cxx:3290
AtlCoolCopy::m_nt_bufferptr
std::vector< void * > m_nt_bufferptr
Definition: AtlCoolCopy.cxx:293
AtlCoolCopy::getBulkRun
bool getBulkRun()
Definition: AtlCoolCopy.cxx:2790
EventContainers::Mode
Mode
Definition: IdentifiableContainerBase.h:13
pool::SimpleUtilityBase
Common base class for POOL utilities.
Definition: SimpleUtilityBase.h:23
AtlCoolCopy::HiTagMap
std::map< std::string, std::string > HiTagMap
Definition: AtlCoolCopy.cxx:272
AtlCoolCopy::m_runemin
cool::ValidityKey m_runemin
Definition: AtlCoolCopy.cxx:221
AtlCoolCopy::writeTagInfo
int writeTagInfo()
Definition: AtlCoolCopy.cxx:3105
AtlCoolCopy::m_prunetags
bool m_prunetags
Definition: AtlCoolCopy.cxx:212
AtlCoolCopy::p_rootfile
TFile * p_rootfile
Definition: AtlCoolCopy.cxx:281
mc.nskip
nskip
Definition: mc.PhPy8EG_Hto4l_NNLOPS_nnlo_30_ggH125_ZZ4l.py:41
AtlCoolCopy::m_bufsize
int m_bufsize
Definition: AtlCoolCopy.cxx:234
AtlCoolCopy::m_timedb
std::string m_timedb
Definition: AtlCoolCopy.cxx:241
AtlCoolCopy::m_analyse
bool m_analyse
Definition: AtlCoolCopy.cxx:184
AtlCoolCopy::m_poollast
PoolMap::iterator m_poollast
Definition: AtlCoolCopy.cxx:269
PixelModuleFeMask_create_db.payload
string payload
Definition: PixelModuleFeMask_create_db.py:69
AtlCoolCopy::m_tags
std::vector< std::string > m_tags
Definition: AtlCoolCopy.cxx:262
AtlCoolCopy::m_nochannel
bool m_nochannel
Definition: AtlCoolCopy.cxx:191
AtlCoolCopy::m_mergecat
std::vector< std::string > m_mergecat
Definition: AtlCoolCopy.cxx:265
pool::IFileCatalog::getFirstPFN
void getFirstPFN(const std::string &fid, std::string &pfn, std::string &tech) const
Get the first PFN + filetype for the given FID.
ir
int ir
counter of the current depth
Definition: fastadd.cxx:49
AtlCoolCopy::m_runinfohost
std::string m_runinfohost
Definition: AtlCoolCopy.cxx:243
CoolTagInfo
Definition: CoolTagInfo.h:16
DeMoAtlasDataLoss.runNumber
string runNumber
Definition: DeMoAtlasDataLoss.py:64
AtlCoolCopy::m_nt_tagid
char m_nt_tagid[256]
Definition: AtlCoolCopy.cxx:291
AtlCoolCopy::m_forcepay
bool m_forcepay
Definition: AtlCoolCopy.cxx:199
python.DataFormatRates.c2
c2
Definition: DataFormatRates.py:123
AtlCoolCopy::m_nodata
bool m_nodata
Definition: AtlCoolCopy.cxx:190
TileSynchronizeBch.copyall
copyall
Definition: TileSynchronizeBch.py:47
AtlCoolCopy::bookOrFindTH1F
static TH1F * bookOrFindTH1F(const std::string &hID, const std::string &htitle, const int chan, const float xlow, const float xhigh)
Definition: AtlCoolCopy.cxx:2083
AtlCoolCopy::m_runfile
std::vector< std::string > m_runfile
Definition: AtlCoolCopy.cxx:248
Athena::Status
Status
Athena specific StatusCode values.
Definition: AthStatusCode.h:22
python.CaloAddPedShiftConfig.int
int
Definition: CaloAddPedShiftConfig.py:45
CoraCoolObjectIterPtr
boost::shared_ptr< CoraCoolObjectIter > CoraCoolObjectIterPtr
Definition: CoraCoolTypes.h:21
CaloSwCorrections.time
def time(flags, cells_name, *args, **kw)
Definition: CaloSwCorrections.py:242
CaloCellTimeCorrFiller.foldertag
string foldertag
Definition: CaloCellTimeCorrFiller.py:20
makeDTCalibBlob_pickPhase.folders
folders
Definition: makeDTCalibBlob_pickPhase.py:346
AtlCoolCopy::addFolder
bool addFolder(const std::string &folder, const bool onlyTags)
Definition: AtlCoolCopy.cxx:460
Pythia8_RapidityOrderMPI.val
val
Definition: Pythia8_RapidityOrderMPI.py:14
AtlCoolCopy::m_cooltagmap
CoolTagMap m_cooltagmap
Definition: AtlCoolCopy.cxx:279
AtlCoolCopy::m_copytaginfo
bool m_copytaginfo
Definition: AtlCoolCopy.cxx:214
AtlCoolCopy::transConn
static std::string transConn(const std::string &inconn)
Definition: AtlCoolCopy.cxx:449
AtlCoolCopy::m_includehead
bool m_includehead
Definition: AtlCoolCopy.cxx:175
AtlCoolCopy::getRunList
bool getRunList()
Definition: AtlCoolCopy.cxx:2882
CoraCoolObjectPtr
boost::shared_ptr< CoraCoolObject > CoraCoolObjectPtr
Definition: CoraCoolTypes.h:18
AtlCoolCopy::m_newdataset
std::string m_newdataset
Definition: AtlCoolCopy.cxx:239
makeDTCalibBlob_pickPhase.outtag
string outtag
Definition: makeDTCalibBlob_pickPhase.py:286
AtlCoolCopy::m_nt_treename
std::string m_nt_treename
Definition: AtlCoolCopy.cxx:292
AtlCoolCopy::m_root
bool m_root
Definition: AtlCoolCopy.cxx:182
python.SystemOfUnits.s
float s
Definition: SystemOfUnits.py:147
CoraCoolDatabase.h
CaloCondBlobAlgs_fillNoiseFromASCII.folder
folder
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:55
AtlCoolCopy::isOpen
bool isOpen() const
Definition: AtlCoolCopy.cxx:296
AtlCoolCopy::m_recreate
bool m_recreate
Definition: AtlCoolCopy.cxx:174
AtlCoolCopy::rootAllocate
bool rootAllocate(const cool::IFieldSpecification &spec, void *&sptr, char &rootID) const
Definition: AtlCoolCopy.cxx:1720
AtlCoolCopy::m_destCoraPtr
CoraCoolDatabasePtr m_destCoraPtr
Definition: AtlCoolCopy.cxx:259
CxxUtils::atoi
int atoi(std::string_view str)
Helper functions to unpack numbers decoded in string into integers and doubles The strings are requir...
Definition: Control/CxxUtils/Root/StringUtils.cxx:85
CoolTagInfo.h
CoraCoolObjectIter.h
WriteCallback
size_t WriteCallback(void *contents, size_t size, size_t nmemb, std::string *s)
Definition: AtlCoolCopy.cxx:64
AtlCoolCopy::m_nocopy
bool m_nocopy
Definition: AtlCoolCopy.cxx:189
python.TrigEgammaMonitorHelper.TH1F
def TH1F(name, title, nxbins, bins_par2, bins_par3=None, path='', **kwargs)
Definition: TrigEgammaMonitorHelper.py:24
LArCellNtuple.ifile
string ifile
Definition: LArCellNtuple.py:133
AtlCoolCopy::getLockedTags
bool getLockedTags()
Definition: AtlCoolCopy.cxx:429
AtlCoolCopy::m_nt_channel
UInt_t m_nt_channel
Definition: AtlCoolCopy.cxx:290
AtlCoolCopy::m_zeronull
bool m_zeronull
Definition: AtlCoolCopy.cxx:183
CaloCondBlobAlgs_fillNoiseFromASCII.tag
string tag
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:23
AtlCoolCopy::AtlCoolCopy
AtlCoolCopy(const std::string &sourcedb, const std::string &destdb, bool allowcreate=false)
Definition: AtlCoolCopy.cxx:300
AtlCoolCopy::m_noclobroot
bool m_noclobroot
Definition: AtlCoolCopy.cxx:220
AtlCoolCopy::setOpts
int setOpts(int argc, const char *argv[])
Definition: AtlCoolCopy.cxx:2383
AtlCoolCopy::m_addlfn
std::vector< std::string > m_addlfn
Definition: AtlCoolCopy.cxx:245
get_generator_info.error
error
Definition: get_generator_info.py:40
AtlCoolCopy::addExclude
bool addExclude(const std::string &folder)
Definition: AtlCoolCopy.cxx:528
AtlCoolCopy::m_anadelt
long long m_anadelt
Definition: AtlCoolCopy.cxx:236
checker_macros.h
Define macros for attributes used to control the static checker.
python.PyAthena.obj
obj
Definition: PyAthena.py:132
Cut::all
@ all
Definition: SUSYToolsAlg.cxx:67
error
Definition: IImpactPoint3dEstimator.h:70
AtlCoolCopy::m_hiparent
std::vector< std::string > m_hiparent
Definition: AtlCoolCopy.cxx:275
Amg::distance
float distance(const Amg::Vector3D &p1, const Amg::Vector3D &p2)
calculates the distance between two point in 3D space
Definition: GeoPrimitivesHelpers.h:54
AtlCoolCopy::m_chdesc
bool m_chdesc
Definition: AtlCoolCopy.cxx:192
AtlCoolCopy::timeString
static std::string timeString(const cool::ValidityKey iovtime)
Definition: AtlCoolCopy.cxx:2581
AtlCoolCopy::rootIOVs
int rootIOVs(const std::string &folder, const cool::IFolderPtr &sourcefl, const std::string &sourcetag, const cool::ValidityKey since, const cool::ValidityKey until, const bool timestamp)
Definition: AtlCoolCopy.cxx:1555
pool::IFileCatalog::addReadCatalog
void addReadCatalog(const std::string &connect)
Add new catalog identified by name to the existing ones.
Definition: IFileCatalog.h:116
ReplicaSorter
Definition: Database/CoolConvUtilities/src/ReplicaSorter.h:12
AtlCoolCopy::m_folderexcl
std::vector< std::string > m_folderexcl
Definition: AtlCoolCopy.cxx:261
AtlCoolCopy::m_allowcreate
bool m_allowcreate
Definition: AtlCoolCopy.cxx:173
AtlCoolCopy::m_forcetime
bool m_forcetime
Definition: AtlCoolCopy.cxx:198
CaloCondBlobAlgs_fillNoiseFromASCII.blob
blob
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:95
AtlCoolCopy::getOnlineRun
bool getOnlineRun()
Definition: AtlCoolCopy.cxx:2702
python.SystemOfUnits.L
float L
Definition: SystemOfUnits.py:92
python.LArMinBiasAlgConfig.float
float
Definition: LArMinBiasAlgConfig.py:65
AtlCoolCopy::m_forcenopay
bool m_forcenopay
Definition: AtlCoolCopy.cxx:200
AtlCoolCopy::isNumeric
static bool isNumeric(const char *input)
Definition: AtlCoolCopy.cxx:2502
AtlCoolCopy::m_addguid
std::vector< std::string > m_addguid
Definition: AtlCoolCopy.cxx:244
AtlCoolCopy::openCoraCool
bool openCoraCool()
Definition: AtlCoolCopy.cxx:409
AtlCoolCopy::m_newtimemin
cool::ValidityKey m_newtimemin
Definition: AtlCoolCopy.cxx:227
PoolMapElement
Definition: PoolMapElement.h:14
pool::IFileCatalog::getLFNs
void getLFNs(const std::string &fid, Files &files) const
Get all logical names for a given FID. Return pairs <LFN,FID>
Definition: IFileCatalog.h:61