 |
ATLAS Offline Software
|
Go to the documentation of this file.
8 #include "GaudiKernel/Bootstrap.h"
9 #include "GaudiKernel/IOpaqueAddress.h"
10 #include "GaudiKernel/GenericAddress.h"
11 #include "GaudiKernel/IAddressCreator.h"
12 #include "GaudiKernel/ISvcLocator.h"
15 #include "CoolKernel/IObject.h"
16 #include "CoolKernel/IObjectIterator.h"
17 #include "CoolKernel/IRecord.h"
18 #include "CoolKernel/IRecordIterator.h"
19 #include "CoralBase/AttributeList.h"
20 #include "CoralBase/AttributeListSpecification.h"
21 #include "CoralBase/Attribute.h"
22 #include "CoralBase/AttributeSpecification.h"
23 #include "CoralBase/Blob.h"
24 #include "TStopwatch.h"
57 #include "CrestApi/CrestApiFs.h"
63 using namespace Crest;
66 const std::string fileSuffix{
".json"};
74 const bool checklock,
const bool outputToFile,
75 const std::string &
source,
77 const std::string & crestTag,
78 const bool crestCoolToFile):
81 p_metaDataTool(metadatatool),
83 m_checklock(checklock),
85 m_chansel(
cool::ChannelSelection::
all()),
86 m_outputToFile{outputToFile},
87 m_crestCoolToFile{crestCoolToFile},
96 m_foldername=folderprop.folderName();
100 m_key=folderprop.key();
101 m_jokey=folderprop.hasKey();
103 m_jotag=folderprop.tag();
105 m_eventstore=folderprop.eventStoreName();
107 m_cachepar = folderprop.cache();
109 m_notagoverride=folderprop.noTagOverride();
110 if (m_source ==
"CREST"){
113 if (m_notagoverride)
ATH_MSG_INFO(
"Inputfile tag override disabled for " << m_foldername );
118 std::string chanspec;
119 if (folderprop.getKey(
"channelSelection",
"",chanspec) && !chanspec.empty()) {
120 m_chanrange=IOVDbNamespace::parseChannelSpec<cool::ChannelId>(chanspec);
125 for(
const auto &
i:m_chanrange){
128 m_chansel = cool::ChannelSelection(
i.first,
i.second);
130 m_chansel.addRange(
i.first,
i.second);
133 }
catch (cool::Exception&
e) {
134 ATH_MSG_ERROR(
"defining channel range (must be given in ascending order)");
138 if (folderprop.overridesIov(
msg)){
139 m_iovoverridden=
true;
140 m_iovoverride=folderprop.iovOverrideValue(
msg);
142 ATH_MSG_INFO(
"Override timestamp to " << m_iovoverride <<
" for folder " << m_foldername );
145 ATH_MSG_INFO(
"Override run/LB number to [" <<
run <<
":" <<
lumi <<
"] for folder " << m_foldername );
149 m_fromMetaDataOnly=folderprop.onlyReadMetadata();
150 if (m_fromMetaDataOnly) {
151 ATH_MSG_INFO(
"Read from meta data only for folder " << m_foldername );
154 m_extensible=folderprop.extensible();
193 const unsigned int time) {
220 return (
m_timestamp ? reftime.timestamp() : reftime.re_time());
226 const unsigned int cacheDiv,
228 const bool ignoreMissChan) {
235 TStopwatch cachetimer;
238 bool vectorPayload{};
242 vectorPayload =
m_crest_mng.value().isVectorPayload();
261 if (cacheq>0) changedCacheLo=vkey - vkey % cacheq;
267 changedCacheLo=vkey & (0x7FFFFFFFLL << 32);
273 if (changedCacheHi>cool::ValidityKeyMax) changedCacheHi=cool::ValidityKeyMax;
304 bool retrievedone=
false;
313 unsigned int attempts=0;
315 ATH_MSG_DEBUG(
"loadCache: Expecting to see " << nChannelsExpected <<
" channels" );
317 while (attempts<2 && !retrievedone) {
324 if (not
m_conn->
valid())
throw std::runtime_error(
"COOL database pointer invalidated");
340 while (itr->hasNext()) {
374 while (itr->goToNext()) {
375 const cool::IObject&
ref=itr->currentRef();
383 cool::IRecordIterator& pitr=
ref.payloadIterator();
384 const cool::IRecordVectorPtr&
pvec=pitr.fetchAllAsVector();
385 for (cool::IRecordVector::const_iterator vitr=
pvec->begin();vitr!=
pvec->end();++vitr) {
417 ATH_MSG_WARNING(
"COOL retrieve attempt " << attempts <<
" failed: " <<
e.what() );
425 unsigned int iadd = 0;
432 const auto & channelNumbers=
m_crest_mng.value().channelIds(
iov.first);
433 for (
auto const &
chan: channelNumbers){
436 std::istringstream tokenStream(
chan);
437 std::getline(tokenStream, token,
':');
442 for (
const auto & attList:vPayload){
471 "," <<
until <<
"]" );
477 const auto missing=std::pair<unsigned int, unsigned int>(nChannelsExpected-nChannelsLo, nChannelsExpected-nChannelsHi);
478 ATH_MSG_DEBUG(
"Cache retrieve missing " << missing.first <<
" lower and " << missing.second <<
" upper channels" );
484 ATH_MSG_DEBUG(
"Lower cache limit extended from " << cacheStart <<
" to " <<
span.first );
488 ATH_MSG_DEBUG(
"Upper cache limit extended from " << cacheStop <<
" tp " <<
span.second );
492 const float timeinc=cachetimer.RealTime();
495 m_iovs.
size() <<
" objects stored in" << std::fixed <<
496 std::setw(8) << std::setprecision(2) << timeinc <<
" s" );
502 const cool::IDatabasePtr& ,
511 unsigned int attempts = 0;
512 bool retrievedone =
false;
515 ATH_MSG_DEBUG(
"loadCacheIfDbChanged: Expecting to see " << nChannelsExpected <<
" channels" );
517 while (attempts<2 && !retrievedone) {
546 const cool::IObject&
ref=itr->currentRef();
556 ATH_MSG_WARNING(
"COOL retrieve attempt " << attempts <<
" failed: " <<
e.what() );
570 ATH_MSG_ERROR(
"IOVDbFolder::specialCacheUpdate - setRange failed for folder "
596 ATH_MSG_ERROR(
"IOVDbFolder::specialCacheUpdate - setRange failed for folder "
626 IAddressCreator* persSvc,
627 const unsigned int poolSvcContext,
628 std::unique_ptr<IOpaqueAddress>&
address,
629 IOVRange&
range,
bool& poolPayloadReq) {
633 std::string strAddress;
637 cool::ValidityKey naystart=0;
638 cool::ValidityKey naystop=cool::ValidityKeyMax;
642 if (not readFromMetaData.
isValid()){
672 if (thisIov.first<=reftime && reftime<thisIov.second) {
678 strAddress=
"POOLContainer_AthenaAttributeList][CLID=x";
696 ATH_MSG_ERROR(
"Writing of CoraCool folders to file metadata not implemented");
703 }
else if (thisIov.second<=reftime && thisIov.second>naystart) {
704 naystart=thisIov.second;
705 }
else if (thisIov.first>reftime && thisIov.first<naystop) {
706 naystop=thisIov.first;
713 std::vector<std::string>::const_iterator nitr=
m_channames.begin();
714 for (std::vector<cool::ChannelId>::const_iterator chitr=
m_channums.begin();
716 attrListColl->
add(*chitr,*nitr);
721 strAddress=
"POOLContainer_CondAttrListCollection][CLID=x";
724 strAddress=
"POOLContainer_CondAttrListVec][CLID=x";
728 ATH_MSG_ERROR(
"COOL object not found in single-channel retrieve, folder "
733 " valid objects found for single-channel retrieve, folder " <<
739 <<
" at IOV " << reftime <<
" channels " << nobj <<
" has range "
744 IOVTime rstart=
range.start();
745 IOVTime rstop=
range.stop();
746 if (tnaystart > rstart || rstop > tnaystop) {
748 <<
" from [" << rstart <<
":" << rstop <<
"] to ["
749 << tnaystart <<
":" << tnaystop <<
"]" );
750 if (tnaystart > rstart) rstart=tnaystart;
751 if (tnaystop < rstop) rstop=tnaystop;
752 range=IOVRange(rstart,rstop);
770 IOpaqueAddress* addrp =
nullptr;
771 if (StatusCode::SUCCESS!=persSvc->createAddress(0,0,strAddress,addrp)) {
772 ATH_MSG_ERROR(
"Could not get IOpaqueAddress from string address "<< strAddress );
775 address = std::unique_ptr<IOpaqueAddress>(addrp);
776 GenericAddress* gAddr=
dynamic_cast<GenericAddress*
>(
address.get());
778 ATH_MSG_ERROR(
"Could not cast IOpaqueAddress to GenericAddress");
783 auto addr = std::make_unique<CondAttrListCollAddress>(*gAddr);
784 addr->setAttrListColl(attrListColl);
787 auto addr = std::make_unique<CondAttrListCollAddress>(gAddr->svcType(),
788 gAddr->clID(),gAddr->par()[0],gAddr->par()[1],
789 poolSvcContext,gAddr->ipar()[1]);
791 addr->setAttrListColl(attrListColl);
796 auto addr = std::make_unique<AthenaAttrListAddress>(*gAddr);
797 addr->setAttrList(attrList);
800 auto addr = std::make_unique<CondAttrListVecAddress>(*gAddr);
801 addr->setAttrListVec(attrListVec);
815 << std::setprecision(2) <<
m_readtime <<
" ))s" );
828 if (
auto newkey=parsedDescription.
key(); not newkey.empty() and not
m_jokey) {
839 if (
auto newAddrHeader = parsedDescription.
addressHeader();not newAddrHeader.empty()){
864 std::unique_ptr<SG::TransientAddress>
866 auto tad = std::make_unique<SG::TransientAddress>(
m_clid,
m_key);
868 for (
const auto & linkname:symlinks){
869 if (not linkname.empty()) {
871 if (StatusCode::SUCCESS==
p_clidSvc->getIDOfTypeName(linkname,sclid)) {
872 tad->setTransientID(sclid);
875 ATH_MSG_ERROR(
"Could not get clid for symlink: "<< linkname );
883 std::unique_ptr<SG::TransientAddress>
937 const auto & linknameVector = folderpar.
symLinks();
949 if (timeIs_nsOfEpoch){
950 long long int clen=600;
959 ATH_MSG_DEBUG(
"Cache length set to " << clen <<
" seconds" );
992 if (
tag==
"HEAD")
return true;
995 ATH_MSG_ERROR(
"No IOVDbSvc.GlobalTag specified on job options or input file" );
1008 const std::vector<std::string>&
taglist=
fptr->listTags();
1015 std::string restag=
fptr->resolveTag(
tag);
1020 }
catch (cool::Exception&
e) {
1029 if (not tagLock.has_value()){
1033 if (not tagLock.value()){
1034 ATH_MSG_ERROR(
"Tag " <<
tag <<
" is not locked and IOVDbSvc.CheckLock is set" );
1045 return (not
tag.empty());
1052 const IOVRange&
range) {
1057 tmpColl.
add(0xFFFF,atrlist);
1078 m_cachespec=
new coral::AttributeListSpecification;
1079 for (
const auto & attribute:atrlist){
1080 const coral::AttributeSpecification& aspec=attribute.specification();
1084 " in folder " <<
m_foldername <<
" will not be counted for bytes-read statistics" );
1106 ATH_MSG_DEBUG(
"channelID:\t"<<(*ci++)<<
"\t since: "<<
iov.first<<
"\t until: "<<
iov.second);
1114 std::vector<IOVHash>
result;
1117 std::vector<std::pair<cool::ValidityKey,std::string>> crestIOVs =
m_crest_mng.value().getIovsForTag(
since,
until);
1118 size_t nIOVs = crestIOVs.size();
1119 if(crestIOVs.empty()){
1126 for(
size_t ind=0; ind<nIOVs-1; ++ind) {
1137 ,
const cool::ValidityKey& vkey
1141 ,
const cool::ValidityKey crestVkey
1145 std::string fMain(dumpName);
1148 std::filesystem::create_directory(fMain);
1150 if (not
myFile.is_open()) {
1151 std::string errorMessage{
"File creation for "+fabricatedName+
" failed."};
1153 throw std::runtime_error(errorMessage);
1162 myFile<<
json->description()<<s_delimiterJson<<std::endl;
1163 myFile<<
json->payloadSpec()<<s_delimiterJson<<std::endl;
1165 myFile<<
json->iov()<<s_delimiterJson<<std::endl;
1172 myFile<<
"\"folder_payloadspec\": \""<<mng->
getPayloadSpec()<<
'\"'<<s_delimiterJson<<std::endl;
1181 if(iovHashVect.empty() ||
until<=iovHashVect[0].first.first) {
1182 if(iovHashVect.empty()) {
1190 unsigned indIOVStart = 0;
1191 for(
const auto& iovhash : iovHashVect) {
1192 if(vkey>=iovhash.first.first && vkey<iovhash.first.second)
1196 unsigned indIOVEnd = indIOVStart;
1197 while(indIOVEnd < iovHashVect.size()) {
1198 if(iovHashVect[indIOVEnd].
first.first <
until
1199 && iovHashVect[indIOVEnd].first.second >=
until) {
1204 std::vector<IOVDbFolder::IOVHash> resIovHashVect;
1205 for(
unsigned ind = indIOVStart; ind <= indIOVEnd; ++ind) {
1206 std::vector<uint64_t> resIovs=
m_crest_mng.value().loadPayloadForHash(iovHashVect[ind].
first.first,iovHashVect[ind].second);
1207 if(resIovs.size()>1){
1208 uint64_t sTmp=iovHashVect[ind].first.first;
1210 for(
unsigned int i=1;
i<resIovs.size();
i++){
1215 if(sTmp!=iovHashVect[ind].
first.second){
1216 resIovHashVect.emplace_back(
IovStore::Iov_t(sTmp, iovHashVect[ind].
first.second),iovHashVect[ind].second);
1219 else if(resIovs.size()==1){
1220 resIovHashVect.emplace_back(
IovStore::Iov_t(resIovs[0],iovHashVect[ind].
first.second),iovHashVect[ind].second);
1226 for(
const auto& iovhash : resIovHashVect) {
1227 if(vkey>=iovhash.first.first && vkey<iovhash.first.second)
1229 if(indIOVStart+1<resIovHashVect.size())
1233 dumpFile(
"crest_dump",vkey,
nullptr,
false,&
m_crest_mng.value(),resIovHashVect[indIOVStart].first.first);
1235 return resIovHashVect;
JetConstituentVector::iterator iterator
const IOVRange & minRange() const
Current minimal IOVRange.
std::string spaceStrip(const std::string &input)
Trim leading and trailing spaces,return a new trimmed string.
const bool m_outputToFile
void dumpFile(const std::string &dumpName, const cool::ValidityKey &vkey, IOVDbNamespace::Cool2Json *json, bool skipCoolIoV, CoralCrestManager *mng=NULL, const cool::ValidityKey crestVkey=0) const
span(T *ptr, std::size_t sz) -> span< T >
A couple needed deduction guides.
std::vector< unsigned int > m_cacheccend
const bool m_crestCoolToFile
bool magicTag(std::string &tag)
IIOVDbMetaDataTool * p_metaDataTool
This file defines the class for a collection of AttributeLists where each one is associated with a ch...
std::string find(const std::string &s)
return a remapped string
unsigned int m_ncacheread
std::vector< Iov_t > & vectorStore()
Return the internal vector store.
This file contains the class definition for the CondAttrListCollAddress class.
Iov_t getMinimumStraddlingSpan() const
Get the minimum straddling span (max 'since' to min 'until')
void addNewStop(const IOVTime &stop)
Add new stop time to minRange - make sure that stop is <= to new stop
const std::string m_crestTag
constexpr unsigned long long ALL_LUMI_BLOCKS
All the lumi blocks in one run.
cool::ValidityKey iovTime(const IOVTime &reftime) const
std::pair< unsigned int, unsigned int > numberOfIovsOnBoundaries() const
Return the current count of added Iovs which straddled cache boundaries.
const std::string & name() const
void clear()
clear the store
std::unique_ptr< SG::TransientAddress > createTransientAddress(const std::vector< std::string > &symlinks)
unsigned int size() const
length of store
void specialCacheUpdate(CoraCoolObject &obj, const ServiceHandle< IIOVSvc > &iovSvc)
std::string dumpPayload(cool::ValidityKey since)
int run(int argc, char *argv[])
cool::ValidityKey m_cachelength
bool getAddress(const cool::ValidityKey reftime, IAddressCreator *persSvc, const unsigned int poolSvcContext, std::unique_ptr< IOpaqueAddress > &address, IOVRange &range, bool &poolPayloadRequested)
void addNewStart(const IOVTime &start)
Add new start time to minRange - make sure that start is >= to new start.
unsigned long long iovTimeFromSeconds(const unsigned long long seconds)
Create a long long time in ns from s.
const std::string & eventStore() const
cool::ChannelSelection m_chansel
void setTagOverride(const std::string &tag, const bool setFlag)
std::vector< IOVHash > fetchCrestIOVs(cool::ValidityKey since, cool::ValidityKey until)
ITagInfoMgr * p_tagInfoMgr
IOVTime makeEpochOrRunLumi(const cool::ValidityKey key, const bool timeIsEpoch)
Create an IOVTime in ns of epoch or run-lumi (determined by the bool) from a ValidityKey.
This class is a collection of AttributeLists where each one is associated with a channel number....
std::pair< cool::ValidityKey, cool::ValidityKey > Iov_t
void setSharedSpec(const coral::AttributeList &atrlist)
bool loadCache(const cool::ValidityKey vkey, const unsigned int cacheDiv, const std::string &globalTag, const bool ignoreMissChan)
bool replaceServiceType71(std::string &addrHeader)
const std::string & folderName() const
T getFolderPtr(const std::string &folderName)
IOVDbNamespace::FolderType m_foldertype
void setLevel(MsgLevel l)
IOVDbFolder(IOVDbConn *conn, const IOVDbParser &folderprop, MsgStream &msg, IClassIDSvc *clidsvc, IIOVDbMetaDataTool *metadatatool, const bool checklock, const bool outputToFile=false, const std::string &source="COOL_DATABASE", const std::string &crestServer="", const std::string &crestTag="", const bool crestCoolToFile=false)
AttrListVec::const_iterator const_iterator
cool::ValidityKey m_iovoverride
A CondAttrListVec is an Athena DataObject holding a vector of CORAL AttributeLists,...
bool addMetaAttrListColl(const CondAttrListCollection *coll)
std::unique_ptr< SG::TransientAddress > preLoadFolder(ITagInfoMgr *tagInfoMgr, const unsigned int cacheRun, const unsigned int cacheTime)
def iadd(self, tool)
associator for public tools -------------------------------------------—
std::optional< CoralCrestManager > m_crest_mng
std::optional< bool > checkTagLock(const cool::IFolderPtr &fptr, const std::string &tag)
Check whether a tag is locked on a folder, if possible.
std::vector< std::string > m_channames
std::string cache() const
This file contains the class definition for theAthenaAttrListAddress class.
bool loadCacheIfDbChanged(const cool::ValidityKey vkey, const std::string &globalTag, const cool::IDatabasePtr &dbPtr, const ServiceHandle< IIOVSvc > &iovSvc)
IOVDbNamespace::IovStore m_iovs
Iov_t getCacheBounds() const
Report the current cache bounds.
Class to provide easy MsgStream access and capabilities.
void setIOVOverride(const unsigned int run, const unsigned int lumiblock, const unsigned int time)
An AttributeList represents a logical row of attributes in a metadata table. The name and type of eac...
std::pair< std::vector< cool::ChannelId >, std::vector< std::string > > channelList(IOVDbConn *pConnection, const std::string &folderName, const bool named)
Retrieve channel information.
void setCacheBounds(const Iov_t &cacheBounds)
Set the cache bound variables.
std::vector< TFile * > fptr
void setFlag(TrigPassFlags *flags, const T *obj, const CONTAINER *container, const std::vector< bool > &flag)
Set the flag at index position.
std::string m_folderDescription
const std::string m_source
uint32_t CLID
The Class ID type.
MsgStream & msg() const
The standard message stream.
boost::shared_ptr< CoraCoolFolder > CoraCoolFolderPtr
bool addMetaAttrList(const coral::AttributeList &atrlist, const IOVRange &range)
void addIOVtoCache(cool::ValidityKey since, cool::ValidityKey until)
boost::shared_ptr< CoraCoolDatabase > CoraCoolDatabasePtr
IOVRange makeRange(const cool::ValidityKey since, const cool::ValidityKey until, const bool timeIsEpoch)
Make an IOVRange from two validity keys.
void addIov(const Iov_t &iov)
Add an iov to the store and update the span variables.
bool getKey(const std::string &key, const std::string &devvalue, std::string &value) const
original 'getKey' method, now implemented using 'at'
std::string to_string(const DetectorType &type)
std::string sanitiseFilename(const std::string &fname)
Replace the '/' of a file path with '^'.
std::vector< ChanRange > m_chanrange
std::vector< cool::ChannelId > m_channums
const std::string & key() const
bool typeSizeIsKnown(const coral::Attribute &attribute)
Return a bool indicating whether the size of a given Attribute can be determined.
bool timebaseIs_nsOfEpoch() const
unsigned int cacheUpdateImplementation(T &obj, const ServiceHandle< IIOVSvc > &iovSvc)
std::string resolveUsingTagInfo(const std::string &tag, const ITagInfoMgr *tagInfoMgr)
coral::AttributeListSpecification * m_cachespec
std::vector< std::string > symLinks() const
std::pair< unsigned long long, unsigned long long > runLumiFromIovTime(const unsigned long long iovTime)
Return a [run,lumi] pair from an IOV time.
This is a Athena service which manages detector description tag information. It maintains a TagInfo o...
boost::shared_ptr< CoraCoolObjectIter > CoraCoolObjectIterPtr
unsigned int attributeListSize(const coral::AttributeList &atrlist)
return the size (in bytes) of an AttributeList
bool resolveTag(const cool::IFolderPtr &fptr, const std::string &globalTag)
def time(flags, cells_name, *args, **kw)
std::pair< bool, std::string > folderMetadata(IOVDbConn *pConnection, const std::string &folderName)
Retrieve (bool) multiversion flag and folder description
void setCacheLength(const bool timeIs_nsOfEpoch, const unsigned int cacheRun, const unsigned int cacheTime)
#define ATH_MSG_WARNING(x)
const boost::regex ref(r_ef)
bool extendCacheHi()
Extend upper cache bound to the minimum 'until' time; return whether this changes its value.
boost::shared_ptr< CoraCoolObject > CoraCoolObjectPtr
bool overrideOptionsFromParsedDescription(const IOVDbParser &parsedDescription)
std::vector< cool::ChannelId > m_cachechan
Iov_t at(const unsigned int idx) const
'at' to read iov at specific index
std::string folderTypeName(const FolderType f)
Give a string name for the folder type.
CLID classId(MsgStream &msg) const
unsigned long long iovTimeFromRunLumi(const unsigned long long run, const unsigned long long lumi)
Create a long long representing the IOV from run, lumi.
FolderType determineFolderType(const std::string &folderDescription, const std::string &spec, const std::vector< cool::ChannelId > &chans)
Determine folder type with optional check using clid service to check clid matches typename.
bool extendCacheLo()
Extend lower cache bound to the maximum 'since' time; return whether this changes its value.
unsigned long long m_nbytesread
int countSelectedChannels(const std::vector< cool::ChannelId > &channels, const cool::ChannelSelection &selected)
Count the number of selected channels in a vector of channels according to cool::Channel selection.
bool looksLikeMagicTag(const std::string &candidateTag)
Looks like it should be magic.
void addSlice(const IOVRange &range, const unsigned int chan, const std::vector< coral::AttributeList > &data, const unsigned int datastart, const unsigned int dataend)
std::vector< unsigned int > m_cacheccstart
std::string addressHeader() const
bool add(ChanNum chanNum, const AttributeList &attributeList)
Adding in chan/attrList pairs.
std::vector< IOVHash > fetchCrestObjects(cool::ValidityKey since, cool::ValidityKey until, cool::ValidityKey vkey)
bool empty() const
is the store empty?
void setIovSpan(const Iov_t &span)
Set span.
CoraCoolDatabasePtr getCoraCoolDb()
This file contains the class definition for the CondAttrListVecAddress class.
bool objectIteratorIsValid(cool::IObjectIteratorPtr &objItr)
const IOVRange & minRange() const
std::vector< coral::AttributeList > m_cacheattr
std::string getPayloadSpec()
void addShared(ChanNum chanNum, const AttributeList &attributeList)
Adding in chan/attrList pairs with shared data.