ATLAS Offline Software
IOVDbSvcConfig.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
2 
3 from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator, ConfigurationError
4 import os
5 from AthenaConfiguration.ComponentFactory import CompFactory
6 from AthenaConfiguration.AccumulatorCache import AccumulatorCache
7 from AthenaCommon.Logging import logging
8 from functools import cache
9 
10 msg = logging.getLogger('IOVDbSvcCfg')
11 
12 def CondInputLoaderCfg(flags, **kwargs):
13  result = ComponentAccumulator()
14  result.addCondAlgo(CompFactory.CondInputLoader(**kwargs))
15  return result
16 
17 
18 def DBReplicaSvcCfg(flags, vetoDBRelease=False, **kwargs):
19  if vetoDBRelease:
20  kwargs.setdefault('COOLSQLiteVetoPattern', '/DBRelease/')
21 
22  result = ComponentAccumulator()
23  result.addService(CompFactory.DBReplicaSvc(**kwargs))
24  return result
25 
26 
27 @AccumulatorCache
28 def IOVDbSvcCfg(flags, **kwargs):
29  # Add the conditions loader, must be the first in the sequence
30  result = CondInputLoaderCfg(flags)
31 
32  kwargs.setdefault('OnlineMode', flags.Common.isOnline)
33  kwargs.setdefault('dbConnection', flags.IOVDb.DBConnection)
34  kwargs.setdefault('crestServer', flags.IOVDb.CrestServer)
35  # setup knowledge of dbinstance in IOVDbSvc, for global tag x-check
36  kwargs.setdefault('DBInstance', flags.IOVDb.DatabaseInstance)
37 
38  if 'FRONTIER_SERVER' in os.environ.keys() and os.environ['FRONTIER_SERVER'] != '':
39  kwargs.setdefault('CacheAlign', 3)
40 
41  # Very important cache settings for use of CoralProxy at P1 (ATR-4646)
42  if flags.Common.isOnline and flags.Trigger.Online.isPartition:
43  kwargs['CacheAlign'] = 0
44  kwargs['CacheRun'] = 0
45  kwargs['CacheTime'] = 0
46 
47  kwargs.setdefault('GlobalTag', flags.IOVDb.GlobalTag)
48  if 'Folders' in kwargs:
49  kwargs['Folders'] = ['/TagInfo<metaOnly/>'] + kwargs['Folders']
50  else:
51  kwargs.setdefault('Folders', ['/TagInfo<metaOnly/>'])
52 
53  # Select CREST backend if needed
54  if flags.IOVDb.GlobalTag and flags.IOVDb.GlobalTag.startswith('CREST-'):
55  kwargs.setdefault('Source', 'CREST')
56 
57  result.addService(CompFactory.IOVDbSvc(**kwargs), primary=True)
58 
59  # Set up POOLSvc with appropriate catalogs
60  from AthenaPoolCnvSvc.PoolCommonConfig import PoolSvcCfg, AthenaPoolCnvSvcCfg
61  result.merge(PoolSvcCfg(flags, withCatalogs=True))
62  result.merge(AthenaPoolCnvSvcCfg(flags))
63  result.addService(CompFactory.CondSvc())
64  result.addService(CompFactory.ProxyProviderSvc(ProviderNames=['IOVDbSvc']))
65 
66  if not flags.Input.isMC:
67  result.merge(DBReplicaSvcCfg(flags, vetoDBRelease=True))
68 
69  # Get TagInfoMgr
70  from EventInfoMgt.TagInfoMgrConfig import TagInfoMgrCfg
71  result.merge(TagInfoMgrCfg(flags))
72 
73  # Set up MetaDataSvc
74  from AthenaServices.MetaDataSvcConfig import MetaDataSvcCfg
75  result.merge(MetaDataSvcCfg(flags, ['IOVDbMetaDataTool']))
76 
77  return result
78 
79 
80 # Convenience method to add folders:
81 def addFolders(flags, folderStrings, detDb=None, className=None, extensible=False, tag=None, db=None, modifiers=''):
82  tagString = ''
83  if tag is not None:
84  tagString = '<tag>%s</tag>' % tag
85 
86  # Convenience hack: Allow a single string as parameter:
87  if isinstance(folderStrings, str):
88  return addFolderList(flags, ((folderStrings + tagString, detDb, className),), extensible, db, modifiers)
89 
90  else: # Got a list of folders
91  folderDefinitions = []
92 
93  for folderString in folderStrings:
94  folderDefinitions.append((folderString + tagString, detDb, className))
95 
96  return addFolderList(flags, folderDefinitions, extensible, db, modifiers)
97 
98 
99 def addFolderList(flags, listOfFolderInfoTuple, extensible=False, db=None, modifiers=''):
100  """Add access to the given set of folders, in the identified subdetector schema.
101  FolerInfoTuple consists of (foldername,detDB,classname)
102 
103  If EXTENSIBLE is set, then if we access an open-ended IOV at the end of the list,
104  the end time for this range will be set to just past the current event.
105  Subsequent accesses will update this end time for subsequent events.
106  This allows the possibility of later adding a new IOV using IOVSvc::setRange."""
107  loadFolders = set()
108  folders = []
109  sqliteFolders=getSqliteContent(flags.IOVDb.SqliteInput,
110  flags.IOVDb.SqliteFolders,
111  flags.IOVDb.DatabaseInstance)
112 
113  for (fs, detDb, className) in listOfFolderInfoTuple:
114  fse= _extractFolder(fs)
115  # Add class-name to CondInputLoader (if reqired)
116  if className is not None:
117  loadFolders.add((className, fse))
118 
119  if fse in sqliteFolders:
120  msg.warning(f'Reading folder {fs} from sqlite, bypassing production database')
121  fs+=sqliteFolders[fse]
122  elif detDb is not None and fs.find('<db>') == -1:
123 
124  if db: # override database name if provided
125  dbName=db
126  else:
127  dbName = flags.IOVDb.DatabaseInstance
128  if detDb in _dblist.keys():
129  fs = f'<db>{_dblist[detDb]}/{dbName}</db> {fs}'
130  elif os.access(detDb, os.R_OK):
131  # Assume slqite file
132  fs = f'<db>sqlite://;schema={detDb};dbname={dbName}</db> {fs}'
133  else:
134  raise ConfigurationError(f'Error, db shorthand {detDb} not known, nor found as sqlite file')
135  # Append database string to folder-name
136 
137  if extensible:
138  fs = fs + '<extensible/>'
139 
140  # Add explicitly given xml-modifiers (like channel-selection)
141  fs += modifiers
142 
143  # Append (modified) folder-name string to IOVDbSvc Folders property
144  folders.append(fs)
145 
146 
147  result = IOVDbSvcCfg(flags)
148  result.getPrimary().Folders+=folders
149  if loadFolders:
150  result.getCondAlgo('CondInputLoader').Load |= loadFolders
151 
152  if flags.IOVDb.CleanerRingSize > 0:
153  #HLT-jobs set IOVDb.CleanerRingSize to 0 to run without the cleaning-service,
154  cleanerSvc = CompFactory.Athena.DelayedConditionsCleanerSvc(RingSize=flags.IOVDb.CleanerRingSize)
155  result.addService(cleanerSvc)
156  result.addService(CompFactory.Athena.ConditionsCleanerSvc(CleanerSvc=cleanerSvc))
157 
158 
159  return result
160 
161 
162 def addFoldersSplitOnline(flags, detDb, onlineFolders, offlineFolders, className=None, extensible=False, addMCString='_OFL', splitMC=False, tag=None, forceDb=None, modifiers=''):
163  """Add access to given folder, using either online_folder or offline_folder. For MC, add addMCString as a postfix (default is _OFL)"""
164 
165  if flags.Common.isOnline and not flags.Input.isMC:
166  folders = onlineFolders
167  elif splitMC and not flags.Input.isMC:
168  folders = onlineFolders
169  else:
170  # MC, so add addMCString
171  detDb = detDb + addMCString
172  folders = offlineFolders
173 
174  return addFolders(flags, folders, detDb, className, extensible, tag=tag, db=forceDb, modifiers=modifiers)
175 
176 
177 _dblist = {
178  'INDET':'COOLONL_INDET',
179  'INDET_ONL':'COOLONL_INDET',
180  'PIXEL':'COOLONL_PIXEL',
181  'PIXEL_ONL':'COOLONL_PIXEL',
182  'SCT':'COOLONL_SCT',
183  'SCT_ONL':'COOLONL_SCT',
184  'TRT':'COOLONL_TRT',
185  'TRT_ONL':'COOLONL_TRT',
186  'LAR':'COOLONL_LAR',
187  'LAR_ONL':'COOLONL_LAR',
188  'TILE':'COOLONL_TILE',
189  'TILE_ONL':'COOLONL_TILE',
190  'MUON':'COOLONL_MUON',
191  'MUON_ONL':'COOLONL_MUON',
192  'MUONALIGN':'COOLONL_MUONALIGN',
193  'MUONALIGN_ONL':'COOLONL_MUONALIGN',
194  'MDT':'COOLONL_MDT',
195  'MDT_ONL':'COOLONL_MDT',
196  'RPC':'COOLONL_RPC',
197  'RPC_ONL':'COOLONL_RPC',
198  'TGC':'COOLONL_TGC',
199  'TGC_ONL':'COOLONL_TGC',
200  'CSC':'COOLONL_CSC',
201  'CSC_ONL':'COOLONL_CSC',
202  'TDAQ':'COOLONL_TDAQ',
203  'TDAQ_ONL':'COOLONL_TDAQ',
204  'GLOBAL':'COOLONL_GLOBAL',
205  'GLOBAL_ONL':'COOLONL_GLOBAL',
206  'TRIGGER':'COOLONL_TRIGGER',
207  'TRIGGER_ONL':'COOLONL_TRIGGER',
208  'CALO':'COOLONL_CALO',
209  'CALO_ONL':'COOLONL_CALO',
210  'FWD':'COOLONL_FWD',
211  'FWD_ONL':'COOLONL_FWD',
212  'INDET_OFL':'COOLOFL_INDET',
213  'PIXEL_OFL':'COOLOFL_PIXEL',
214  'SCT_OFL':'COOLOFL_SCT',
215  'TRT_OFL':'COOLOFL_TRT',
216  'LAR_OFL':'COOLOFL_LAR',
217  'TILE_OFL':'COOLOFL_TILE',
218  'MUON_OFL':'COOLOFL_MUON',
219  'MUONALIGN_OFL':'COOLOFL_MUONALIGN',
220  'MDT_OFL':'COOLOFL_MDT',
221  'RPC_OFL':'COOLOFL_RPC',
222  'TGC_OFL':'COOLOFL_TGC',
223  'CSC_OFL':'COOLOFL_CSC',
224  'TDAQ_OFL':'COOLOFL_TDAQ',
225  'DCS_OFL':'COOLOFL_DCS',
226  'GLOBAL_OFL':'COOLOFL_GLOBAL',
227  'TRIGGER_OFL':'COOLOFL_TRIGGER',
228  'CALO_OFL':'COOLOFL_CALO',
229  'FWD_OFL':'COOLOFL_FWD'
230 }
231 
232 
233 def addOverride(flags, folder, tag, db=None):
234  """Add a tag override for the specified folder"""
235  suffix = ''
236  if db:
237  suffix = f' <db>{db}</db>'
238  return IOVDbSvcCfg(flags, overrideTags=(f'<prefix>{folder}</prefix> <tag>{tag}</tag>{suffix}',))
239 
240 
241 def _extractFolder(folderString):
242  """Extract the folder name (non-XML text) from a IOVDbSvc.Folders entry"""
243  folderName = ''
244  xmlTag = ''
245  ix = 0
246  while ix < len(folderString):
247  if (folderString[ix] == '<' and xmlTag == ''):
248  ix2 = folderString.find('>', ix)
249  if ix2 != -1:
250  xmlTag = folderString[ix + 1 : ix2].strip()
251  ix = ix2 + 1
252  elif folderString[ix:ix+2] == '</' and xmlTag != '':
253  ix2 = folderString.find('>', ix)
254  if ix2 != -1:
255  xmlTag = ''
256  ix = ix2 + 1
257  else:
258  ix2 = folderString.find('<', ix)
259  if ix2 == -1:
260  ix2 = len(folderString)
261  if xmlTag == '':
262  folderName = folderName + folderString[ix : ix2]
263  ix = ix2
264  return folderName.strip()
265 
266 
267 @cache #Fill only once
268 def getSqliteContent(sqliteInput,takeFolders,databaseInstance):
269  if sqliteInput == "": return []
270  sqliteFolders=dict()
271  if isinstance(takeFolders, str):
272  takeFolders=[takeFolders,]
273  dbStr="sqlite://;schema="+ sqliteInput+";dbname="+databaseInstance
274  from PyCool import cool
275  dbSvc = cool.DatabaseSvcFactory.databaseService()
276  db = dbSvc.openDatabase(dbStr)
277  nodelist=db.listAllNodes()
278  for node in nodelist:
279  if db.existsFolder(node):
280  if (len(takeFolders)>0 and str(node) not in takeFolders): continue
281  connStr="<db>"+dbStr+"</db>"
282  f=db.getFolder(node)
283  if f.versioningMode is not cool.FolderVersioning.SINGLE_VERSION:
284  tags=f.listTags()
285  if len(tags)==1:
286  connStr+="<tag>"+tags[0]+"</tag>"
287  sqliteFolders[str(node)]=connStr
288  db.closeDatabase()
289 
290  if len(takeFolders)>0:
291  missedFolders=set(takeFolders)-set(sqliteFolders.keys())
292  if len(missedFolders):
293  msg.error("The following folders were requested via the flag IOVSvc.sqliteFolder but not found in the sqlite file %s",(sqliteInput))
294  for f in missedFolders:
295  msg.error(f)
296 
297  msg.info("The following folders/tags are read from sqlite:")
298  for v in sqliteFolders.items():
299  msg.info("\t"+str(v))
300  return sqliteFolders
301 
302 
303 if __name__ == '__main__':
304  from AthenaConfiguration.AllConfigFlags import initConfigFlags
305  from AthenaConfiguration.TestDefaults import defaultTestFiles
306  flags = initConfigFlags()
307  flags.Input.Files = defaultTestFiles.RAW_RUN2
308  flags.lock()
309 
310  acc = IOVDbSvcCfg(flags)
311 
312  with open('test.pkl','wb') as f:
313  acc.store(f)
python.IOVDbSvcConfig.CondInputLoaderCfg
def CondInputLoaderCfg(flags, **kwargs)
Definition: IOVDbSvcConfig.py:12
python.JetAnalysisCommon.ComponentAccumulator
ComponentAccumulator
Definition: JetAnalysisCommon.py:302
python.IOVDbSvcConfig.addFolderList
def addFolderList(flags, listOfFolderInfoTuple, extensible=False, db=None, modifiers='')
Definition: IOVDbSvcConfig.py:99
python.TagInfoMgrConfig.TagInfoMgrCfg
def TagInfoMgrCfg(flags, tagValuePairs={})
Definition: TagInfoMgrConfig.py:6
python.IOVDbSvcConfig.addOverride
def addOverride(flags, folder, tag, db=None)
Definition: IOVDbSvcConfig.py:233
python.IOVDbSvcConfig.addFolders
def addFolders(flags, folderStrings, detDb=None, className=None, extensible=False, tag=None, db=None, modifiers='')
Definition: IOVDbSvcConfig.py:81
python.IOVDbSvcConfig.IOVDbSvcCfg
def IOVDbSvcCfg(flags, **kwargs)
Definition: IOVDbSvcConfig.py:28
CxxUtils::set
constexpr std::enable_if_t< is_bitmask_v< E >, E & > set(E &lhs, E rhs)
Convenience function to set bits in a class enum bitmask.
Definition: bitmask.h:232
python.PoolCommonConfig.PoolSvcCfg
def PoolSvcCfg(flags, withCatalogs=False, **kwargs)
Definition: PoolCommonConfig.py:7
python.PoolCommonConfig.AthenaPoolCnvSvcCfg
def AthenaPoolCnvSvcCfg(flags, **kwargs)
Definition: PoolCommonConfig.py:30
python.IOVDbSvcConfig.addFoldersSplitOnline
def addFoldersSplitOnline(flags, detDb, onlineFolders, offlineFolders, className=None, extensible=False, addMCString='_OFL', splitMC=False, tag=None, forceDb=None, modifiers='')
Definition: IOVDbSvcConfig.py:162
python.IOVDbSvcConfig._extractFolder
def _extractFolder(folderString)
Definition: IOVDbSvcConfig.py:241
python.IOVDbSvcConfig.DBReplicaSvcCfg
def DBReplicaSvcCfg(flags, vetoDBRelease=False, **kwargs)
Definition: IOVDbSvcConfig.py:18
Trk::open
@ open
Definition: BinningType.h:40
python.MetaDataSvcConfig.MetaDataSvcCfg
def MetaDataSvcCfg(flags, toolNames=[], tools=[])
Definition: MetaDataSvcConfig.py:6
python.AllConfigFlags.initConfigFlags
def initConfigFlags()
Definition: AllConfigFlags.py:19
str
Definition: BTagTrackIpAccessor.cxx:11
python.IOVDbSvcConfig.getSqliteContent
def getSqliteContent(sqliteInput, takeFolders, databaseInstance)
Definition: IOVDbSvcConfig.py:268