Loading [MathJax]/extensions/tex2jax.js
ATLAS Offline Software
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
AutoConfigFlags.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
2 
3 from PyUtils.MetaReader import read_metadata, lite_primary_keys_to_keep, lite_TagInfo_keys_to_keep
4 from AthenaCommon.Logging import logging
5 from functools import lru_cache
6 
7 msg = logging.getLogger('AutoConfigFlags')
8 
9 # Module level cache of file-metadata:
10 _fileMetaData = dict()
11 
13  def __init__(self, filename, maxLevel='peeker'):
14  self.metadata = {}
15  self.filename = filename
16  self.currentAccessLevel = 'lite'
17  self.maxAccessLevel = maxLevel
18  thisFileMD = read_metadata(filename, None, 'lite')
19  self.metadata.update(thisFileMD[self.filename])
20  msg.debug("Loaded using 'lite' %s", str(self.metadata))
21 
22  def _loadMore(self, level):
23  self.currentAccessLevel = level
24 
25  thisFileMD = read_metadata(self.filename, None, level)
26  self.metadata.update(thisFileMD[self.filename])
27 
28  def get(self, key, default):
29  if key in self.metadata:
30  return self.metadata[key]
31  if key in lite_primary_keys_to_keep or key in lite_TagInfo_keys_to_keep:
32  # no need to load more
33  return default
34 
35  if self.currentAccessLevel == self.maxAccessLevel:
36  return default
37 
38  levels = []
39  if self.currentAccessLevel == 'lite':
40  levels = ['peeker', 'full'] if self.maxAccessLevel == 'full' else ['peeker']
41  elif self.currentAccessLevel == 'peeker':
42  levels = ['full']
43 
44  for level in levels:
45  msg.info("Looking into the file in '%s' mode as the configuration requires more details: %s ", level, key)
46  self._loadMore(level)
47  if key in self.metadata:
48  return self.metadata[key]
49 
50  return default
51 
52  def __contains__(self, key):
53  self.get(key, None)
54  return key in self.metadata
55 
56  def __getitem__(self, key):
57  return self.get(key, None)
58 
59  def __repr__(self):
60  return repr(self.metadata)
61 
62  def keys(self):
63  return self.metadata.keys()
64 
65 def GetFileMD(filenames, allowEmpty=True, maxLevel='peeker'):
66  if not filenames:
67  if allowEmpty:
68  msg.info("Running an input-less job. Will have empty metadata.")
69  return {}
70  raise RuntimeError("Metadata can not be read in an input-less job.")
71  if isinstance(filenames, str):
72  filenames = [filenames]
73  if '_ATHENA_GENERIC_INPUTFILE_NAME_' in filenames:
74  raise RuntimeError('Input file name not set, instead _ATHENA_GENERIC_INPUTFILE_NAME_ found. Cannot read metadata.')
75  for filename in filenames:
76  if filename not in _fileMetaData:
77  msg.info("Obtaining metadata of auto-configuration by peeking into '%s'", filename)
78  _fileMetaData[filename] = DynamicallyLoadMetadata(filename, maxLevel)
79  if _fileMetaData[filename].maxAccessLevel != maxLevel:
80  _fileMetaData[filename].maxAccessLevel = maxLevel
81  if _fileMetaData[filename]['nentries'] not in [None, 0]:
82  return _fileMetaData[filename]
83  else:
84  msg.info("The file: %s has no entries, going to the next one for harvesting the metadata", filename)
85  msg.info("No file with events found, returning anyways metadata associated to the first file %s", filenames[0])
86  return _fileMetaData[filenames[0]]
87 
88 def _initializeGeometryParameters(geoTag,sqliteDB,sqliteDBFullPath):
89  """Read geometry database for all detectors"""
90 
91  from AtlasGeoModel import CommonGeoDB
92  from PixelGeoModel import PixelGeoDB
93  from LArGeoAlgsNV import LArGeoDB
94  from MuonGeoModel import MuonGeoDB
95 
96  if not sqliteDB:
97  # Read parameters from Oracle/Frontier
98  from AtlasGeoModel.AtlasGeoDBInterface import AtlasGeoDBInterface
99  dbGeomCursor = AtlasGeoDBInterface(geoTag)
100  dbGeomCursor.ConnectAndBrowseGeoDB()
101 
102  params = { 'Common' : CommonGeoDB.InitializeGeometryParameters(dbGeomCursor),
103  'Pixel' : PixelGeoDB.InitializeGeometryParameters(dbGeomCursor),
104  'LAr' : LArGeoDB.InitializeGeometryParameters(dbGeomCursor),
105  'Muon' : MuonGeoDB.InitializeGeometryParameters(dbGeomCursor),
106  'Luminosity' : CommonGeoDB.InitializeLuminosityDetectorParameters(dbGeomCursor),
107  }
108 
109  msg.debug('Config parameters retrieved from Geometry DB (Frontier/Oracle):')
110  for key in params.keys():
111  msg.debug(f'{key} -> {params[key]}')
112  else:
113  # Read parameters from SQLite
114  from AtlasGeoModel.AtlasGeoDBInterface import AtlasGeoDBInterface_SQLite
115  sqliteReader = AtlasGeoDBInterface_SQLite(geoTag,sqliteDBFullPath)
116  sqliteReader.ConnectToDB()
117 
118  params = { 'Common' : CommonGeoDB.InitializeGeometryParameters_SQLite(sqliteReader),
119  'Pixel' : PixelGeoDB.InitializeGeometryParameters_SQLite(sqliteReader),
120  'LAr' : LArGeoDB.InitializeGeometryParameters_SQLite(sqliteReader),
121  'Muon' : MuonGeoDB.InitializeGeometryParameters_SQLite(sqliteReader),
122  'Luminosity' : CommonGeoDB.InitializeLuminosityDetectorParameters_SQLite(sqliteReader),
123  }
124 
125  msg.debug('Config parameters retrieved from Geometry DB (SQLite):')
126  for key in params.keys():
127  msg.debug(f'{key} -> {params[key]}')
128 
129  return params
130 
131 
132 @lru_cache(maxsize=4) # maxsize=1 should be enough for most jobs
133 def DetDescrInfo(geoTag, sqliteDB, sqliteDBFullPath):
134  """Query geometry DB for detector description. Returns dictionary with
135  detector description. Queries DB for each tag only once.
136 
137  geoTag: geometry tag (e.g. ATLAS-R2-2016-01-00-01)
138  """
139  if not geoTag:
140  raise ValueError("No geometry tag specified")
141 
142  detDescrInfo = _initializeGeometryParameters(geoTag,sqliteDB,sqliteDBFullPath)
143  detDescrInfo["geomTag"] = geoTag
144  return detDescrInfo
145 
146 
147 @lru_cache(maxsize=4) # maxsize=1 should be enough for most jobs
148 def getDefaultDetectors(geoTag, sqliteDB, sqliteDBFullPath, includeForward=False):
149  """Query geometry DB for detector description.
150  Returns a set of detectors used in a geometry tag.
151 
152  geoTag: geometry tag (e.g. ATLAS-R2-2016-01-00-01)
153  """
154  detectors = DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Common']['Detectors']
155 
156  manualConfig = not DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Common']["DetectorsConfigured"]
157 
158 
159  if manualConfig:
160  detectors.add('Bpipe')
161  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Common']['Run'] not in ['RUN1', 'RUN2', 'RUN3']: # RUN4 and beyond
162  detectors.add('ITkPixel')
163  detectors.add('ITkStrip')
164  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Luminosity']['BCMPrime']:
165  detectors.add('BCMPrime') # since https://gitlab.cern.ch/Atlas-Inner-Tracking/ITKLayouts/-/merge_requests/347 introduced BCMPrime geometry
166  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Luminosity']['PLR']:
167  detectors.add('PLR')
168  else:
169  detectors.add('Pixel')
170  detectors.add('SCT')
171  detectors.add('TRT')
172  detectors.add('BCM')
173 
174  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Common']['Run'] not in ['RUN1', 'RUN2', 'RUN3']: # RUN4 and beyond
175  detectors.add('HGTD')
176 
177  detectors.add('LAr')
178  detectors.add('Tile')
179 
180 
181  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Common']['Run'] in ['RUN1', 'RUN2', 'RUN3']:
182  detectors.add('MBTS')
183 
184  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Muon']['HasMDT']:
185  detectors.add('MDT')
186  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Muon']['HasRPC']:
187  detectors.add('RPC')
188  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Muon']['HasTGC']:
189  detectors.add('TGC')
190  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Muon']['HasCSC']:
191  detectors.add('CSC')
192  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Muon']['HasSTGC']:
193  detectors.add('sTGC')
194  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Muon']['HasMM']:
195  detectors.add('MM')
196 
197  if includeForward:
198  detectors.add('Lucid')
199  if DetDescrInfo(geoTag,sqliteDB,sqliteDBFullPath)['Common']['Run'] not in ['RUN1']:
200  detectors.add('AFP')
201  detectors.add('ZDC')
202  detectors.add('ALFA')
203  detectors.add('FwdRegion')
204 
205  return detectors
206 
207 
208 # Based on RunDMCFlags.py
210  # this wrapper is intended to avoid an initial import
211  from .RunToTimestampData import RunToTimestampDict
212  return RunToTimestampDict
213 
214 
216  """This is used to hold a dictionary of the form
217  {152166:1269948352889940910, ...} to allow the
218  timestamp to be determined from the run.
219  """
220  run2timestampDict = getRunToTimestampDict()
221  timeStamps = [run2timestampDict.get(runNumber,1) for runNumber in runNumbers] # Add protection here?
222  return timeStamps
223 
224 
225 def getGeneratorsInfo(flags):
226  """Read in GeneratorsInfo from the input file
227  """
228  from AthenaConfiguration.Enums import ProductionStep
229  inputFiles = flags.Input.Files
230  if flags.Common.ProductionStep in [ProductionStep.Overlay, ProductionStep.FastChain] and flags.Input.SecondaryFiles and not flags.Overlay.ByteStream:
231  # Do something special for MC Overlay
232  inputFiles = flags.Input.SecondaryFiles
233  generatorsString = ""
234  from AthenaConfiguration.AutoConfigFlags import GetFileMD
235  if inputFiles:
236  generatorsString = GetFileMD(inputFiles).get("generators", "")
237  from GeneratorConfig.Versioning import generatorsGetFromMetadata
238  return generatorsGetFromMetadata( generatorsString )
239 
240 
242  """Read in special simulation job option fragments based on metadata
243  passed by the evgen stage
244  """
245  specialConfigDict = dict()
246  legacyPreIncludeToCAPostInclude = { 'SimulationJobOptions/preInclude.AMSB.py' : 'Charginos.CharginosConfig.AMSB_Cfg',
247  'SimulationJobOptions/preInclude.Monopole.py' : 'Monopole.MonopoleConfig.MonopoleCfg',
248  'SimulationJobOptions/preInclude.Quirks.py' : 'Quirks.QuirksConfig.QuirksCfg',
249  'SimulationJobOptions/preInclude.SleptonsLLP.py' : 'Sleptons.SleptonsConfig.SleptonsLLPCfg',
250  'SimulationJobOptions/preInclude.GMSB.py' : 'Sleptons.SleptonsConfig.GMSB_Cfg',
251  'SimulationJobOptions/preInclude.Qball.py' : 'Monopole.MonopoleConfig.QballCfg',
252  'SimulationJobOptions/preInclude.RHadronsPythia8.py' : 'RHadrons.RHadronsConfig.RHadronsCfg',
253  'SimulationJobOptions/preInclude.fcp.py' : 'Monopole.MonopoleConfig.fcpCfg' }
254  legacyPreIncludeToCAPreInclude = { 'SimulationJobOptions/preInclude.AMSB.py' : None,
255  'SimulationJobOptions/preInclude.Monopole.py' : 'Monopole.MonopoleConfig.MonopolePreInclude',
256  'SimulationJobOptions/preInclude.Quirks.py' : None,
257  'SimulationJobOptions/preInclude.SleptonsLLP.py' : None,
258  'SimulationJobOptions/preInclude.GMSB.py' : None,
259  'SimulationJobOptions/preInclude.Qball.py' : 'Monopole.MonopoleConfig.QballPreInclude',
260  'SimulationJobOptions/preInclude.RHadronsPythia8.py' : 'RHadrons.RHadronsConfig.RHadronsPreInclude',
261  'SimulationJobOptions/preInclude.fcp.py' : 'Monopole.MonopoleConfig.fcpPreInclude' }
262  specialConfigString = ''
263  from AthenaConfiguration.Enums import ProductionStep
264  inputFiles = flags.Input.Files
265  secondaryInputFiles = flags.Input.SecondaryFiles
266  if flags.Common.ProductionStep in [ProductionStep.Overlay, ProductionStep.FastChain] and not flags.Overlay.DataOverlay and flags.Input.SecondaryFiles:
267  # Do something special for MC Overlay
268  inputFiles = flags.Input.SecondaryFiles
269  secondaryInputFiles = flags.Input.Files
270  from AthenaConfiguration.AutoConfigFlags import GetFileMD
271  if len(inputFiles)>0:
272  specialConfigString = GetFileMD(inputFiles).get('specialConfiguration', '')
273  if (not len(specialConfigString) or specialConfigString == 'NONE') and len(secondaryInputFiles)>0:
274  # If there is no specialConfiguration metadata in the primary
275  # input try the secondary inputs (MC Overlay case)
276  specialConfigString = GetFileMD(secondaryInputFiles).get('specialConfiguration', '')
277  if len(specialConfigString)>0:
278 
280  spcitems = specialConfigString.split(";")
281  for spcitem in spcitems:
282  #print spcitem
283 
284  if not spcitem or spcitem.upper() == "NONE":
285  continue
286 
287  if "=" not in spcitem:
288  spcitem = "preInclude=" + spcitem
289 
290  k, v = spcitem.split("=")
291  if k == "preInclude" and v.endswith('.py'): # Translate old preIncludes into CA-based versions.
292  if v == 'SimulationJobOptions/preInclude.RhadronsPythia8.py':
293  v = 'SimulationJobOptions/preInclude.RHadronsPythia8.py' # ATLASSIM-6687 Fixup for older EVNT files
294  v1 = legacyPreIncludeToCAPreInclude[v]
295  if v1 is not None:
296  specialConfigDict[k] = v1
297  v2 = legacyPreIncludeToCAPostInclude[v]
298  if v2 is not None:
299  specialConfigDict['postInclude'] = v2
300  else:
301  specialConfigDict[k] = v
302  return specialConfigDict
303 
python.AutoConfigFlags.GetFileMD
def GetFileMD(filenames, allowEmpty=True, maxLevel='peeker')
Definition: AutoConfigFlags.py:65
python.AutoConfigFlags.getGeneratorsInfo
def getGeneratorsInfo(flags)
Definition: AutoConfigFlags.py:225
PixelGeoDB.InitializeGeometryParameters_SQLite
def InitializeGeometryParameters_SQLite(sqliteDbReader)
Definition: PixelGeoDB.py:54
python.AutoConfigFlags.getDefaultDetectors
def getDefaultDetectors(geoTag, sqliteDB, sqliteDBFullPath, includeForward=False)
Definition: AutoConfigFlags.py:148
python.MetaReader.read_metadata
def read_metadata(filenames, file_type=None, mode='lite', promote=None, meta_key_filter=None, unique_tag_info_values=True, ignoreNonExistingLocalFiles=False)
Definition: MetaReader.py:53
python.AutoConfigFlags._initializeGeometryParameters
def _initializeGeometryParameters(geoTag, sqliteDB, sqliteDBFullPath)
Definition: AutoConfigFlags.py:88
MuonGeoDB.InitializeGeometryParameters
def InitializeGeometryParameters(dbGeomCursor)
Definition: MuonGeoDB.py:3
python.AutoConfigFlags.DynamicallyLoadMetadata.maxAccessLevel
maxAccessLevel
Definition: AutoConfigFlags.py:17
python.AutoConfigFlags.DynamicallyLoadMetadata.currentAccessLevel
currentAccessLevel
Definition: AutoConfigFlags.py:16
python.AutoConfigFlags.DynamicallyLoadMetadata.get
def get(self, key, default)
Definition: AutoConfigFlags.py:28
LArGeoDB.InitializeGeometryParameters
def InitializeGeometryParameters(dbGeomCursor)
Definition: LArGeoDB.py:3
python.AutoConfigFlags.DynamicallyLoadMetadata._loadMore
def _loadMore(self, level)
Definition: AutoConfigFlags.py:22
python.AutoConfigFlags.DynamicallyLoadMetadata.__getitem__
def __getitem__(self, key)
Definition: AutoConfigFlags.py:56
python.AutoConfigFlags.DynamicallyLoadMetadata
Definition: AutoConfigFlags.py:12
python.AutoConfigFlags.DynamicallyLoadMetadata.filename
filename
Definition: AutoConfigFlags.py:15
PyAthena::repr
std::string repr(PyObject *o)
returns the string representation of a python object equivalent of calling repr(o) in python
Definition: PyAthenaUtils.cxx:106
python.AutoConfigFlags.DynamicallyLoadMetadata.metadata
metadata
Definition: AutoConfigFlags.py:14
python.AutoConfigFlags.getRunToTimestampDict
def getRunToTimestampDict()
Definition: AutoConfigFlags.py:209
python.AutoConfigFlags.DynamicallyLoadMetadata.__repr__
def __repr__(self)
Definition: AutoConfigFlags.py:59
python.AutoConfigFlags.DynamicallyLoadMetadata.__init__
def __init__(self, filename, maxLevel='peeker')
Definition: AutoConfigFlags.py:13
LArGeoDB.InitializeGeometryParameters_SQLite
def InitializeGeometryParameters_SQLite(sqliteDbReader)
Definition: LArGeoDB.py:34
python.AutoConfigFlags.getInitialTimeStampsFromRunNumbers
def getInitialTimeStampsFromRunNumbers(runNumbers)
Definition: AutoConfigFlags.py:215
python.AutoConfigFlags.DynamicallyLoadMetadata.__contains__
def __contains__(self, key)
Definition: AutoConfigFlags.py:52
get
T * get(TKey *tobj)
get a TObject* from a TKey* (why can't a TObject be a TKey?)
Definition: hcg.cxx:127
python.AutoConfigFlags.DynamicallyLoadMetadata.keys
def keys(self)
Definition: AutoConfigFlags.py:62
PixelGeoDB.InitializeGeometryParameters
def InitializeGeometryParameters(dbGeomCursor)
Definition: PixelGeoDB.py:3
python.Versioning.generatorsGetFromMetadata
def generatorsGetFromMetadata(metadataString)
Definition: Versioning.py:47
python.AutoConfigFlags.DetDescrInfo
def DetDescrInfo(geoTag, sqliteDB, sqliteDBFullPath)
Definition: AutoConfigFlags.py:133
str
Definition: BTagTrackIpAccessor.cxx:11
python.AutoConfigFlags.getSpecialConfigurationMetadata
def getSpecialConfigurationMetadata(flags)
Definition: AutoConfigFlags.py:241
MuonGeoDB.InitializeGeometryParameters_SQLite
def InitializeGeometryParameters_SQLite(sqliteDbReader)
Definition: MuonGeoDB.py:31