ATLAS Offline Software
RAWtoALL_Skeleton.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
2 
3 from PyJobTransforms.CommonRunArgsToFlags import commonRunArgsToFlags
4 from PyJobTransforms.TransformUtils import processPreExec, processPreInclude, processPostExec, processPostInclude
5 
6 from AthenaCommon import SystemOfUnits as Units
7 
8 # force no legacy job properties
9 from AthenaCommon import JobProperties
10 JobProperties.jobPropertiesDisallowed = True
11 
12 
13 
14 def fromRunArgs(runArgs):
15  from AthenaCommon.Logging import logging
16  log = logging.getLogger('RAWtoALL')
17  log.info('****************** STARTING Reconstruction (RAWtoALL) *****************')
18 
19  log.info('**** Transformation run arguments')
20  log.info(str(runArgs))
21 
22  import time
23  timeStart = time.time()
24 
25  from PyUtils.Helpers import ROOTSetup
26  ROOTSetup(batch=True)
27 
28  log.info('**** Setting-up configuration flags')
29  from AthenaConfiguration.AllConfigFlags import initConfigFlags
30  flags = initConfigFlags()
31  # set per-event timeout (in ns)
32  flags.Exec.EventTimeOut = 3600*Units.second
33 
34  commonRunArgsToFlags(runArgs, flags)
35 
36  # Autoconfigure enabled subdetectors
37  if hasattr(runArgs, 'detectors'):
38  detectors = runArgs.detectors
39  else:
40  detectors = None
41 
42  # TODO: event service?
43 
44 
46  inputsDRAW = [prop for prop in dir(runArgs) if prop.startswith('inputDRAW') and prop.endswith('File')]
47  if hasattr(runArgs, 'inputBSFile'):
48  if inputsDRAW:
49  raise RuntimeError('Impossible to run RAWtoALL with input BS and DRAW files (one input type only!)')
50  flags.Input.Files = runArgs.inputBSFile
51  if len(inputsDRAW) == 1:
52  flags.Input.Files = getattr(runArgs, inputsDRAW[0])
53  elif len(inputsDRAW) > 1:
54  raise RuntimeError('Impossible to run RAWtoALL with multiple input DRAW files (viz.: {0})'.format(inputsDRAW))
55 
56  # RDO
57  if hasattr(runArgs, 'inputRDOFile'):
58  flags.Input.Files = runArgs.inputRDOFile
59  if hasattr(runArgs, 'inputRDO_TRIGFile'):
60  flags.Input.Files = runArgs.inputRDO_TRIGFile
61  # TODO: trigger setup
62 
63  # TODO: EVNT?
64 
65 
66  if hasattr(runArgs, 'outputESDFile'):
67  flags.Output.ESDFileName = runArgs.outputESDFile
68  log.info("---------- Configured ESD output")
69 
70  if hasattr(runArgs, 'outputAODFile'):
71  flags.Output.AODFileName = runArgs.outputAODFile
72  log.info("---------- Configured AOD output")
73 
74  if hasattr(runArgs, 'outputHISTFile'):
75  flags.Output.HISTFileName = runArgs.outputHISTFile
76  flags.DQ.doMonitoring = True
77  log.info("---------- Configured HIST output")
78 
79  if hasattr(runArgs, 'outputHIST_R2AFile'):
80  flags.Output.HISTFileName = runArgs.outputHIST_R2AFile
81  flags.DQ.doMonitoring = True
82  log.info("---------- Configured HIST_R2A output")
83 
84  if hasattr(runArgs, 'outputDAOD_IDTIDEFile'):
85  flagString = 'Output.DAOD_IDTIDEFileName'
86  flags.addFlag(flagString, runArgs.outputDAOD_IDTIDEFile)
87  flags.Output.doWriteDAOD = True
88  flags.addFlag('Output.doWriteDAOD_IDTIDE', True)
89  log.info("---------- Configured DAOD_IDTIDE output")
90 
91  if hasattr(runArgs, 'outputDAOD_PIXELVALIDFile'):
92  flagString = 'Output.DAOD_PIXELVALIDFileName'
93  flags.addFlag(flagString, runArgs.outputDAOD_PIXELVALIDFile)
94  flags.Output.doWriteDAOD = True
95  flags.addFlag('Output.doWriteDAOD_PIXELVALID', True)
96  log.info("---------- Configured DAOD_PIXELVALID output")
97 
98  if hasattr(runArgs, 'outputDAOD_IDTRKVALIDFile'):
99  flagString = 'Output.DAOD_IDTRKVALIDFileName'
100  flags.addFlag(flagString, runArgs.outputDAOD_IDTRKVALIDFile)
101  flags.Output.doWriteDAOD = True
102  flags.addFlag('Output.doWriteDAOD_IDTRKVALID', True)
103  log.info("---------- Configured DAOD_IDTRKVALID output")
104 
105  if hasattr(runArgs, 'outputDAOD_SCTVALIDFile'):
106  flagString = 'Output.DAOD_SCTVALIDFileName'
107  flags.addFlag(flagString, runArgs.outputDAOD_SCTVALIDFile)
108  flags.Output.doWriteDAOD = True
109  flags.addFlag('Output.doWriteDAOD_SCTVALID', True)
110  log.info("---------- Configured DAOD_SCTVALID output")
111 
112  if hasattr(runArgs, 'outputDESDM_MCPFile'):
113  flagString = 'Output.DESDM_MCPFileName'
114  flags.addFlag(flagString, runArgs.outputDESDM_MCPFile)
115  flags.Output.doWriteDAOD = True
116  flags.addFlag('Output.doWriteDESDM_MCP', True)
117  log.info("---------- Configured DESDM_MCP output")
118 
119  if hasattr(runArgs, 'outputDESDM_EXOTHIPFile'):
120  flagString = 'Output.DESDM_EXOTHIPFileName'
121  flags.addFlag(flagString, runArgs.outputDESDM_EXOTHIPFile)
122  flags.Output.doWriteDAOD = True
123  flags.addFlag('Output.doWriteDESDM_EXOTHIP', True)
124  log.info("---------- Configured DESDM_EXOTHIP output")
125 
126  if hasattr(runArgs, 'outputDRAW_ZMUMUFile'):
127  flagString = 'Output.DRAW_ZmumuFileName'
128  flags.addFlag(flagString, runArgs.outputDRAW_ZMUMUFile)
129  flags.Output.doWriteBS = True
130  log.info("---------- Configured DRAW ZMUMU output")
131 
132  if hasattr(runArgs, 'outputDRAW_EGZFile'):
133  flagString = 'Output.DRAW_EGZFileName'
134  flags.addFlag(flagString, runArgs.outputDRAW_EGZFile)
135  flags.Output.doWriteBS = True
136  log.info("---------- Configured DRAW_EGZ output")
137 
138  if hasattr(runArgs, 'outputDRAW_TAULHFile'):
139  flagString = 'Output.DRAW_TAULHFileName'
140  flags.addFlag(flagString, runArgs.outputDRAW_TAULHFile)
141  flags.Output.doWriteBS = True
142  log.info("---------- Configured DRAW_TAULH output")
143 
144  if hasattr(runArgs, 'outputDRAW_JETFile'):
145  flagString = 'Output.DRAW_JETFileName'
146  flags.addFlag(flagString, runArgs.outputDRAW_JETFile)
147  flags.Output.doWriteBS = True
148  log.info("---------- Configured DRAW_JET output")
149 
150  if hasattr(runArgs, 'outputDAOD_L1CALO1File'):
151  flagString = 'Output.DAOD_L1CALO1FileName'
152  flags.addFlag(flagString, runArgs.outputDAOD_L1CALO1File)
153  flags.Output.doWriteDAOD = True
154  flags.addFlag('Output.doWriteDAOD_L1CALO1', True)
155  log.info("---------- Configured DAOD_L1CALO1 output")
156 
157  if hasattr(runArgs, 'outputDAOD_L1CALO2File'):
158  flagString = 'Output.DAOD_L1CALO2FileName'
159  flags.addFlag(flagString, runArgs.outputDAOD_L1CALO2File)
160  flags.Output.doWriteDAOD = True
161  flags.addFlag('Output.doWriteDAOD_L1CALO2', True)
162  log.info("---------- Configured DAOD_L1CALO2 output")
163 
164  if hasattr(runArgs, 'outputDESDM_PHOJETFile'):
165  flagString = 'Output.DESDM_PHOJETFileName'
166  flags.addFlag(flagString, runArgs.outputDESDM_PHOJETFile)
167  flags.Output.doWriteDAOD = True
168  flags.addFlag('Output.doWriteDESDM_PHOJET', True)
169  log.info("---------- Configured DESDM_PHOJET output")
170 
171  if hasattr(runArgs, 'outputDESDM_ALLCELLSFile'):
172  streamName = 'DESDM_ALLCELLS'
173  flags.addFlag(f'Output.{streamName}FileName', runArgs.outputDESDM_ALLCELLSFile)
174  flags.addFlag(f'Output.doWrite{streamName}', True)
175  log.info("---------- Configured "+streamName+" output")
176 
177  # Reconstruction flags should be parsed after inputs are set
178  from RecJobTransforms.RecoConfigFlags import recoRunArgsToFlags
179  recoRunArgsToFlags(runArgs, flags)
180 
181  from AthenaConfiguration.Enums import ProductionStep
182  flags.Common.ProductionStep=ProductionStep.Reconstruction
183 
184  # Setup detector flags
185  from AthenaConfiguration.DetectorConfigFlags import setupDetectorFlags
186  setupDetectorFlags(flags, detectors, use_metadata=True, toggle_geometry=True, keep_beampipe=True)
187  # Print reco domain status
188  from RecJobTransforms.RecoConfigFlags import printRecoFlags
189  printRecoFlags(flags)
190 
191  # TODO: DESD, DAOD and DRAW
192 
193  # outputHIST_R2AFile
194  # outputTAG_COMMFile
195  # outputTXT_FTKIPFile
196  # outputNTUP_MUONCALIBFile
197  # outputTXT_JIVEXMLTGZFile
198 
199  # Setup perfmon flags from runargs
200  from PerfMonComps.PerfMonConfigHelpers import setPerfmonFlagsFromRunArgs
201  setPerfmonFlagsFromRunArgs(flags, runArgs)
202 
203  # Pre-include
204  processPreInclude(runArgs, flags)
205 
206  # Pre-exec
207  processPreExec(runArgs, flags)
208 
209  # To respect --athenaopts
210  flags.fillFromArgs()
211 
212  # Disable event timeout if debugging has been requested.
213  if flags.Exec.DebugStage != '':
214  flags.Exec.EventTimeOut = 0
215 
216  # Lock flags
217  flags.lock()
218 
219  # Main reconstruction steering
220  from RecJobTransforms.RecoSteering import RecoSteering
221  cfg = RecoSteering(flags)
222 
223  # Performance DPDs
224  cfg.flagPerfmonDomain('PerfDPD')
225  # IDTIDE
226  for flag in [key for key in flags._flagdict.keys() if ("Output.DAOD_IDTIDEFileName" in key)]:
227  from DerivationFrameworkInDet.IDTIDE import IDTIDECfg
228  cfg.merge(IDTIDECfg(flags))
229  log.info("---------- Configured IDTIDE perfDPD")
230 
231  # PixelVALID
232  for flag in [key for key in flags._flagdict.keys() if ("Output.DAOD_PIXELVALIDFileName" in key)]:
233  from DerivationFrameworkInDet.PIXELVALID import PixelVALIDCfg
234  cfg.merge(PixelVALIDCfg(flags))
235  log.info("---------- Configured PixelVALID perfDPD")
236 
237  # SCTVALID
238  for flag in [key for key in flags._flagdict.keys() if ("Output.DAOD_SCTVALIDFileName" in key)]:
239  from InDetPrepRawDataToxAOD.SCTVALID import SCTVALIDCfg
240  cfg.merge(SCTVALIDCfg(flags))
241  log.info("---------- Configured SCTVALID perfDPD")
242 
243  # IDTRKVALID
244  for flag in [key for key in flags._flagdict.keys() if ("Output.DAOD_IDTRKVALIDFileName" in key)]:
245  from DerivationFrameworkInDet.IDTRKVALID import IDTRKVALIDCfg
246  cfg.merge(IDTRKVALIDCfg(flags))
247  log.info("---------- Configured IDTRKVALID perfDPD")
248 
249  # DESDM_MCP
250  for flag in [key for key in flags._flagdict.keys() if ("Output.DESDM_MCPFileName" in key)]:
251  from PrimaryDPDMaker.DESDM_MCP import DESDM_MCPCfg
252  cfg.merge(DESDM_MCPCfg(flags))
253  log.info("---------- Configured DESDM_MCP perfDPD")
254 
255  # DESDM_EXOTHIP
256  for flag in [key for key in flags._flagdict.keys() if ("Output.DESDM_EXOTHIPFileName" in key)]:
257  from PrimaryDPDMaker.DESDM_EXOTHIP import DESDM_EXOTHIPCfg
258  cfg.merge(DESDM_EXOTHIPCfg(flags))
259  log.info("---------- Configured DESDM_EXOTHIP perfDPD")
260 
261  # DRAW ZMUMU
262  for flag in [key for key in flags._flagdict.keys() if ("Output.DRAW_ZmumuFileName" in key)]:
263  from PrimaryDPDMaker.DRAW_ZMUMU import DRAW_ZmumuCfg
264  cfg.merge(DRAW_ZmumuCfg(flags))
265  log.info("---------- Configured DRAW_ZMUMU perfDPD")
266 
267  #DRAW_EGZ
268  for flag in [key for key in flags._flagdict.keys() if ("Output.DRAW_EGZFileName" in key)]:
269  from PrimaryDPDMaker.DRAW_EGZ import DRAW_EGZCfg
270  cfg.merge(DRAW_EGZCfg(flags))
271  log.info("---------- Configured DRAW_EGZ perfDPD")
272 
273  #DRAW_TAULH
274  for flag in [key for key in flags._flagdict.keys() if ("Output.DRAW_TAULHFileName" in key)]:
275  from PrimaryDPDMaker.DRAW_TAULH import DRAW_TAULHCfg
276  cfg.merge(DRAW_TAULHCfg(flags))
277  log.info("---------- Configured DRAW_TAULH perfDPD")
278 
279  #DRAW_JET
280  for flag in [key for key in flags._flagdict.keys() if ("Output.DRAW_JETFileName" in key)]:
281  from PrimaryDPDMaker.DRAW_JET import DRAW_JETCfg
282  cfg.merge(DRAW_JETCfg(flags))
283  log.info("---------- Configured DRAW_JET perfDPD")
284 
285  # L1CALO1/2
286  for flag in [key for key in flags._flagdict.keys() if ("Output.DAOD_L1CALO1FileName" in key)]:
287  from DerivationFrameworkL1Calo.L1CALO1 import L1CALO1Cfg
288  cfg.merge(L1CALO1Cfg(flags))
289  log.info("---------- Configured L1CALO1 perfDPD")
290 
291  for flag in [key for key in flags._flagdict.keys() if ("Output.DAOD_L1CALO2FileName" in key)]:
292  from DerivationFrameworkL1Calo.L1CALO2 import L1CALO2Cfg
293  cfg.merge(L1CALO2Cfg(flags))
294  log.info("---------- Configured L1CALO2 perfDPD")
295 
296  # DESDM PHOJET
297  for flag in [key for key in flags._flagdict.keys() if ("Output.DESDM_PHOJETFileName" in key)]:
298  from PrimaryDPDMaker.DESDM_PHOJET import DESDM_PHOJETCfg
299  cfg.merge(DESDM_PHOJETCfg(flags))
300  log.info("---------- Configured PHOJET perfDPD")
301 
302  # DESDM ALLCELLS
303  for flag in [key for key in flags._flagdict.keys() if ("Output.DESDM_ALLCELLSFileName" in key)]:
304  from PrimaryDPDMaker.DESDM_ALLCELLS import DESDM_ALLCELLSCfg
305  cfg.merge(DESDM_ALLCELLSCfg(flags))
306  log.info("---------- Configured ALLCELLS perfDPD")
307 
308  # Special message service configuration
309  from DigitizationConfig.DigitizationSteering import DigitizationMessageSvcCfg
310  cfg.merge(DigitizationMessageSvcCfg(flags))
311 
312  # Post-include
313  processPostInclude(runArgs, flags, cfg)
314 
315  # Post-exec
316  processPostExec(runArgs, flags, cfg)
317 
318  from AthenaConfiguration.Utils import setupLoggingLevels
319  setupLoggingLevels(flags, cfg)
320 
321  # Write some metadata into TagInfo
322  from EventInfoMgt.TagInfoMgrConfig import TagInfoMgrCfg
323  cfg.merge(
325  flags,
326  tagValuePairs={
327  "beam_type": flags.Beam.Type.value,
328  "beam_energy": str(int(flags.Beam.Energy)),
329  "triggerStreamOfFile": ""
330  if flags.Input.isMC
331  else flags.Input.TriggerStream,
332  "project_name": "IS_SIMULATION"
333  if flags.Input.isMC
334  else flags.Input.ProjectName,
335  f"AtlasRelease_{runArgs.trfSubstepName}": flags.Input.Release or "n/a",
336  },
337  )
338  )
339  if not flags.Input.isMC and flags.Input.DataYear > 0:
340  cfg.merge(TagInfoMgrCfg(flags, tagValuePairs={
341  "data_year": str(flags.Input.DataYear)
342  }))
343 
344  # Write AMI tag into in-file metadata
345  from PyUtils.AMITagHelperConfig import AMITagCfg
346  cfg.merge(AMITagCfg(flags, runArgs))
347 
348  # Print PerfMon domain information when running detailed monitoring
349  if flags.PerfMon.doFullMonMT:
350  cfg.printPerfmonDomains()
351 
352  timeConfig = time.time()
353  log.info("configured in %d seconds", timeConfig - timeStart)
354 
355  log.info("Configured according to flag values listed below")
356  flags.dump()
357 
358  # Print sum information about AccumulatorCache performance
359  from AthenaConfiguration.AccumulatorCache import AccumulatorDecorator
360  AccumulatorDecorator.printStats()
361 
362  # Run the final accumulator
363  sc = cfg.run()
364  timeFinal = time.time()
365  log.info("Run RAWtoALL_skeleton in %d seconds (running %d seconds)", timeFinal - timeStart, timeFinal - timeConfig)
366 
367  import sys
368  sys.exit(not sc.isSuccess())
python.RAWtoALL_Skeleton.fromRunArgs
def fromRunArgs(runArgs)
Definition: RAWtoALL_Skeleton.py:14
python.RecoSteering.RecoSteering
def RecoSteering(flags)
Definition: RecoSteering.py:6
python.DESDM_PHOJET.DESDM_PHOJETCfg
def DESDM_PHOJETCfg(configFlags)
Definition: DESDM_PHOJET.py:40
python.TransformUtils.processPreExec
def processPreExec(runArgs, flags)
Definition: Tools/PyJobTransforms/python/TransformUtils.py:41
vtune_athena.format
format
Definition: vtune_athena.py:14
python.TransformUtils.processPostExec
def processPostExec(runArgs, flags, cfg)
Definition: Tools/PyJobTransforms/python/TransformUtils.py:50
python.DESDM_MCP.DESDM_MCPCfg
def DESDM_MCPCfg(flags)
Definition: DESDM_MCP.py:357
python.DESDM_ALLCELLS.DESDM_ALLCELLSCfg
def DESDM_ALLCELLSCfg(configFlags)
Definition: DESDM_ALLCELLS.py:27
L1CALO1.L1CALO1Cfg
def L1CALO1Cfg(flags)
Definition: L1CALO1.py:14
python.Helpers.ROOTSetup
def ROOTSetup(batch=False)
Definition: Tools/PyUtils/python/Helpers.py:18
python.RecoConfigFlags.printRecoFlags
def printRecoFlags(flags)
Definition: RecoConfigFlags.py:155
python.TransformUtils.processPostInclude
def processPostInclude(runArgs, flags, cfg)
Definition: Tools/PyJobTransforms/python/TransformUtils.py:69
python.RecoConfigFlags.recoRunArgsToFlags
def recoRunArgsToFlags(runArgs, flags)
Definition: RecoConfigFlags.py:198
python.TransformUtils.processPreInclude
def processPreInclude(runArgs, flags)
Definition: Tools/PyJobTransforms/python/TransformUtils.py:62
python.TagInfoMgrConfig.TagInfoMgrCfg
def TagInfoMgrCfg(flags, tagValuePairs={})
Definition: TagInfoMgrConfig.py:6
python.DigitizationSteering.DigitizationMessageSvcCfg
def DigitizationMessageSvcCfg(flags)
Definition: DigitizationSteering.py:217
python.DRAW_TAULH.DRAW_TAULHCfg
def DRAW_TAULHCfg(flags)
Definition: DRAW_TAULH.py:121
python.Utils.setupLoggingLevels
def setupLoggingLevels(flags, ca)
Definition: Control/AthenaConfiguration/python/Utils.py:50
python.SCTVALID.SCTVALIDCfg
def SCTVALIDCfg(flags)
Definition: SCTVALID.py:198
python.PerfMonConfigHelpers.setPerfmonFlagsFromRunArgs
def setPerfmonFlagsFromRunArgs(flags, runArgs)
Definition: PerfMonConfigHelpers.py:3
python.CommonRunArgsToFlags.commonRunArgsToFlags
def commonRunArgsToFlags(runArgs, configFlags)
Definition: CommonRunArgsToFlags.py:12
python.PIXELVALID.PixelVALIDCfg
def PixelVALIDCfg(flags)
Definition: PIXELVALID.py:205
beamspotman.dir
string dir
Definition: beamspotman.py:621
python.DetectorConfigFlags.setupDetectorFlags
def setupDetectorFlags(flags, custom_list=None, use_metadata=False, toggle_geometry=False, validate_only=False, keep_beampipe=False)
Definition: DetectorConfigFlags.py:292
python.DRAW_JET.DRAW_JETCfg
def DRAW_JETCfg(flags)
Definition: DRAW_JET.py:64
python.AMITagHelperConfig.AMITagCfg
def AMITagCfg(flags, runArgs=None, fixBroken=False)
Definition: AMITagHelperConfig.py:77
python.DRAW_EGZ.DRAW_EGZCfg
def DRAW_EGZCfg(flags)
Definition: DRAW_EGZ.py:94
python.CaloAddPedShiftConfig.int
int
Definition: CaloAddPedShiftConfig.py:45
python.AllConfigFlags.initConfigFlags
def initConfigFlags()
Definition: AllConfigFlags.py:19
python.DESDM_EXOTHIP.DESDM_EXOTHIPCfg
def DESDM_EXOTHIPCfg(flags)
Definition: DESDM_EXOTHIP.py:52
python.IDTRKVALID.IDTRKVALIDCfg
def IDTRKVALIDCfg(flags)
Definition: IDTRKVALID.py:230
str
Definition: BTagTrackIpAccessor.cxx:11
python.IDTIDE.IDTIDECfg
def IDTIDECfg(flags)
Definition: IDTIDE.py:319
L1CALO2.L1CALO2Cfg
def L1CALO2Cfg(flags)
Definition: L1CALO2.py:14
python.DRAW_ZMUMU.DRAW_ZmumuCfg
def DRAW_ZmumuCfg(flags)
Definition: DRAW_ZMUMU.py:83