ATLAS Offline Software
Merge_tf.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 
3 # Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
4 
5 
6 
7 import sys
8 import time
9 
10 # Setup core logging here
11 from PyJobTransforms.trfLogger import msg
12 msg.info('logging set in %s', sys.argv[0])
13 
14 from PyJobTransforms.transform import transform
15 from PyJobTransforms.trfExe import athenaExecutor, POOLMergeExecutor
16 from PyJobTransforms.trfArgs import addAthenaArguments, addDetectorArguments
17 from PyJobTransforms.trfDecorators import stdTrfExceptionHandler, sigUsrStackTrace
18 from RecJobTransforms.recTransformUtils import addCommonRecTrfArgs
19 from PyJobTransforms.trfExe import DQMergeExecutor
20 from PyJobTransforms.trfExe import bsMergeExecutor
21 from PyJobTransforms.trfArgs import addExtraDPDTypes
22 from PATJobTransforms.PATTransformUtils import addNTUPMergeSubsteps, addPhysValidationMergeFiles
23 from PATJobTransforms.PATTransformUtils import addDAODArguments, addDAODMergerSubsteps
24 
25 import PyJobTransforms.trfArgClasses as trfArgClasses
26 
27 @stdTrfExceptionHandler
28 @sigUsrStackTrace
29 def main():
30 
31  msg.info('This is %s', sys.argv[0])
32 
33  trf = getTransform()
34  trf.parseCmdLineArgs(sys.argv[1:])
35  trf.execute()
36  trf.generateReport()
37 
38  msg.info("%s stopped at %s, trf exit code %d", sys.argv[0], time.asctime(), trf.exitCode)
39  sys.exit(trf.exitCode)
40 
41 def getTransform():
42  executorSet = set()
43  executorSet.add(POOLMergeExecutor(name = 'ESDMerge', inData = ['ESD'], outData = ['ESD_MRG']))
44  executorSet.add(POOLMergeExecutor(name = 'AODMerge', inData = ['AOD'], outData = ['AOD_MRG']))
45  executorSet.add(DQMergeExecutor(name = 'DQHistogramMerge', inData = [('HIST_ESD', 'HIST_AOD'), 'HIST'], outData = ['HIST_MRG']))
46  executorSet.add(bsMergeExecutor(name = 'RAWFileMerge', exe = 'file_merging', inData = set(['BS']), outData = set(['BS_MRG'])))
47  executorSet.add(athenaExecutor(name = 'EVNTMerge', skeletonCA = 'EvgenJobTransforms.EVNTMerge_Skeleton', inData = ['EVNT'], outData = ['EVNT_MRG']))
48 
49  addDAODMergerSubsteps(executorSet)
50  addNTUPMergeSubsteps(executorSet)
51 
52  trf = transform(executor = executorSet)
53 
54  addAthenaArguments(trf.parser)
55  addDetectorArguments(trf.parser)
56  addCommonRecTrfArgs(trf.parser)
57  addMyArgs(trf.parser)
58 
59  addDAODArguments(trf.parser)
60  addPhysValidationMergeFiles(trf.parser)
61  addExtraDPDTypes(trf.parser, transform=trf, NTUPMergerArgs = True)
62 
63  # Add HITSMerge and RDOMerge only if SimuJobTransforms is available
64  try:
65  from SimuJobTransforms.SimTransformUtils import addHITSMergeArguments
66  addHITSMergeArguments(trf.parser)
67  trf.appendToExecutorSet(athenaExecutor(name = 'HITSMerge', substep='hitsmerge',
68  skeletonCA = 'SimuJobTransforms.HITSMerge_Skeleton',
69  tryDropAndReload = False, inData = ['HITS'], outData = ['HITS_MRG']))
70 
71  trf.appendToExecutorSet(athenaExecutor(name = 'RDOMerge',
72  skeletonCA = 'SimuJobTransforms.RDOMerge_Skeleton',
73  inData = ['RDO'], outData = ['RDO_MRG']))
74 
75  trf.parser.defineArgGroup('RDOMerge_tf', 'RDO merge job specific options')
76  trf.parser.add_argument('--inputRDOFile', nargs='+',
77  type=trfArgClasses.argFactory(trfArgClasses.argRDOFile, io='input'),
78  help='Input RDO file', group='RDOMerge_tf')
79  trf.parser.add_argument('--outputRDO_MRGFile', '--outputRDOFile',
80  type=trfArgClasses.argFactory(trfArgClasses.argRDOFile, io='output'),
81  help='Output merged RDO file', group='RDOMerge_tf')
82  trf.parser.add_argument('--PileUpPresampling',
83  type=trfArgClasses.argFactory(trfArgClasses.argBool),
84  help='Run digitization with pile-up presampling configuration.', group='RDOMerge_tf')
85  except ImportError as e:
86  msg.warning('Failed to import simulation arguments ({0}). HITSMerge and RDOMerge will not be available.'.format(e))
87 
88 
89  return trf
90 
91 
92 def addMyArgs(parser):
93  # Use arggroup to get these arguments in their own sub-section (of --help)
94  parser.defineArgGroup('ESDMerge_tf', 'ESD merge job specific options')
95  parser.add_argument('--inputESDFile', nargs='+',
96  type=trfArgClasses.argFactory(trfArgClasses.argPOOLFile, io='input'),
97  help='Input ESD file', group='ESDMerge_tf')
98  parser.add_argument('--outputESD_MRGFile', '--outputESDFile',
99  type=trfArgClasses.argFactory(trfArgClasses.argPOOLFile, io='output'),
100  help='Output merged ESD file', group='ESDMerge_tf')
101 
102  parser.defineArgGroup('AODMerge_tf', 'AOD merge job specific options')
103  parser.add_argument('--inputAODFile', nargs='+',
104  type=trfArgClasses.argFactory(trfArgClasses.argPOOLFile, io='input', type='aod'),
105  help='Input AOD file', group='AODMerge_tf')
106  parser.add_argument('--inputAOD_MRGFile', nargs='+',
107  type=trfArgClasses.argFactory(trfArgClasses.argPOOLFile, io='input', type='aod'),
108  help='Input (premarged) AOD file', group='AODMerge_tf')
109  parser.add_argument('--outputAOD_MRGFile', '--outputAODFile',
110  type=trfArgClasses.argFactory(trfArgClasses.argPOOLFile, io='output', type='aod'),
111  help='Output merged AOD file', group='AODMerge_tf')
112 
113  parser.defineArgGroup('DQHistMerge_tf', 'DQ merge specific options')
114  parser.add_argument('--inputHISTFile', nargs='+',
115  type=trfArgClasses.argFactory(trfArgClasses.argHISTFile, io='input', runarg=True, type='hist'),
116  help='Input DQ HIST file', group='DQHistMerge_tf')
117  parser.add_argument('--inputHIST_AODFile', nargs='+',
118  type=trfArgClasses.argFactory(trfArgClasses.argHISTFile, io='input', runarg=True, type='hist_aod',
119  countable=False),
120  help='Input DQ AOD step monitoring file', group='DQHistMerge_tf')
121  parser.add_argument('--inputHIST_ESDFile', nargs='+',
122  type=trfArgClasses.argFactory(trfArgClasses.argHISTFile, io='input', runarg=True, type='hist_esd',
123  countable=False),
124  help='Input DQ AOD step monitoring file', group='DQHistMerge_tf')
125  parser.add_argument('--outputHIST_MRGFile', '--outputHISTFile', nargs='+',
126  type=trfArgClasses.argFactory(trfArgClasses.argHISTFile, io='output', runarg=True, type='hist'),
127  help='Output DQ monitoring file', group='DQHistMerge_tf')
128 
129  parser.defineArgGroup('RAWMerge_tf', 'RAWMerge specific options')
130  parser.add_argument('--inputBSFile', nargs='+',
131  type=trfArgClasses.argFactory(trfArgClasses.argBSFile, io='input'),
132  help='Input BS file(s)', group='RAWMerge_tf')
133  parser.add_argument('--outputBS_MRGFile', '--outputBSFile',
134  type=trfArgClasses.argFactory(trfArgClasses.argBSFile, io='output'),
135  help='Output merged BS file (best if this file ends in ._0001.data, but see allowRename option below)',
136  group='RAWMerge_tf')
137  parser.add_argument('--maskEmptyInputs', type=trfArgClasses.argFactory(trfArgClasses.argBool), group='RAWMerge_tf',
138  help='If true then empty BS files are not included in the merge (default True)',
139  default=trfArgClasses.argBool('True'))
140  parser.add_argument('--allowRename', type=trfArgClasses.argFactory(trfArgClasses.argBool), group='RAWMerge_tf',
141  help='If true merged BS file will be forcibly renamed to the value of "outputBSFile" (default True)',
142  default=trfArgClasses.argBool('True'))
143 
144  parser.defineArgGroup('EVNTMerge_tf', 'EVNT merge job specific options')
145  parser.add_argument('--inputEVNTFile', nargs='+',
146  type=trfArgClasses.argFactory(trfArgClasses.argEVNTFile, io='input', runarg=True, type='evnt'),
147  help='Input EVNT file', group='EVNTMerge_tf')
148  parser.add_argument('--outputEVNT_MRGFile', '--outputEVNTFile',
149  type=trfArgClasses.argFactory(trfArgClasses.argEVNTFile, io='output', runarg=True, type='evnt'),
150  help='Output merged EVNT file', group='EVNTMerge_tf')
151  parser.add_argument('--eventService', type=trfArgClasses.argFactory(trfArgClasses.argBool), metavar = "BOOL",
152  help='Switch AthenaMP to the Event Service configuration', group='EVNTMerge_tf')
153 
154 
155 if __name__ == '__main__':
156  main()
python.PATTransformUtils.addDAODArguments
def addDAODArguments(parser, mergerTrf=True)
Import list of known DAODs from the derivation framework and.
Definition: PATTransformUtils.py:48
python.trfArgs.addDetectorArguments
def addDetectorArguments(parser)
Options related to the setup of the ATLAS detector (used in simulation and digitisation as well as re...
Definition: trfArgs.py:229
python.trfArgs.addExtraDPDTypes
def addExtraDPDTypes(parser, pick=None, transform=None, multipleOK=False, NTUPMergerArgs=False)
Add additional DPD arguments.
Definition: trfArgs.py:440
vtune_athena.format
format
Definition: vtune_athena.py:14
Merge_tf.addMyArgs
def addMyArgs(parser)
Definition: Merge_tf.py:92
PyJobTransforms.trfArgClasses
Transform argument class definitions.
python.recTransformUtils.addCommonRecTrfArgs
def addCommonRecTrfArgs(parser)
Definition: recTransformUtils.py:14
python.PATTransformUtils.addNTUPMergeSubsteps
def addNTUPMergeSubsteps(executorSet)
Definition: PATTransformUtils.py:32
python.trfArgs.addAthenaArguments
def addAthenaArguments(parser, maxEventsDefaultSubstep='first', addValgrind=True, addPerfMon=True, addVTune=True)
Options related to running athena in general TODO: Some way to mask certain options (perExec,...
Definition: trfArgs.py:59
Amg::transform
Amg::Vector3D transform(Amg::Vector3D &v, Amg::Transform3D &tr)
Transform a point from a Trasformation3D.
Definition: GeoPrimitivesHelpers.h:156
python.PATTransformUtils.addPhysValidationMergeFiles
def addPhysValidationMergeFiles(parser)
Definition: PATTransformUtils.py:22
Merge_tf.getTransform
def getTransform()
Definition: Merge_tf.py:41
CxxUtils::set
constexpr std::enable_if_t< is_bitmask_v< E >, E & > set(E &lhs, E rhs)
Convenience function to set bits in a class enum bitmask.
Definition: bitmask.h:232
PyJobTransforms.trfExe
Transform execution functions.
Merge_tf.main
def main()
Definition: Merge_tf.py:29
PyJobTransforms.trfLogger
Logging configuration for ATLAS job transforms.
python.PATTransformUtils.addDAODMergerSubsteps
def addDAODMergerSubsteps(executorSet)
Definition: PATTransformUtils.py:68
PyJobTransforms.transform
Main package for new style ATLAS job transforms.
python.SimTransformUtils.addHITSMergeArguments
def addHITSMergeArguments(parser)
Definition: SimTransformUtils.py:51
Generated on Sat Jan 4 2025 21:14:18 for ATLAS Offline Software by   doxygen 1.8.18