ATLAS Offline Software
Loading...
Searching...
No Matches
Public Member Functions | Public Attributes | Protected Member Functions | List of all members
python.TrigValSteering.ExecStep.ExecStep Class Reference
Inheritance diagram for python.TrigValSteering.ExecStep.ExecStep:
Collaboration diagram for python.TrigValSteering.ExecStep.ExecStep:

Public Member Functions

 __init__ (self, name=None)
 construct_name (self)
 get_log_file_name (self)
 configure (self, test)
 configure_type (self)
 configure_input (self)
 configure_job_options (self)
 add_trf_precommand (self, precommand)
 configure_args (self, test)

Public Attributes

 input = None
 input_object = None
 job_options = None
list flags = []
int threads = None
int concurrent_events = None
int forks = None
int max_events = None
 skip_events = None
bool use_pickle = False
bool imf = True
bool perfmon = True
bool fpe_auditor = True
bool costmon = False
bool malloc = False
bool prmon = True
bool config_only = False
bool auto_report_result = True
bool required = True
bool depends_on_previous = True
str type = 'other':
 name
str log_file_name = self.name+'.log'
str executable
str args
int prmon_interval = 5

Protected Member Functions

 _is_good_file (self, path)

Detailed Description

Step executing the main job of a Trigger ART test. This can be either
athena or athenaHLT or a transform. There can be several ExecSteps in
one Test

Definition at line 16 of file ExecStep.py.

Constructor & Destructor Documentation

◆ __init__()

python.TrigValSteering.ExecStep.ExecStep.__init__ ( self,
name = None )

Definition at line 23 of file ExecStep.py.

23 def __init__(self, name=None):
24 super(ExecStep, self).__init__(name)
25 self.input = None
26 self.input_object = None
27 self.job_options = None
28 self.flags = []
29 self.threads = None
30 self.concurrent_events = None
31 self.forks = None
32 self.max_events = None
33 self.skip_events = None
34 self.use_pickle = False
35 self.imf = True
36 self.perfmon = True
37 self.fpe_auditor = True
38 self.costmon = False
39 self.malloc = False
40 self.prmon = True
41 self.config_only = False
42 self.auto_report_result = True
43 self.required = True
44 self.depends_on_previous = True
45

Member Function Documentation

◆ _is_good_file()

python.TrigValSteering.ExecStep.ExecStep._is_good_file ( self,
path )
protected

Definition at line 107 of file ExecStep.py.

107 def _is_good_file(self, path):
108 if os.path.isfile(path):
109 return True
110 from ROOT import TFile
111 with TFile.Open(path, "READ") as f:
112 if not f.IsOpen() or f.IsZombie():
113 return False
114 return True

◆ add_trf_precommand()

python.TrigValSteering.ExecStep.ExecStep.add_trf_precommand ( self,
precommand ) 
Add preExec to transform command

Definition at line 158 of file ExecStep.py.

158 def add_trf_precommand(self, precommand):
159 '''Add preExec to transform command'''
160
161 if self.type in ['Reco_tf', 'Trig_reco_tf', 'Derivation_tf']:
162 if 'inputBS_RDOFile' in self.args:
163 precommand = 'BSRDOtoRAW:' + precommand
164 elif 'outputRDO_TRIGFile' in self.args or 'doRDO_TRIG' in self.args:
165 precommand = 'RDOtoRDOTrigger:' + precommand
166 else:
167 self.log.debug('Skip adding precommand %s to step %s because it is a transform which '
168 'does not run Trigger', precommand, self.name)
169 return
170
171 # match --preExec, --preExec= ignoring spaces
172 m = re.search(r'--preExec\s*=?\s*', self.args)
173 if m is None:
174 self.args += f' --preExec "{precommand}" '
175 else:
176 # Insert new preExec. It is important to not use the '=' sign so we
177 # can chain multiple preExecs.
178 self.args = self.args[:m.span()[0]] + f' --preExec "{precommand}" ' + self.args[m.span()[1]:]
179
debug
const bool debug
Definition MakeUncertaintyPlots.cxx:53

◆ configure()

python.TrigValSteering.ExecStep.ExecStep.configure ( self,
test )

Definition at line 60 of file ExecStep.py.

60 def configure(self, test):
61 # Construct the name
62 self.name = self.construct_name()
63 self.log_file_name = self.name+'.log'
64 self.log.debug('Configuring step %s', self.name)
65
66 # Parse configuration
67 self.configure_type()
68 self.configure_input()
69 self.configure_job_options()
70 self.configure_args(test)
71
72 super(ExecStep, self).configure(test)
73
configure
bool configure(asg::AnaToolHandle< ITrigGlobalEfficiencyCorrectionTool > &tool, ToolHandleArray< IAsgElectronEfficiencyCorrectionTool > &electronEffToolsHandles, ToolHandleArray< IAsgElectronEfficiencyCorrectionTool > &electronSFToolsHandles, ToolHandleArray< CP::IMuonTriggerScaleFactors > &muonToolsHandles, ToolHandleArray< IAsgPhotonEfficiencyCorrectionTool > &photonEffToolsHandles, ToolHandleArray< IAsgPhotonEfficiencyCorrectionTool > &photonSFToolsHandles, const std::string &triggers, const std::map< std::string, std::string > &legsPerTool, unsigned long nToys, bool debug)
Definition TrigGlobEffCorrValidation.cxx:514

◆ configure_args()

python.TrigValSteering.ExecStep.ExecStep.configure_args ( self,
test )

Definition at line 180 of file ExecStep.py.

180 def configure_args(self, test):
181 self.log.debug('Configuring args for step %s', self.name)
182 if self.args is None:
183 self.args = ''
184 athenaopts = ''
185
186 # Disable prmon for Reco_tf because it is already started inside the transform
187 if self.type == 'Reco_tf' or self.type == 'Derivation_tf':
188 self.prmon = False
189
190 # Disable perfmon for multi-fork jobs as it cannot deal well with them
191 if self.forks and self.forks > 1 and self.perfmon:
192 self.log.debug('Disabling perfmon because forks=%d > 1', self.forks)
193 self.perfmon = False
194 # Disable perfmon for transforms (Reco_tf enables it itself, Trig_reco_tf would need special handling
195 # depending on whether it runs athena or athenaHLT)
196 if self.type.endswith('_tf') and self.perfmon:
197 self.log.debug('Disabling perfmon for the transform step type %s', self.type)
198 self.perfmon = False
199
200 # Append imf/perfmon
201 if self.type != 'other':
202 if self.imf:
203 athenaopts += ' --imf'
204 if self.perfmon:
205 if self.type == 'athenaHLT' or self.type == 'athenaEF':
206 athenaopts += ' --perfmon'
207 elif self.type == 'athena':
208 athenaopts += ' --perfmon=fastmonmt'
209 if self.malloc:
210 athenaopts += " --stdcmalloc "
211
212 # Enable CostMonitoring/FPEAuditor
213 if self.type != 'other':
214 if self.costmon:
215 self.flags.append('Trigger.CostMonitoring.monitorAllEvents=True')
216 if self.fpe_auditor:
217 self.flags.append('Exec.FPE=1')
218
219 # Run config-only if requested
220 if self.config_only :
221
222 if self.type == 'athenaHLT' or self.type == 'athenaEF' or (self.type == "other" and self.executable == "athenaHLT.py") or (self.type == "other" and self.executable == "athenaEF.py") :
223 athenaopts += ' --dump-config-exit'
224
225 elif self.type == 'athena' or self.type == 'Reco_tf' or self.type == 'Derivation_tf' or (self.type == "other" and self.executable == "athena.py") :
226 athenaopts += ' --config-only=' + self.name + '.pkl'
227
228 # No current support if it isn't clear exactly what's being run
229 # This includes Trig_reco_tf and 'other' where the executable is not known
230 else :
231 self.misconfig_abort('Cannot determine what config-only option is needed. Consider adding the appropriate flag to "args" instead.')
232
233 # Default threads/concurrent_events/forks
234 if test.package_name == 'TrigP1Test' and self.type == 'athenaHLT':
235 if self.threads is None:
236 self.threads = 1
237 if self.concurrent_events is None:
238 self.concurrent_events = 1
239 if self.forks is None:
240 self.forks = 1
241 if test.package_name == 'TrigP1Test' and self.type == 'athenaEF':
242 if self.threads is None:
243 self.threads = 1
244 if self.concurrent_events is None:
245 self.concurrent_events = 1
246
247 # Append threads/concurrent_events/forks
248 if self.threads is not None:
249 athenaopts += ' --threads={}'.format(self.threads)
250 if self.concurrent_events is not None:
251 athenaopts += ' --concurrent-events={}'.format(
252 self.concurrent_events)
253 if self.forks is not None and self.type != 'athenaEF':
254 athenaopts += ' --nprocs={}'.format(self.forks)
255
256 # Append athenaopts
257 athenaopts = athenaopts.strip()
258 if self.type.endswith('_tf'):
259 self.args += ' --athenaopts="{}"'.format(athenaopts)
260 else:
261 self.args += ' '+athenaopts
262
263 # Default max events
264 if self.max_events is None:
265 if test.art_type == 'build':
266 if test.package_name == 'TrigP1Test':
267 self.max_events = 80
268 else:
269 self.max_events = 20
270 else:
271 self.max_events = 1000
272
273 # Set prmon interval based on max events
274 if self.prmon:
275 if self.max_events <= 100:
276 self.prmon_interval = 5
277 else:
278 self.prmon_interval = 10
279
280 # Append max/skip events
281 if self.type == 'athena':
282 self.args += ' --evtMax={}'.format(self.max_events)
283 elif self.type == 'athenaHLT' or self.type == 'athenaEF':
284 self.args += ' --number-of-events={}'.format(self.max_events)
285 elif self.type.endswith('_tf'):
286 self.args += ' --maxEvents={}'.format(self.max_events)
287 if self.skip_events is not None:
288 if self.type == 'athena':
289 self.args += ' --skipEvents={}'.format(self.skip_events)
290 elif self.type == 'athenaHLT' or self.type == 'athenaEF':
291 self.args += ' --skip-events={}'.format(self.skip_events)
292 elif self.type.endswith('_tf'):
293 self.args += ' --skipEvents={}'.format(self.skip_events)
294
295 # Append input
296 if len(self.input) > 0:
297 if self.input_object is not None:
298 if self.type == 'athenaHLT' or self.type == 'athenaEF':
299 input_str = ' --file='.join(self.input_object.paths)
300 else:
301 input_str = ','.join(self.input_object.paths)
302 else:
303 input_str = self.input
304 if self.type == 'athena':
305 self.args += ' --filesInput={}'.format(input_str)
306 elif self.type == 'athenaHLT' or self.type == 'athenaEF':
307 self.args += ''.join([f" --file={inputFile}" for inputFile in input_str.split(',')])
308 elif self.type.endswith('_tf'):
309 if self.input_object is None:
310 self.misconfig_abort(
311 'Cannot build inputXYZFile string for transform '
312 ' from explicit input path. Use input=\'\' and '
313 'specify the input explicitly in args')
314 if self.type == 'Trig_reco_tf' and '--prodSysBSRDO True' in self.args:
315 self.args += ' --inputBS_RDOFile={}'.format(input_str)
316 else:
317 self.args += ' --input{}File={}'.format(
318 self.input_object.format, input_str)
319
320
321 # Append job options
322 if self.job_options is not None:
323 self.args += ' '+self.job_options
324
325 # Append flags
326 if self.flags:
327 if not isinstance(self.flags, (list, tuple)):
328 self.misconfig_abort('Wrong type for flags. Expected list or tuple.')
329
330 if self.type.endswith('_tf'): # for transform, set flags as pre-exec
331 if self.type == 'Trig_reco_tf':
332 # No 'flags.' prefix for the trigger transform
333 self.add_trf_precommand(' '.join(f'{flag}' for flag in self.flags))
334 else:
335 self.add_trf_precommand(';'.join(f'flags.{flag}' for flag in self.flags))
336 else: # athena(HLT)
337 self.args += ' ' + ' '.join(self.flags)
338
339 # Strip extra whitespace
340 self.args = self.args.strip()

◆ configure_input()

python.TrigValSteering.ExecStep.ExecStep.configure_input ( self)

Definition at line 115 of file ExecStep.py.

115 def configure_input(self):
116 self.log.debug('Configuring input for step %s', self.name)
117 if self.input is None:
118 self.misconfig_abort(
119 'Input not provided for this step. To configure'
120 'a step without input, use an empty string')
121
122 # Step with no input
123 if len(self.input) == 0:
124 return
125
126 # Try to interpret input as keyword
127 if is_input_defined(self.input):
128 self.input_object = get_input(self.input)
129 if self.input_object is None:
130 self.misconfig_abort('Failed to load input with keyword %s', self.input)
131 return
132
133 # Try to interpret explicit paths
134 input_paths = self.input.split(',')
135 for path in input_paths:
136 if not self._is_good_file(path):
137 self.misconfig_abort(f"The provided input does not exist: {path}")
138 self.log.debug('Using explicit input: %s', self.input)
139
split
std::vector< std::string > split(const std::string &s, const std::string &t=":")
Definition hcg.cxx:179

◆ configure_job_options()

python.TrigValSteering.ExecStep.ExecStep.configure_job_options ( self) 
Check job options configuration

Definition at line 140 of file ExecStep.py.

140 def configure_job_options(self):
141 '''Check job options configuration'''
142 self.log.debug('Configuring job options for step %s', self.name)
143 if self.type == 'other':
144 self.log.debug('Skipping job options check for step.type=other')
145 return
146 if self.use_pickle:
147 self.log.debug('Skipping job options check for step running from a pickle file')
148 return
149
150 if self.type.endswith('_tf'):
151 if self.job_options is None:
152 return
153 else:
154 self.misconfig_abort('Transform %s does not accept job options', self.type)
155 elif self.job_options is None or len(self.job_options) == 0:
156 self.misconfig_abort('Job options not provided for this step')
157

◆ configure_type()

python.TrigValSteering.ExecStep.ExecStep.configure_type ( self)

Definition at line 74 of file ExecStep.py.

74 def configure_type(self):
75 self.log.debug('Configuring type for step %s', self.name)
76 # Check if type or executable is specified
77 if self.type is None and self.executable is None:
78 self.misconfig_abort('Cannot configure a step without specified type or executable')
79
80 # Configure executable from type
81 known_types = ['athena', 'athenaHLT', 'athenaEF', 'Reco_tf', 'Trig_reco_tf', 'Derivation_tf']
82 if self.type in known_types:
83 if self.executable is not None:
84 self.log.warning('type=%s was specified, so executable=%s '
85 'will be ignored and deduced automatically '
86 'from type', self.type, self.executable)
87 self.executable = '{}.py'.format(self.type)
88 elif self.type == 'other' or self.type is None:
89 self.type = 'other'
90 else:
91 self.misconfig_abort('Cannot determine type of this step')
92
93 if self.executable.endswith('_tf.py'):
94 def del_env(varname):
95 if varname in os.environ:
96 del os.environ[varname]
97
98 # Ensure no log duplication for transforms
99 os.environ['TRF_NOECHO'] = '1'
100 del_env('TRF_ECHO')
101
102 # We don't use the Reco_tf auto-configuration for MP/MT transforms,
103 # instead we pass our parameters of choice to athenaopts
104 del_env('ATHENA_NPROC_NUM')
105 del_env('ATHENA_CORE_NUMBER')
106

◆ construct_name()

python.TrigValSteering.ExecStep.ExecStep.construct_name ( self)

Definition at line 46 of file ExecStep.py.

46 def construct_name(self):
47 if self.name and self.type == 'other':
48 return self.name
49 name = self.type if self.type else self.executable
50 if self.name:
51 name += '.'+self.name
52 return name
53

◆ get_log_file_name()

python.TrigValSteering.ExecStep.ExecStep.get_log_file_name ( self)

Definition at line 54 of file ExecStep.py.

54 def get_log_file_name(self):
55 if self.log_file_name is not None:
56 return self.log_file_name
57 else:
58 return self.construct_name()+'.log'
59

Member Data Documentation

◆ args

str python.TrigValSteering.ExecStep.ExecStep.args

Definition at line 162 of file ExecStep.py.

◆ auto_report_result

bool python.TrigValSteering.ExecStep.ExecStep.auto_report_result = True

Definition at line 42 of file ExecStep.py.

◆ concurrent_events

int python.TrigValSteering.ExecStep.ExecStep.concurrent_events = None

Definition at line 30 of file ExecStep.py.

◆ config_only

bool python.TrigValSteering.ExecStep.ExecStep.config_only = False

Definition at line 41 of file ExecStep.py.

◆ costmon

bool python.TrigValSteering.ExecStep.ExecStep.costmon = False

Definition at line 38 of file ExecStep.py.

◆ depends_on_previous

bool python.TrigValSteering.ExecStep.ExecStep.depends_on_previous = True

Definition at line 44 of file ExecStep.py.

◆ executable

str python.TrigValSteering.ExecStep.ExecStep.executable

Definition at line 86 of file ExecStep.py.

◆ flags

python.TrigValSteering.ExecStep.ExecStep.flags = []

Definition at line 28 of file ExecStep.py.

◆ forks

python.TrigValSteering.ExecStep.ExecStep.forks = None

Definition at line 31 of file ExecStep.py.

◆ fpe_auditor

bool python.TrigValSteering.ExecStep.ExecStep.fpe_auditor = True

Definition at line 37 of file ExecStep.py.

◆ imf

bool python.TrigValSteering.ExecStep.ExecStep.imf = True

Definition at line 35 of file ExecStep.py.

◆ input

python.TrigValSteering.ExecStep.ExecStep.input = None

Definition at line 25 of file ExecStep.py.

◆ input_object

python.TrigValSteering.ExecStep.ExecStep.input_object = None

Definition at line 26 of file ExecStep.py.

◆ job_options

python.TrigValSteering.ExecStep.ExecStep.job_options = None

Definition at line 27 of file ExecStep.py.

◆ log_file_name

str python.TrigValSteering.ExecStep.ExecStep.log_file_name = self.name+'.log'

Definition at line 63 of file ExecStep.py.

◆ malloc

bool python.TrigValSteering.ExecStep.ExecStep.malloc = False

Definition at line 39 of file ExecStep.py.

◆ max_events

int python.TrigValSteering.ExecStep.ExecStep.max_events = None

Definition at line 32 of file ExecStep.py.

◆ name

python.TrigValSteering.ExecStep.ExecStep.name

Definition at line 50 of file ExecStep.py.

◆ perfmon

bool python.TrigValSteering.ExecStep.ExecStep.perfmon = True

Definition at line 36 of file ExecStep.py.

◆ prmon

bool python.TrigValSteering.ExecStep.ExecStep.prmon = True

Definition at line 40 of file ExecStep.py.

◆ prmon_interval

int python.TrigValSteering.ExecStep.ExecStep.prmon_interval = 5

Definition at line 276 of file ExecStep.py.

◆ required

bool python.TrigValSteering.ExecStep.ExecStep.required = True

Definition at line 43 of file ExecStep.py.

◆ skip_events

python.TrigValSteering.ExecStep.ExecStep.skip_events = None

Definition at line 33 of file ExecStep.py.

◆ threads

int python.TrigValSteering.ExecStep.ExecStep.threads = None

Definition at line 29 of file ExecStep.py.

◆ type

python.TrigValSteering.ExecStep.ExecStep.type = 'other':

Definition at line 47 of file ExecStep.py.

◆ use_pickle

bool python.TrigValSteering.ExecStep.ExecStep.use_pickle = False

Definition at line 34 of file ExecStep.py.

The documentation for this class was generated from the following file:
Generated on for ATLAS Offline Software by doxygen 1.16.1