ATLAS Offline Software
Loading...
Searching...
No Matches
python.ScriptUtils Namespace Reference

Functions

logging.Logger setup_logger (str name)
ArgumentParser setup_parser ()
TestSetup get_test_setup (str name, Namespace options, logging.Logger log)
None parse_test_string (TestSetup setup, Namespace options)
list[WorkflowCheck] get_standard_performance_checks (TestSetup setup)
None run_tests (TestSetup setup, list[WorkflowTest] tests)
bool run_checks (TestSetup setup, list[WorkflowTest] tests, list[WorkflowCheck] performance_checks)
None run_summary (TestSetup setup, list[WorkflowTest] tests, bool status)

Variables

 _default_fmt = fmt

Function Documentation

◆ get_standard_performance_checks()

list[WorkflowCheck] get_standard_performance_checks ( TestSetup setup)

Definition at line 226 of file ScriptUtils.py.

226def get_standard_performance_checks(setup: TestSetup) -> list[WorkflowCheck]:
227 return [
228 SimpleCheck(setup, "CPU Time" , "evtloop_time", "msec/event", 4, 0.4),
229 SimpleCheck(setup, "Physical Memory", "VmRSS", "kBytes", 4, 0.2),
230 SimpleCheck(setup, "Virtual Memory" , "VmSize", "kBytes", 4, 0.2),
231 SimpleCheck(setup, "Memory Leak" , "leakperevt_evt11", "kBytes/event", 7, 0.05),
232 WarningsComparisonCheck(setup),
233 ]
234
235

◆ get_test_setup()

TestSetup get_test_setup ( str name,
Namespace options,
logging.Logger log )

Definition at line 136 of file ScriptUtils.py.

136def get_test_setup(name: str, options: Namespace, log: logging.Logger) -> TestSetup:
137 # define test setup
138 setup = TestSetup(log)
139 setup.validation_run_path = Path(options.validation_run_path) if options.validation_run_path else Path.cwd()
140 setup.reference_run_path = Path(options.reference_run_path) if options.reference_run_path else Path(f"/tmp/{environ['USER']}")
141 setup.diff_rules_path = Path(options.diff_rules_path) if options.diff_rules_path is not None else None
142 setup.disable_release_setup = options.ci_mode
143 setup.validation_only = options.validation_only
144 setup.run_only = options.run_only
145 if options.unique_ID:
146 setup.checks_only = True
147 setup.unique_ID = options.unique_ID
148 setup.parallel_execution = options.fast_mode
149 setup.disable_output_checks = options.disable_output_checks
150 setup.custom_threads = options.threads
151 setup.detailed_comparison = options.detailed_comparison
152 setup.allow_tolerance = options.allow_tolerance or environ.get("NODE_NAME", "").startswith("aibuild-gpu")
153 # not in global setup:
154 # options.extra_args
155
156 if options.ami_tag and not options.workflow:
157 log.error("Custom AMI tags supported only with specific workflows!")
158 exit(1)
159
160 # Are we running in CI
161 if setup.disable_release_setup:
162 log.info("You're running in CI mode.")
163 log.info("This mode assumes athena is setup w/ necessary changes and only runs validation tests.")
164 log.info("Then results are checked against reference files and no performance test is run.")
165 log.info("If you don't know what this mode does, you shouldn't be using it.\n")
166 setup.validation_only = True
167
168 # Does the clean run head directory exist?
169 if setup.validation_only:
170 log.info("You are running in validation-only mode whereby only tests against your build are being run.")
171 log.info("In this mode ESD and AOD outputs are compared with pre-defined reference files found in the directory")
172 log.info(f"{references_CVMFS_path}\n")
173 if not Path(references_CVMFS_path).exists():
174 log.error(f"Exit. Validation-only mode can only be run on nodes with access to {references_CVMFS_path}")
175 exit(2)
176 elif setup.reference_run_path.exists():
177 log.info(f"The job unique ID is '{setup.unique_ID}' (can be used to re-run the checks)\n")
178 else:
179 log.error("Exit. Please specify a directory that exists for the argument of the --reference-path option\n")
180 log.error(f"{name}.py --reference-path <ExistingDirectory>")
181 exit(1)
182
183 # Is an ATLAS release setup?
184 if "AtlasPatchVersion" not in environ and "AtlasArea" not in environ and "AtlasBaseDir" not in environ and "AtlasVersion" not in environ:
185 log.warning("Not running in a standard ATLAS release setup.\n")
186
187 # setup reference path
188 setup.reference_run_path /= f"reference_test_{setup.unique_ID}"
189
190 # Release setup & list the packages in the local InstallArea
191 setup.setup_release(options.reference_release, options.validation_release)
192
193 # Parse test string if needed
194 parse_test_string(setup, options)
195
196 return setup
197
198
bool exists(const std::string &filename)
does a file exist

◆ parse_test_string()

None parse_test_string ( TestSetup setup,
Namespace options )

Definition at line 199 of file ScriptUtils.py.

199def parse_test_string(setup: TestSetup, options: Namespace) -> None:
200 if not options.test:
201 return
202
203 test_string = options.test.lower()
204
205 # simulation
206 if test_string in ["s", "sim", "simulation"]:
207 options.simulation = True
208 return
209
210 # overlay
211 if test_string in ["o", "overlay"]:
212 options.overlay = True
213 return
214
215 # pile-up
216 if test_string in ["p", "pileup", "pile-up"]:
217 options.pileup = True
218 return
219
220 # reco
221 if test_string in ["r", "reco", "reconstruction"]:
222 options.reco = True
223 return
224
225

◆ run_checks()

bool run_checks ( TestSetup setup,
list[WorkflowTest] tests,
list[WorkflowCheck] performance_checks )

Definition at line 271 of file ScriptUtils.py.

271def run_checks(setup: TestSetup, tests: list[WorkflowTest], performance_checks: list[WorkflowCheck]) -> bool:
272 all_passed = True
273 # define common checks
274 main_check = FailedOrPassedCheck(setup)
275 fpe_check = FPECheck(setup)
276 # run checks
277 for test in tests:
278 test_succeeded = main_check.run(test)
279 test_succeeded = fpe_check.run(test) and test_succeeded
280 all_passed = test_succeeded and all_passed
281 if test_succeeded and not setup.run_only:
282 all_passed = test.run_checks(performance_checks) and all_passed
283 return all_passed
284
285

◆ run_summary()

None run_summary ( TestSetup setup,
list[WorkflowTest] tests,
bool status )

Definition at line 286 of file ScriptUtils.py.

286def run_summary(setup: TestSetup, tests: list[WorkflowTest], status: bool) -> None:
287 setup.logger.info("-----------------------------------------------------")
288 setup.logger.info("---------------------- Summary ----------------------")
289 if status:
290 setup.logger.info("ALL TESTS: PASSED (0)")
291 else:
292 setup.logger.error("ALL TESTS: FAILED (10)")
293 exit(10)

◆ run_tests()

None run_tests ( TestSetup setup,
list[WorkflowTest] tests )

Definition at line 236 of file ScriptUtils.py.

236def run_tests(setup: TestSetup, tests: list[WorkflowTest]) -> None:
237 if setup.checks_only:
238 return
239
240 threads = {}
241 setup.logger.info("------------------ Run Athena workflow test jobs---------------")
242 if setup.parallel_execution and not setup.validation_only:
243 for test in tests:
244 threads[f"{test.ID}_reference"] = threading.Thread(target=lambda test=test: test.run_reference())
245 threads[f"{test.ID}_validation"] = threading.Thread(target=lambda test=test: test.run_validation())
246 threads[f"{test.ID}_reference"].start()
247 threads[f"{test.ID}_validation"].start()
248
249 for thread in threads.items():
250 thread.join()
251 elif setup.validation_only:
252 for test in tests:
253 threads[f"{test.ID}_validation"] = threading.Thread(target=lambda test=test: test.run_validation())
254 threads[f"{test.ID}_validation"].start()
255 if not setup.parallel_execution:
256 threads[f"{test.ID}_validation"].join()
257
258 if setup.parallel_execution:
259 for thread in threads.items():
260 thread.join()
261 else:
262 for test in tests:
263 threads[f"{test.ID}_reference"] = threading.Thread(target=lambda test=test: test.run_reference())
264 threads[f"{test.ID}_validation"] = threading.Thread(target=lambda test=test: test.run_validation())
265 threads[f"{test.ID}_reference"].start()
266 threads[f"{test.ID}_validation"].start()
267 threads[f"{test.ID}_reference"].join()
268 threads[f"{test.ID}_validation"].join()
269
270
void run_tests()

◆ setup_logger()

logging.Logger setup_logger ( str name)

Definition at line 14 of file ScriptUtils.py.

14def setup_logger(name: str) -> logging.Logger:
15 # Add level for plain printing
16 printLevel = logging.INFO + 5
17 printName = 'PRINT'
18 printMethodName = 'print'
19
20 def logForLevel(self, message, *args, **kwargs):
21 if self.isEnabledFor(logging.PRINT):
22 self._log(logging.PRINT, message, args, **kwargs)
23 def logToRoot(message, *args, **kwargs):
24 logging.log(logging.PRINT, message, *args, **kwargs)
25
26 logging.addLevelName(printLevel, printName)
27 setattr(logging, printName, printLevel)
28 setattr(logging.getLoggerClass(), printMethodName, logForLevel)
29 setattr(logging, printMethodName, logToRoot)
30 logging.addLevelName(logging.INFO + 1, 'PRINT')
31
32 # Setup global logging
33 class CustomFormatter(logging.Formatter):
34 """Custom formatter."""
35 def __init__(self, fmt):
36 self._default_fmt = fmt
37 super().__init__(fmt, datefmt="%m-%d %H:%M")
38
39 def format(self, record):
40 if record.levelno == logging.PRINT:
41 self._style._fmt = "%(message)s"
42 else:
43 self._style._fmt = self._default_fmt
44 return super().format(record)
45
46 fileFormatter = CustomFormatter("%(asctime)s %(levelname)-8s %(message)s")
47 fileHandler = logging.FileHandler(f"./{name}.log", mode="w")
48 fileHandler.setFormatter(fileFormatter)
49
50 streamFormatter = CustomFormatter("%(levelname)-8s %(message)s")
51 streamHandler = logging.StreamHandler(stdout)
52 streamHandler.setFormatter(streamFormatter)
53
54 logger = logging.getLogger()
55 logger.addHandler(fileHandler)
56 logger.addHandler(streamHandler)
57 logger.setLevel(logging.INFO)
58
59 return logger
60
61

◆ setup_parser()

ArgumentParser setup_parser ( )

Definition at line 62 of file ScriptUtils.py.

62def setup_parser() -> ArgumentParser:
63 parser = ArgumentParser()
64 common = parser.add_argument_group("common")
65 common.add_argument("-e", "--extra", type=str, dest="extra_args", default="",
66 help="Define additional args to pass e.g. --preExec 'r2e':'...' ")
67 common.add_argument("-f", "--fast", action="store_true", dest="fast_mode", default=False,
68 help="""Fast option will run all q tests simultaneously,
69 such that it will run faster if you have 4 cpu core slots on which to run. Be
70 warned! Only recommended when running on a high performance machine, not
71 lxplus!""")
72 common.add_argument("-v", "--validation", action="store_true", dest="validation_only", default=False,
73 help=f"""Run validation only.
74 File output comparisons will only be performed against pre-defined
75 reference files stored in the directory
76 {references_CVMFS_path}
77 and performance comparison tests will not be run.""")
78 common.add_argument("--run-only", action="store_true", dest="run_only", default=False,
79 help="Run only the main command(s).")
80 common.add_argument("--checks-only", type=str, dest="unique_ID", nargs="?", default=None, const='local',
81 help="Re-run only the checks.")
82
83 advanced = parser.add_argument_group("advanced")
84 advanced.add_argument("--CI", action="store_true", dest="ci_mode", default=False,
85 help="Will not setup Athena - only for CI tests!")
86 advanced.add_argument("--threads", type=int, dest="threads", default=None,
87 help="Override the number of threads to run the test with.")
88 advanced.add_argument("--ref", type=str, dest="reference_release", default=None,
89 help="Define a particular reference release.")
90 advanced.add_argument("--val", type=str, dest="validation_release", default=None,
91 help="Define a particular validation release")
92 advanced.add_argument("--output-path", type=str, dest="validation_run_path", default="",
93 help="Specify the head directory for running the validation tests. The default is ${PWD}")
94 advanced.add_argument("--reference-path", type=str, dest="reference_run_path", default="",
95 help="Specify the head directory for running the reference tests. The default is /tmp/${USER}")
96 advanced.add_argument("-z", "--exclusion-lists", "--interest-lists", type=str, dest="diff_rules_path", default=None,
97 help="""Specify the directory that contains the lists of variables that will be omitted
98 while comparing the outputs. The default is ./ and the format of the files is
99 ${test}_${format}_diff-exclusion-list.txt, e.g. q445_AOD_diff-exclusion-list.txt or
100 ${test}_${format}_diff-interest-list.txt, e.g. q445_AOD_diff-interest-list.txt.
101 The file should contain one regexp per line.""")
102 advanced.add_argument("--no-output-checks", action="store_true", dest="disable_output_checks", default=False,
103 help="Disable output checks")
104 advanced.add_argument("--detailed-comparison", action="store_true", dest="detailed_comparison", default=False,
105 help="Detailed output comparison")
106 advanced.add_argument("--allow-tolerance", action="store_true", dest="allow_tolerance", default=False,
107 help="Allow tolerance in output comparison")
108
109 tests = parser.add_argument_group("tests")
110 tests.add_argument("-t", "--test", type=str, dest="test", default=None,
111 help="Specify a test to run. Supported options are: sim, overlay, pileup, reco")
112 tests.add_argument("-a", "--tag", type=str, dest="ami_tag", default=None,
113 help="Override the AMI tag of the test.")
114 tests.add_argument("-w", "--workflow", type=WorkflowType, dest="workflow", choices=list(WorkflowType), default=None,
115 help="Specify the workflow that is being run (required for AMI tags or if you want to run only one workflow)")
116 tests.add_argument("--dsid", type=str, dest="dsid", default=None,
117 help="Override the DSID of the test (only for generation).")
118
119 # shortcuts
120 tests.add_argument("-g", "--gen", action="store_true", dest="generation", default=False,
121 help="Run generation test using Gen_tf.py")
122 tests.add_argument("-s", "--sim", action="store_true", dest="simulation", default=False,
123 help="Run simulation test using Sim_tf.py")
124 tests.add_argument("-o", "--overlay", action="store_true", dest="overlay", default=False,
125 help="Run overlay test using Overlay_tf.py")
126 tests.add_argument("-p", "--pileup", action="store_true", dest="pileup", default=False,
127 help="Run MC reconstruction chain with pile-up")
128 tests.add_argument("-r", "--reco", action="store_true", dest="reco", default=False,
129 help="Run MC reconstruction (in case the default execution also runs simulation)")
130 tests.add_argument("-d", "--derivation", action="store_true", dest="derivation", default=False,
131 help="Run derivation test using Derivation_tf.py")
132
133 return parser
134
135

Variable Documentation

◆ _default_fmt

python.ScriptUtils._default_fmt = fmt
protected

Definition at line 36 of file ScriptUtils.py.