ATLAS Offline Software
Tools/WorkflowTestRunner/python/Test.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
2 from enum import Enum
3 from logging import Logger
4 from os import environ
5 from pathlib import Path
6 from typing import List, Optional
7 from uuid import uuid4
8 import subprocess
9 
10 from .Helpers import get_release_setup, list_changed_packages
11 from .Inputs import references_CVMFS_path, references_override_url
12 from .References import references_map
13 
14 
15 class TestSetup:
16  """Test setup."""
17 
18  def __init__(self, logger: Logger) -> None:
19  self.logger = logger
20  self.validation_run_path = Path.cwd()
21  self.reference_run_path = Path("/tmp")
22  self.diff_rules_path = None
23  self.unique_ID = str(uuid4())
24  self.disable_release_setup = False
25  self.validation_only = False
26  self.run_only = False
27  self.checks_only = False
30  self.release_ID = "main" # The following is not flexible enough, can probably be hardcoded: environ["AtlasVersion"][0:4]
31  self.parallel_execution = False
32  self.disable_output_checks = False
33  self.custom_threads = None
34  self.detailed_comparison = False
35 
36  def setup_release(self, reference=None, validation=None) -> None:
37  if reference and validation:
38  self.release_reference = reference
39  self.release_validation = validation
40  self.logger.info(f"WARNING: You have specified a dedicated release as reference {reference} and as validation {validation} release.")
41  self.logger.info("Your local setup area will not be considered!!!")
42  self.logger.info("this option is mainly designed for comparing release versions!!")
43  elif reference:
44  self.release_reference = reference
45  self.release_validation = ''
46  self.logger.info(f"You have specified a dedicated release as reference {reference}.")
47  else:
50  try:
52  except Exception:
53  self.logger.warning("Cannot list changed packages...\n")
54 
55 
56 class WorkflowRun(Enum):
57  Run2 = "Run2"
58  Run3 = "Run3"
59  Run4 = "Run4"
60 
61  def __str__(self):
62  return self.name
63 
64 
65 class WorkflowType(Enum):
66  Generation = "Generation"
67  FullSim = "FullSim"
68  AF3 = "AF3"
69  HitsMerge = "HitsMerge"
70  HitsFilter = "HitsFilter"
71  MCOverlay = "MCOverlay"
72  DataOverlay = "DataOverlay"
73  MCReco = "MCReco"
74  MCPileUpReco = "MCPileUpReco"
75  DataReco = "DataReco"
76  PileUpPresampling = "PileUpPresampling"
77  MinbiasPreprocessing = "MinbiasPreprocessing"
78  Derivation = "Derivation"
79 
80  def __str__(self):
81  return self.name
82 
83 
85  """Workflow check base class."""
86 
87  def __init__(self, setup: TestSetup) -> None:
88  self.setup = setup
89  self.logger = setup.logger
90 
91  def reference_file(self, test: "WorkflowTest", file_name: str) -> Optional[Path]:
92  reference_path: Path = test.reference_path
93  reference_file = reference_path / file_name
94 
95  # Read references from CVMFS
96  if self.setup.validation_only:
97  # Resolve the subfolder first. Results are stored like: main_folder/branch/test/version/.
98  reference_revision = references_map[f"{test.ID}"]
99  cvmfs_path = Path(references_CVMFS_path)
100  rel_path = Path(self.setup.release_ID) / test.ID / reference_revision
101  reference_path = cvmfs_path / rel_path
102  reference_file = reference_path / file_name
103 
104  if not reference_path.exists():
105  self.logger.error(f"CVMFS reference location {reference_path} does not exist!")
106  return None
107 
108  if references_override_url is not None:
109  import requests
110 
111  url = references_override_url
112  if not url.endswith("/"): url += "/"
113  url += str(rel_path / file_name)
114  self.logger.info("Checking for reference override at %s", url)
115  if requests.head(url).ok: # file exists at url
116  reference_file = Path.cwd() / f"reference_{file_name}"
117  self.logger.info("Downloading reference from %s to %s", url, reference_file)
118  r = requests.get(url, stream=True)
119  with reference_file.open('wb') as f:
120  for chunk in r.iter_content(chunk_size=1024):
121  if chunk: # filter out keep-alive new chunks
122  f.write(chunk)
123  else:
124  self.logger.info("No reference override found")
125 
126  return reference_file
127 
128 
130  """Workflow test base class."""
131 
132  def __init__(self, ID: str, run: WorkflowRun, type: WorkflowType, steps: List[str], setup: TestSetup) -> None:
133  if not hasattr(self, "ID"):
134  self.ID = ID
135 
136  if not hasattr(self, "tag"):
137  self.tag = ID
138 
139  if not hasattr(self, "steps"):
140  self.steps = steps
141 
142  if not self.command:
143  raise NotImplementedError("Command needs to be defined")
144 
145  if not hasattr(self, "output_checks"):
146  self.output_checks = []
147 
148  if not hasattr(self, "digest_checks"):
149  self.digest_checks = []
150 
151  if not hasattr(self, "skip_performance_checks"):
153 
154  self.run = run
155  self.type = type
156  self.setup = setup
157  self.logger = setup.logger
158  self.validation_path: Path = self.setup.validation_run_path / f"run_{self.ID}"
159  self.reference_path: Path = self.setup.reference_run_path / f"run_{self.ID}"
160 
161  def run_reference(self) -> None:
162  self.logger.info(f"Running reference in rel {self.setup.release_reference}")
163  self.logger.info(f"\"{self.command}\"")
164 
165  self.reference_path.mkdir(parents=True, exist_ok=True)
166 
167  cmd = (f"cd {self.reference_path};"
168  f"source $AtlasSetup/scripts/asetup.sh {self.setup.release_reference} >& /dev/null;")
169  cmd += f"TRF_NOECHO=1 {self.command} > {self.ID}.log 2>&1"
170 
171  subprocess.call(cmd, shell=True)
172 
173  self.logger.info(f"Finished clean in rel {self.setup.release_reference}")
174  self.logger.info(f"\"{self.command}\"")
175 
176  def run_validation(self) -> None:
177  self.logger.info(f"Running validation in rel {self.setup.release_validation}")
178  self.logger.info(f"\"{self.command}\"")
179 
180  self.validation_path.mkdir(parents=True, exist_ok=True)
181 
182  cmd = f"cd {self.validation_path};"
183  if self.setup.disable_release_setup or not self.setup.release_validation:
184  pass
185  elif "WorkDir_DIR" in environ:
186  cmake_build_dir = environ["WorkDir_DIR"]
187  cmd += (f"source $AtlasSetup/scripts/asetup.sh {self.setup.release_validation} >& /dev/null;"
188  f"source {cmake_build_dir}/setup.sh;")
189  else:
190  cmd += f"source $AtlasSetup/scripts/asetup.sh {self.setup.release_validation} >& /dev/null;"
191  cmd += f"TRF_NOECHO=1 {self.command} > {self.ID}.log 2>&1"
192 
193  subprocess.call(cmd, shell=True)
194 
195  self.logger.info(f"Finished validation in rel {self.setup.release_validation}")
196  self.logger.info(f"\"{self.command}\"")
197 
198  def run_checks(self, performance_checks: List[WorkflowCheck]) -> bool:
199  self.logger.info("-----------------------------------------------------")
200  self.logger.info(f"----------- Post-processing of {self.ID} Test -----------")
201  result = True
202 
203  # digest checks
204  for check in self.digest_checks:
205  result = check.run(self) and result
206 
207  # output checks
208  if not self.setup.disable_output_checks:
209  for check in self.output_checks:
210  result = check.run(self) and result
211 
212  if self.setup.validation_only or self.skip_performance_checks:
213  return result # Performance checks against static references not possible
214 
215  # performance checks
216  for check in performance_checks:
217  result = check.run(self) and result
218 
219  return result
grepfile.info
info
Definition: grepfile.py:38
python.Helpers.get_release_setup
str get_release_setup(Logger logger, no_setup=False)
Definition: Tools/WorkflowTestRunner/python/Helpers.py:9
python.Test.TestSetup.logger
logger
Definition: Tools/WorkflowTestRunner/python/Test.py:19
python.Test.WorkflowCheck
Definition: Tools/WorkflowTestRunner/python/Test.py:84
python.Test.TestSetup.custom_threads
custom_threads
Definition: Tools/WorkflowTestRunner/python/Test.py:33
plot_material.mkdir
def mkdir(path, recursive=True)
Definition: plot_material.py:16
python.Test.WorkflowCheck.setup
setup
Definition: Tools/WorkflowTestRunner/python/Test.py:88
python.Helpers.list_changed_packages
None list_changed_packages(Logger logger, no_setup=False)
Definition: Tools/WorkflowTestRunner/python/Helpers.py:41
python.Test.TestSetup.validation_run_path
validation_run_path
Definition: Tools/WorkflowTestRunner/python/Test.py:20
python.Test.TestSetup
Definition: Tools/WorkflowTestRunner/python/Test.py:15
python.Test.TestSetup.detailed_comparison
detailed_comparison
Definition: Tools/WorkflowTestRunner/python/Test.py:34
python.Test.TestSetup.checks_only
checks_only
Definition: Tools/WorkflowTestRunner/python/Test.py:27
python.Test.TestSetup.disable_release_setup
disable_release_setup
Definition: Tools/WorkflowTestRunner/python/Test.py:24
python.Test.WorkflowTest.skip_performance_checks
skip_performance_checks
Definition: Tools/WorkflowTestRunner/python/Test.py:152
python.Test.TestSetup.release_ID
release_ID
Definition: Tools/WorkflowTestRunner/python/Test.py:30
python.Test.WorkflowTest.output_checks
output_checks
Definition: Tools/WorkflowTestRunner/python/Test.py:146
python.Test.TestSetup.__init__
None __init__(self, Logger logger)
Definition: Tools/WorkflowTestRunner/python/Test.py:18
python.Test.TestSetup.validation_only
validation_only
Definition: Tools/WorkflowTestRunner/python/Test.py:25
python.Test.WorkflowType.__str__
def __str__(self)
Definition: Tools/WorkflowTestRunner/python/Test.py:80
python.Test.WorkflowRun.__str__
def __str__(self)
Definition: Tools/WorkflowTestRunner/python/Test.py:61
python.Test.WorkflowCheck.logger
logger
Definition: Tools/WorkflowTestRunner/python/Test.py:89
python.Test.TestSetup.release_reference
release_reference
Definition: Tools/WorkflowTestRunner/python/Test.py:28
python.Test.WorkflowTest.digest_checks
digest_checks
Definition: Tools/WorkflowTestRunner/python/Test.py:149
python.Test.WorkflowType
Definition: Tools/WorkflowTestRunner/python/Test.py:65
python.Test.TestSetup.run_only
run_only
Definition: Tools/WorkflowTestRunner/python/Test.py:26
python.Test.TestSetup.unique_ID
unique_ID
Definition: Tools/WorkflowTestRunner/python/Test.py:23
python.Test.TestSetup.diff_rules_path
diff_rules_path
Definition: Tools/WorkflowTestRunner/python/Test.py:22
python.Test.WorkflowTest.run_checks
bool run_checks(self, List[WorkflowCheck] performance_checks)
Definition: Tools/WorkflowTestRunner/python/Test.py:198
python.Test.WorkflowRun
Definition: Tools/WorkflowTestRunner/python/Test.py:56
python.Test.WorkflowTest.tag
tag
Definition: Tools/WorkflowTestRunner/python/Test.py:137
python.Test.TestSetup.disable_output_checks
disable_output_checks
Definition: Tools/WorkflowTestRunner/python/Test.py:32
python.Test.WorkflowTest.run_validation
None run_validation(self)
Definition: Tools/WorkflowTestRunner/python/Test.py:176
python.Test.TestSetup.parallel_execution
parallel_execution
Definition: Tools/WorkflowTestRunner/python/Test.py:31
python.Test.TestSetup.release_validation
release_validation
Definition: Tools/WorkflowTestRunner/python/Test.py:29
python.Test.WorkflowTest
Definition: Tools/WorkflowTestRunner/python/Test.py:129
python.Test.WorkflowTest.run_reference
None run_reference(self)
Definition: Tools/WorkflowTestRunner/python/Test.py:161
python.Test.WorkflowCheck.reference_file
Optional[Path] reference_file(self, "WorkflowTest" test, str file_name)
Definition: Tools/WorkflowTestRunner/python/Test.py:91
python.Test.TestSetup.reference_run_path
reference_run_path
Definition: Tools/WorkflowTestRunner/python/Test.py:21
python.Test.WorkflowTest.__init__
None __init__(self, str ID, WorkflowRun run, WorkflowType type, List[str] steps, TestSetup setup)
Definition: Tools/WorkflowTestRunner/python/Test.py:132
python.Test.WorkflowTest.ID
ID
Definition: Tools/WorkflowTestRunner/python/Test.py:134
python.Test.WorkflowTest.logger
logger
Definition: Tools/WorkflowTestRunner/python/Test.py:157
str
Definition: BTagTrackIpAccessor.cxx:11
python.Test.WorkflowTest.type
type
Definition: Tools/WorkflowTestRunner/python/Test.py:155
error
Definition: IImpactPoint3dEstimator.h:70
python.Test.TestSetup.setup_release
None setup_release(self, reference=None, validation=None)
Definition: Tools/WorkflowTestRunner/python/Test.py:36
python.Test.WorkflowTest.setup
setup
Definition: Tools/WorkflowTestRunner/python/Test.py:156
python.Test.WorkflowCheck.__init__
None __init__(self, TestSetup setup)
Definition: Tools/WorkflowTestRunner/python/Test.py:87
python.Test.WorkflowTest.steps
steps
Definition: Tools/WorkflowTestRunner/python/Test.py:140
python.Test.WorkflowTest.run
run
Definition: Tools/WorkflowTestRunner/python/Test.py:154