ATLAS Offline Software
Loading...
Searching...
No Matches
python.Checks.AODDigestCheck Class Reference
Inheritance diagram for python.Checks.AODDigestCheck:
Collaboration diagram for python.Checks.AODDigestCheck:

Public Member Functions

None __init__ (self, TestSetup setup, int max_events=-1)
bool run (self, WorkflowTest test)
Optional[Path] reference_file (self, "WorkflowTest" test, str file_name)

Public Attributes

 max_events = str(max_events)
 setup = setup
 logger = setup.logger

Detailed Description

Run AOD Digest Check.

Definition at line 382 of file Checks.py.

Constructor & Destructor Documentation

◆ __init__()

None python.Checks.AODDigestCheck.__init__ ( self,
TestSetup setup,
int max_events = -1 )

Definition at line 385 of file Checks.py.

385 def __init__(self, setup: TestSetup, max_events: int = -1) -> None:
386 super().__init__(setup)
387 self.max_events = str(max_events)
388

Member Function Documentation

◆ reference_file()

Optional[Path] python.Test.WorkflowCheck.reference_file ( self,
"WorkflowTest" test,
str file_name )
inherited

Definition at line 93 of file Tools/WorkflowTestRunner/python/Test.py.

93 def reference_file(self, test: "WorkflowTest", file_name: str) -> Optional[Path]:
94 reference_path: Path = test.reference_path
95 reference_file = reference_path / file_name
96
97 # Read references from CVMFS
98 if self.setup.validation_only:
99 # Resolve the subfolder first. Results are stored like: main_folder/branch/test/version/.
100 reference_revision = references_map[f"{test.ID}"]
101 cvmfs_path = Path(references_CVMFS_path)
102 rel_path = Path(self.setup.release_ID) / test.ID / reference_revision
103 reference_path = cvmfs_path / rel_path
104 reference_file = reference_path / file_name
105
106 if not reference_path.exists():
107 self.logger.error(f"CVMFS reference location {reference_path} does not exist!")
108 return None
109
110 if references_override_url is not None:
111 import requests
112
113 url = references_override_url
114 if not url.endswith("/"): url += "/"
115 url += str(rel_path / file_name)
116 self.logger.info("Checking for reference override at %s", url)
117 if requests.head(url).ok: # file exists at url
118 reference_file = Path.cwd() / f"reference_{file_name}"
119 self.logger.info("Downloading reference from %s to %s", url, reference_file)
120 r = requests.get(url, stream=True)
121 with reference_file.open('wb') as f:
122 for chunk in r.iter_content(chunk_size=1024):
123 if chunk: # filter out keep-alive new chunks
124 f.write(chunk)
125 else:
126 self.logger.info("No reference override found")
127
128 return reference_file
129
130

◆ run()

bool python.Checks.AODDigestCheck.run ( self,
WorkflowTest test )

Definition at line 389 of file Checks.py.

389 def run(self, test: WorkflowTest) -> bool:
390 self.logger.info("---------------------------------------------------------------------------------------")
391 self.logger.info(f"Running {test.ID} AOD digest")
392
393 file_name = "myAOD.pool.root"
394 output_name = f"{test.ID}_AOD_digest.txt"
395
396 validation_file = test.validation_path / file_name
397 validation_output = test.validation_path / output_name
398 validation_log_file = test.validation_path / f"AODdigest-{test.ID}.log"
399 validation_command = f"xAODDigest.py {validation_file} {validation_output} > {validation_log_file} 2>&1"
400
401 output_val, error_val = subprocess.Popen(["/bin/bash", "-c", validation_command], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
402 output_val, error_val = output_val.decode("utf-8"), error_val.decode("utf-8")
403 if error_val:
404 self.logger.error(f"Something went wrong with the digest calculation for test {test.ID}:")
405 self.logger.error(error_val)
406
407 # Read references
408 if self.setup.validation_only:
409 # try to get the reference
410 reference_path = test.validation_path
411 reference_output_name = f"{test.ID}_AOD_digest.ref"
412 reference_output = reference_path / reference_output_name
413 subprocess.Popen(["/bin/bash", "-c", f"cd {reference_path}; get_files -remove -data {reference_output_name}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
414 if not reference_output.exists():
415 self.logger.info(f"No reference file '{reference_output_name}' to compare the digest with. Printing the full digest:")
416 with validation_output.open() as f:
417 for line in f:
418 self.logger.print(f" {line.strip()}")
419 return True
420 else:
421 reference_path = test.reference_path
422 reference_output = reference_path / output_name
423 reference_file = reference_path / file_name
424 reference_log_file = test.reference_path / f"AODdigest-{test.ID}.log"
425
426 reference_command = f"xAODDigest.py {reference_file} {reference_output} > {reference_log_file} 2>&1"
427 subprocess.Popen(["/bin/bash", "-c", reference_command], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
428
429 # Compute the diff
430 diff_output, diff_error = subprocess.Popen(["/bin/bash", "-c", f"diff {reference_output} {validation_output}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
431 diff_output, diff_error = diff_output.decode("utf-8"), diff_error.decode("utf-8")
432
433 result = False
434 if not diff_output and not diff_error:
435 self.logger.info("Passed!\n")
436 result = True
437 else:
438 # print CI helper directly to avoid logger decorations
439 if self.setup.disable_release_setup:
440 self.logger.print(f"ATLAS-CI-ADD-LABEL: {test.run.value}-{test.type.value}-output-changed")
441 self.logger.print("")
442
443 self.logger.error(f"Your change breaks the digest in test {test.ID}.")
444 self.logger.error("Please make sure this has been discussed in the correct meeting (RIG or Simulation) meeting and approved by the relevant experts.")
445 if self.setup.validation_only:
446 self.logger.error(f"The output '{output_name}' (>) differs from the reference '{reference_output_name}' (<):")
447 else:
448 self.logger.error(f"The output '{validation_output}' (>) differs from the reference '{reference_output}' (<):")
449 if diff_output:
450 with reference_output.open() as file:
451 self.logger.print(file.readline())
452 self.logger.print(diff_output)
453 if diff_error:
454 self.logger.print(diff_error)
455 self.logger.info("-----------------------------------------------------\n")
456
457 return result
458
459
void print(char *figname, TCanvas *c1)
int run(int argc, char *argv[])

Member Data Documentation

◆ logger

python.Test.WorkflowCheck.logger = setup.logger
inherited

Definition at line 91 of file Tools/WorkflowTestRunner/python/Test.py.

◆ max_events

python.Checks.AODDigestCheck.max_events = str(max_events)

Definition at line 387 of file Checks.py.

◆ setup

python.Test.WorkflowCheck.setup = setup
inherited

Definition at line 90 of file Tools/WorkflowTestRunner/python/Test.py.


The documentation for this class was generated from the following file: