Inheritance diagram for python.Checks.AODContentCheck:

Collaboration diagram for python.Checks.AODContentCheck:

Public Member Functions
bool	run (self, WorkflowTest test)

Optional[Path]	reference_file (self, "WorkflowTest" test, str file_name)

Public Attributes
	setup

	logger

Detailed Description

Run AOD Content Check.

Definition at line 303 of file Checks.py.

Member Function Documentation

◆ reference_file()

Optional[Path] python.Test.WorkflowCheck.reference_file	(		self,
		"WorkflowTest"	test,
		str	file_name
	)

inherited

Definition at line 91 of file Tools/WorkflowTestRunner/python/Test.py.

     def reference_file(self, test: "WorkflowTest", file_name: str) -> Optional[Path]:
         reference_path: Path = test.reference_path
         reference_file = reference_path / file_name
  
         # Read references from CVMFS
         if self.setup.validation_only:
             # Resolve the subfolder first. Results are stored like: main_folder/branch/test/version/.
             reference_revision = references_map[f"{test.ID}"]
             cvmfs_path = Path(references_CVMFS_path)
             rel_path = Path(self.setup.release_ID) / test.ID / reference_revision
             reference_path = cvmfs_path / rel_path
             reference_file = reference_path / file_name
  
             if not reference_path.exists():
                 self.logger.error(f"CVMFS reference location {reference_path} does not exist!")
                 return None
  
             if references_override_url is not None:
                 import requests
  
                 url = references_override_url
                 if not url.endswith("/"): url += "/"
                 url += str(rel_path / file_name)
                 self.logger.info("Checking for reference override at %s", url)
                 if requests.head(url).ok: # file exists at url
                     reference_file = Path.cwd() / f"reference_{file_name}"
                     self.logger.info("Downloading reference from %s to %s", url, reference_file)
                     r = requests.get(url, stream=True)
                     with reference_file.open('wb') as f:
                         for chunk in r.iter_content(chunk_size=1024):
                             if chunk: # filter out keep-alive new chunks
                                 f.write(chunk)
                 else:
                     self.logger.info("No reference override found")
  
         return reference_file
  
  

◆ run()

bool python.Checks.AODContentCheck.run	(		self,
		WorkflowTest	test
	)

Definition at line 306 of file Checks.py.

     def run(self, test: WorkflowTest) -> bool:
         self.logger.info("---------------------------------------------------------------------------------------")
         self.logger.info(f"Running {test.ID} AOD content check")
  
         file_name = "myAOD.pool.root"
         output_name = f"{test.ID}_AOD_content.txt"
  
         validation_file = test.validation_path / file_name
         validation_output = test.validation_path / output_name
         validation_command = f"acmd.py chk-file {validation_file} | awk '/---/{{flag=1;next}}/===/{{flag=0}}flag' | awk '{{print $10}}' | LC_ALL=C sort | uniq > {validation_output}"
  
         output_val, error_val = subprocess.Popen(["/bin/bash", "-c", validation_command], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
         output_val, error_val = output_val.decode("utf-8"), error_val.decode("utf-8")
         if error_val:
             self.logger.error(f"Something went wrong with retrieving the content for test {test.ID}:")
             self.logger.error(error_val)
  
         # Read references
         if self.setup.validation_only:
             # try to get the reference
             reference_path = test.validation_path
             reference_output_name = f"{test.ID}_AOD_content.ref"
             reference_output = reference_path / reference_output_name
             subprocess.Popen(["/bin/bash", "-c", f"cd {reference_path}; get_files -remove -data {reference_output_name}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
             if not reference_output.exists():
                 self.logger.info(f"No reference file '{reference_output_name}' to compare the content with.")
                 return True
         else:
             reference_path = test.reference_path
             reference_output = reference_path / output_name
             reference_file = reference_path / file_name
  
             reference_command = f"acmd.py chk-file {reference_file} | awk '/---/{{flag=1;next}}/===/{{flag=0}}flag' | awk '{{print $10}}' | LC_ALL=C sort | uniq > {reference_output}"
             subprocess.Popen(["/bin/bash", "-c", reference_command], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
  
         # Remove HLT containers in some cases
         extra_diff_args = ""
         if test.type == WorkflowType.MCReco or test.type == WorkflowType.MCPileUpReco:
             extra_diff_args = "-I '^HLT' -I '^LVL1' -I '^L1'"
  
         # Compute the diff
         diff_output, diff_error = subprocess.Popen(["/bin/bash", "-c", f"diff {extra_diff_args} {reference_output} {validation_output}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
         diff_output, diff_error = diff_output.decode("utf-8"), diff_error.decode("utf-8")
  
         result = False
         if not diff_output and not diff_error:
             self.logger.info("Passed!\n")
             result = True
         else:
             # print CI helper directly to avoid logger decorations
             if self.setup.disable_release_setup:
                 self.logger.print(f"ATLAS-CI-ADD-LABEL: {test.run.value}-{test.type.value}-output-changed")
                 self.logger.print("")
  
             self.logger.error(f"Your change modifies the output in test {test.ID}.")
             self.logger.error("Please make sure this has been discussed in the correct meeting (RIG or Simulation) meeting and approved by the relevant experts.")
             if self.setup.validation_only:
                 self.logger.error(f"The output '{output_name}' (>) differs from the reference '{reference_output_name}' (<):")
             else:
                 self.logger.error(f"The output '{validation_output}' (>) differs from the reference '{reference_output}' (<):")
             if diff_output:
                 self.logger.print("")
                 self.logger.print(diff_output)
             if diff_error:
                 self.logger.print(diff_error)
             self.logger.info("-----------------------------------------------------\n")
  
         return result
  
  