6 Definitions of post-exec check steps in Trigger ART tests
15 from TrigValTools.TrigValSteering.Step
import Step, get_step_from_list
16 from TrigValTools.TrigValSteering.ExecStep
import ExecStep
17 from TrigValTools.TrigValSteering.Common
import art_input_eos, art_input_cvmfs, running_in_CI
20 '''Base class for steps comparing a file to a reference'''
23 super(RefComparisonStep, self).
__init__(name)
31 self.misconfig_abort(
'Both options "reference" and "ref_test_name" used. Use at most one of them.')
39 return super(RefComparisonStep, self).
configure(test)
42 return super(RefComparisonStep, self).
configure(test)
44 full_path = subprocess.check_output(
'find_data.py {}'.
format(self.
reference), shell=
True).
decode(
'utf-8').strip()
45 if os.path.isfile(full_path):
46 self.log.
debug(
'%s using reference %s', self.name, full_path)
48 return super(RefComparisonStep, self).
configure(test)
51 '%s failed to find reference %s - wrong path?',
53 return super(RefComparisonStep, self).
configure(test)
56 self.misconfig_abort(
'input_file not specified')
58 branch = os.environ.get(
'AtlasBuildBranch')
60 branch = branch.split(
'--')[0]
62 branch = os.environ.get(
'gitlabTargetBranch')
64 jobName = os.environ.get(
'JOB_NAME')
66 branch = jobName.split(
'_')[0].
split(
'--')[0]
68 msg =
'Cannot determine the branch name, all variables are empty: AtlasBuildBranch, gitlabTargetBranch, JOB_NAME'
70 self.misconfig_abort(msg)
73 branch =
'UNKNOWN_BRANCH'
75 sub_path =
'{}/ref/{}/test_{}/'.
format(
77 ref_eos = art_input_eos + sub_path + self.
input_file
78 ref_cvmfs = art_input_cvmfs + sub_path + self.
input_file
79 if os.path.isfile(ref_eos)
and os.access(ref_eos, os.R_OK):
80 self.log.
debug(
'%s using reference from EOS: %s',
83 elif os.path.isfile(ref_cvmfs)
and os.access(ref_cvmfs, os.R_OK):
84 self.log.
debug(
'%s using reference from CVMFS: %s',
88 self.log.warning(
'%s failed to find reference %s in %s or %s',
90 art_input_eos, art_input_cvmfs)
93 return super(RefComparisonStep, self).
configure(test)
97 '''Base class for steps executed only if the input file exists'''
100 super(InputDependentStep, self).
__init__(name)
103 def run(self, dry_run=False):
105 self.log.
error(
'%s misconfiguration - no input file specified',
108 if self.auto_report_result:
110 return self.
result,
'# (internal) {} -> failed'.
format(self.name)
112 if not dry_run
and not os.path.isfile(self.
input_file):
113 self.log.
debug(
'Skipping %s because %s does not exist',
116 return self.
result,
'# (internal) {} -> skipped'.
format(self.name)
118 return super(InputDependentStep, self).
run(dry_run)
122 '''Merge several log files into one for post-processing'''
125 super(LogMergeStep, self).
__init__(name)
134 for step
in test.exec_steps:
138 self.misconfig_abort(
139 'output log name %s is same as one of the input log names.'
140 ' This will lead to infinite loop, aborting.', self.
merged_name)
141 super(LogMergeStep, self).
configure(test)
145 files = os.listdir(
'.')
147 match_files =
filter(r.match, files)
148 for f
in match_files:
155 if not os.path.isfile(log_name):
157 self.log.warning(
'Cannot open %s', log_name)
159 '### WARNING Missing {} ###\n'.
format(log_name))
161 with open(log_name, encoding=
'utf-8')
as log_file:
162 merged_file.write(
'### {} ###\n'.
format(log_name))
164 if "Derivation" in log_name:
165 for line
in log_file:
166 merged_file.write(line.replace(
'Selected dynamic Aux',
'Selected Dynamic Aux'))
168 for line
in log_file:
169 merged_file.write(line)
172 self.log.
error(
'%s merging failed due to OSError: %s',
173 self.name, e.strerror)
176 def run(self, dry_run=False):
179 self.
log_files.sort(key=
lambda f : os.path.getmtime(f)
if os.path.isfile(f)
else 0)
180 self.log.
info(
'Running %s merging logs %s into %s',
191 Merge root files with hadd. Parameters are:
192 input_file - file(s) to be merged
193 merged_file - output file name
194 rename_suffix - if merged_file exists, it is renamed by adding this suffix
198 super(RootMergeStep, self).
__init__(name)
206 super(RootMergeStep, self).
configure(test)
208 def run(self, dry_run=False):
214 if new_name
in file_list_to_check:
215 file_list_to_check.remove(new_name)
217 self.log.
debug(
'%s checking if the input files exist: %s', self.name,
str(file_list_to_check))
219 for file_name
in file_list_to_check:
220 if len(glob.glob(file_name)) < 1:
221 self.log.warning(
'%s: file %s requested to be merged but does not exist', self.name, file_name)
224 return super(RootMergeStep, self).
run(dry_run)
228 '''Compress a large log file'''
246 '''Execute CheckLog looking for errors or warnings in a log file'''
249 super(CheckLogStep, self).
__init__(name)
255 self.
args =
'--showexcludestats'
263 if test.package_name ==
'TrigP1Test':
265 elif test.package_name ==
'TrigValTools':
270 if len(test.exec_steps) == 1:
271 self.
log_file = test.exec_steps[0].name+
'.log'
275 self.
args +=
' --errors'
277 self.
args +=
' --warnings'
281 self.
output_stream = Step.OutputStream.FILE_AND_STDOUT
if errors_only
else Step.OutputStream.FILE_ONLY
289 super(CheckLogStep, self).
configure(test)
293 '''Execute RegTest comparing a log file against a reference'''
296 super(RegTestStep, self).
__init__(name)
305 RefComparisonStep.configure(self, test)
307 Step.configure(self, test)
311 if not os.path.isfile(log_file):
312 self.log.
error(
'%s input file %s is missing', self.name, log_file)
314 with open(log_file, encoding=
'utf-8')
as f_in:
315 matches = re.findall(
'({}.*).*$'.
format(self.
regex),
316 f_in.read(), re.MULTILINE)
319 linestr =
str(line[0])
if type(line)
is tuple
else line
320 f_out.write(linestr+
'\n')
326 new_name = os.path.basename(self.
reference) +
'.new'
328 new_name = os.path.basename(self.
input_file) +
'.new'
332 self.log.warning(
'Failed to rename %s to %s',
335 def run(self, dry_run=False):
337 self.log.
error(
'%s failed in prepare_inputs()', self.name)
341 return self.
result,
'# (internal) {} -> failed'.
format(self.name)
343 self.log.
error(
'Missing reference for %s', self.name)
349 return self.
result,
'# (internal) {} -> failed'.
format(self.name)
350 retcode, cmd = super(RegTestStep, self).
run(dry_run)
357 '''Execute RootComp comparing histograms against a reference'''
360 super(RootCompStep, self).
__init__(name)
366 RefComparisonStep.configure(self, test)
369 self.args +=
' --noRoot --noPS'
371 Step.configure(self, test)
373 def run(self, dry_run=False):
377 'Skipping %s because both reference and input are missing',
380 return self.
result,
'# (internal) {} -> skipped'.
format(self.name)
382 self.log.
error(
'Missing reference for %s', self.name)
386 return self.
result,
'# (internal) {} -> failed'.
format(self.name)
387 retcode, cmd = super(RootCompStep, self).
run(dry_run)
392 '''Execute the PerfMon ntuple post-processing'''
395 super(PerfMonStep, self).
__init__(name)
402 num_athenaHLT_steps =
sum([1
for step
in test.exec_steps
if step.type ==
'athenaHLT'])
403 if num_athenaHLT_steps > 0:
404 self.
input_file =
'athenaHLT_workers/athenaHLT-01/ntuple.pmon.gz'
412 '''Copy the last N lines of a log file into a separate file'''
415 super(TailStep, self).
__init__(name)
424 split = os.path.splitext(self.
log_file)
435 '''Execute art.py download to get results from previous days'''
438 super(DownloadRefStep, self).
__init__(name)
453 super(DownloadRefStep, self).
configure(test)
457 '''Execute histSizes.py to count histograms in a ROOT file'''
460 super(HistCountStep, self).
__init__(name)
467 super(HistCountStep, self).
configure(test)
472 Execute chainDump.py to print trigger counts from histograms to text files
476 super(ChainDumpStep, self).
__init__(name)
483 super(ChainDumpStep, self).
configure(test)
488 Execute chainComp.py to compare counts from chainDump.py to a reference
492 super(ChainCompStep, self).
__init__(name)
503 RefComparisonStep.configure(self, test)
508 Step.configure(self, test)
512 '''Execute trig-test-json.py to create extra-results.json file'''
515 super(TrigTestJsonStep, self).
__init__(name)
521 Execute checkFile and checkxAOD for POOL files.
522 executable and input_file can have multiple comma-separated values
525 def __init__(self,name='CheckFile',input_file='AOD.pool.root'):
526 super(CheckFileStep, self).
__init__(name)
534 test_types = [step.type
for step
in test.exec_steps]
535 num_athenaHLT =
sum(1
for tt
in test_types
if tt ==
'athenaHLT')
536 if num_athenaHLT == len(test_types):
537 self.log.
debug(
'%s will be skipped because all exec steps use athenaHLT')
543 super(CheckFileStep, self).
configure(test)
545 def run(self, dry_run=False):
553 ex_base = ex.split(
'.')[0:-1]
555 ret, cmd = super(CheckFileStep, self).
run(dry_run)
556 ret_codes.append(ret)
562 if '(internal)' not in cmd:
563 merged_cmd += cmd+
'; '
564 if len(merged_cmd) == 0:
565 merged_cmd =
'# (internal) {} -> skipped'.
format(self.name)
568 return max(ret_codes), merged_cmd
573 Check if all counts are zero.
574 input_file can have multiple comma-separated values
578 super(ZeroCountsStep, self).
__init__(name)
588 if not os.path.isfile(input_file):
590 'Skipping %s for %s because the file does not exist',
591 self.name, input_file)
594 with open(input_file, encoding=
'utf-8')
as f_in:
595 for line
in f_in.readlines():
596 split_line = line.split()
598 if int(split_line[-1]) != 0:
600 if lines_checked == 0:
601 self.log.
error(
'Failed to read counts from %s', input_file)
604 def run(self, dry_run=False):
612 cmd =
'# (internal) {} -> skipped'.
format(self.name)
615 self.log.
info(
'Running %s step', self.name)
622 '''Count messages printed inside event loop'''
625 super(MessageCountStep, self).
__init__(name)
627 self.
log_regex =
r'(athena\.(?!.*tail).*log$|athenaHLT:.*\.out$|^log\.(.*to.*|Derivation))'
629 self.
start_pattern =
r'(HltEventLoopMgr|AthenaHiveEventLoopMgr).*INFO Starting loop on events'
630 self.
end_pattern =
r'(HltEventLoopMgr.*INFO All events processed|AthenaHiveEventLoopMgr.*INFO.*Loop Finished)'
642 self.args +=
' --saveAll'
644 max_events = test.exec_steps[0].max_events
if isinstance(test.exec_steps[0], ExecStep)
else 0
655 super(MessageCountStep, self).
configure(test)
657 def run(self, dry_run=False):
658 files = os.listdir(
'.')
660 log_files = [f
for f
in filter(r.match, files)
if f
not in self.
skip_logs]
661 if not log_files
and not dry_run:
662 self.log.
error(
'%s found no log files matching the pattern %s', self.name, self.
log_regex)
666 return self.
result,
'# (internal) {} -> failed'.
format(self.name)
667 self.args +=
' ' +
' '.
join(log_files)
670 ret, cmd = super(MessageCountStep, self).
run(dry_run)
673 self.log.
error(
'%s failed', self.name)
679 for log_file
in log_files:
680 json_file =
'MessageCount.{:s}.json'.
format(log_file)
682 all_json_file =
'Messages.{:s}.json'.
format(log_file)
683 if not os.path.isfile(json_file):
684 self.log.warning(
'%s cannot open file %s', self.name, json_file)
685 with open(json_file)
as f:
686 summary = json.load(f)
688 if summary[level] > threshold:
691 '%s Number of %s messages %s in %s is higher than threshold %s',
692 self.name, level, summary[level], log_file, threshold)
694 self.log.
info(
'%s Printing all %s messages from %s', self.name, level, log_file)
695 with open(all_json_file)
as af:
696 all_msg = json.load(af)
697 for msg
in all_msg[level]:
707 Helper function checking whether a Step output_stream value
708 indicates that it will produce a log file
710 return step.output_stream == Step.OutputStream.FILE_ONLY
or \
711 step.output_stream == Step.OutputStream.FILE_AND_STDOUT
714 def default_check_steps(test, checkfile_input='AOD.pool.root,ESD.pool.root,RDO_TRIG.pool.root,DAOD_PHYS.DAOD.pool.root'):
716 Create the default list of check steps for a test. The configuration
717 depends on the package name and the type of exec steps (athena or
718 athenaHLT or transforms).
724 if len(test.exec_steps) == 1:
725 exec_step = test.exec_steps[0]
726 if exec_step.type ==
'athenaHLT' and produces_log(exec_step):
728 logmerge.merged_name =
'athena.log'
729 logmerge.log_files = [
'athenaHLT.log']
730 nforks = 1
if exec_step.forks
is None else exec_step.forks
731 for n
in range(1, 1+nforks):
732 logmerge.log_files.append(
'athenaHLT:{:02d}.out'.
format(n))
733 logmerge.log_files.append(
'athenaHLT:{:02d}.err'.
format(n))
734 check_steps.append(logmerge)
737 logmerge.merged_name =
'athena.log'
738 logmerge.log_files = []
739 for exec_step
in test.exec_steps:
742 logmerge.log_files.append(exec_step.get_log_file_name())
743 if exec_step.type ==
'athenaHLT':
744 logmerge.extra_log_regex =
'athenaHLT:.*(.out|.err)'
745 check_steps.append(logmerge)
748 if len(check_steps) > 0
and isinstance(check_steps[-1], LogMergeStep):
749 log_to_check = check_steps[-1].merged_name
750 log_to_zip = check_steps[-1].merged_name
753 reco_tf_steps = [step
for step
in test.exec_steps
if step.type
in [
'Reco_tf',
'Trig_reco_tf',
'Derivation_tf']]
754 if len(reco_tf_steps) > 0:
756 reco_tf_logmerge.warn_if_missing =
False
758 tf_names = [
'HITtoRDO',
'Overlay',
'RDOtoRDOTrigger',
'RAWtoESD',
'ESDtoAOD',
759 'PhysicsValidation',
'RAWtoALL',
760 'BSRDOtoRAW',
'DRAWCOSTtoNTUPCOST',
'AODtoNTUPRATE',
'Derivation',
'AODtoDAOD']
761 reco_tf_logmerge.log_files = [
'log.'+tf_name
for tf_name
in tf_names]
763 for step
in reco_tf_steps:
764 reco_tf_logmerge.log_files.append(step.get_log_file_name())
765 reco_tf_logmerge.merged_name =
'athena.merged.log'
766 log_to_zip = reco_tf_logmerge.merged_name
767 if log_to_check
is not None:
768 reco_tf_logmerge.log_files.append(log_to_check)
769 log_to_check = reco_tf_logmerge.merged_name
770 log_to_check = reco_tf_logmerge.merged_name
771 check_steps.append(reco_tf_logmerge)
774 num_athenaHLT_steps =
sum([1
for step
in test.exec_steps
if step.type ==
'athenaHLT'])
775 if num_athenaHLT_steps > 0:
777 histmerge.merged_file =
'expert-monitoring.root'
778 histmerge.input_file =
'athenaHLT_workers/*/expert-monitoring.root expert-monitoring-mother.root'
779 histmerge.rename_suffix =
'-mother'
780 check_steps.append(histmerge)
784 if log_to_check
is not None:
785 checklog.log_file = log_to_check
786 check_steps.append(checklog)
790 checkwarn.check_errors =
False
791 checkwarn.check_warnings =
True
792 if log_to_check
is not None:
793 checkwarn.log_file = log_to_check
794 check_steps.append(checkwarn)
798 for logmerge
in [step
for step
in check_steps
if isinstance(step, LogMergeStep)]:
799 msgcount.skip_logs.append(logmerge.merged_name)
800 check_steps.append(msgcount)
804 if log_to_check
is not None:
805 tail.log_file = log_to_check
806 check_steps.append(tail)
820 check_steps.append(
CheckFileStep(input_file=checkfile_input))
823 if log_to_zip
is not None:
825 zip_step.zip_input = log_to_zip
826 zip_step.zip_output = log_to_zip+
'.tar.gz'
827 check_steps.append(zip_step)
835 Insert step_to_add into step_list after the last step of type ref_type.
836 If the list has no steps of type ref_type, append step_to_add at the end of the list.
839 for index, step
in enumerate(step_list):
840 if isinstance(step, ref_type):
841 index_to_add = index+1
843 step_list.insert(index_to_add, step_to_add)
845 step_list.append(step_to_add)