runTrigART Namespace Reference

Functions
def	minimal_pattern (package)
def	get_parser ()
def	get_patterns (args)
def	analyse_result_python (test_results)
def	analyse_result_shell (test_results)
def	analyse_results (all_test_results)
def	print_summary (all_test_results, failed_tests)
def	print_failed_ref_comp (all_test_results, failed_tests)
def	prep_dirs (topdir, scripts)
def	main ()

Function Documentation

analyse_result_python()

def runTrigART.analyse_result_python

(

test_results

)

Analyses test results based on the final exit code, which determines
the status for ART tests based on TrigValSteering python framework

Definition at line 86 of file runTrigART.py.

def analyse_result_python(test_results):
    '''Analyses test results based on the final exit code, which determines
    the status for ART tests based on TrigValSteering python framework'''
    result_string = ''
    for step in test_results['result']:
        result_string += "%s: %d, " % (step['name'], step['result'])
    result_string += "exit: %d" % test_results['exit_code']
    is_success = (test_results['exit_code'] == 0)
    return result_string, is_success
 
 

analyse_result_shell()

def runTrigART.analyse_result_shell

(

test_results

)

Analyses test results based on individual hard-coded step names, as
the final status cannot be unambiguously determined in the old shell-based
ART tests not using the TrigValSteering framework

Definition at line 97 of file runTrigART.py.

def analyse_result_shell(test_results):
    '''Analyses test results based on individual hard-coded step names, as
    the final status cannot be unambiguously determined in the old shell-based
    ART tests not using the TrigValSteering framework'''
    result_string = ''
    steps_to_ignore = ['RootComp', 'MessageCount']
    is_success = True
    for step in test_results['result']:
        result_string += "%s: %d, " % (step['name'], step['result'])
        if step['name'] not in steps_to_ignore and step['result'] != 0:
            is_success = False
    result_string += "exit: %d" % test_results['exit_code']
    return result_string, is_success
 
 

analyse_results()

def runTrigART.analyse_results

(

all_test_results

)

Prints a summary table of all results and returns two lists. One includes names of failed tests,
the other names of tests in which only the RootComp step failed. If only RootComp fails, the test is
not added to the first list, as we currently do not enforce updating RootComp references on every change.

Definition at line 112 of file runTrigART.py.

def analyse_results(all_test_results):
    '''Prints a summary table of all results and returns two lists. One includes names of failed tests,
    the other names of tests in which only the RootComp step failed. If only RootComp fails, the test is
    not added to the first list, as we currently do not enforce updating RootComp references on every change.'''
    failed_tests = []
    table = {}  # test name : results
    for test_name in all_test_results.keys():
        if test_name.endswith('.py'):
            result_string, is_success = analyse_result_python(all_test_results[test_name])
        else:
            result_string, is_success = analyse_result_shell(all_test_results[test_name])
        table[test_name] = result_string
        if not is_success:
            failed_tests.append(test_name)
    max_len_col1 = len(max(table.keys(), key=len))
    max_len_col2 = len(max(table.values(), key=len))
    logging.info('-'*(max_len_col1+max_len_col2+7))
    for k, v in table.items():
        logging.info('| {col1:<{width1}} | {col2:<{width2}} |'.format(
            col1=k, width1=max_len_col1,
            col2=v, width2=max_len_col2))
    logging.info('-'*(max_len_col1+max_len_col2+7))
    return failed_tests
 
 

get_parser()

def runTrigART.get_parser

(

)

Definition at line 29 of file runTrigART.py.

def get_parser():
    packages = ['TriggerTest', 'TrigAnalysisTest', 'TrigP1Test', 'ALL']
    parser = argparse.ArgumentParser(usage='%(prog)s [options] [PackageName]')
    parser.add_argument('package',
                        metavar='PackageName',
                        default='ALL',
                        nargs='?',
                        help='Name of the package from which to run ART tests. Options are: %(choices)s.'
                             ' If no name is provided, %(default)s is used.',
                        choices=packages)
    parser.add_argument('-m', '--minimal',
                        action='store_true',
                        help='Run a small pre-defined set of tests for basic verification')
    parser.add_argument('-n', '--testName',
                        metavar='pattern',
                        help='Run only tests with a given pattern in the name')
    parser.add_argument('-r', '--rerun-failed',
                        action='store_true',
                        help='Run tests that failed previously')
    parser.add_argument('-s', '--summary',
                        action='store_true',
                        help='Print summary of previously run tests')
    parser.add_argument('-t', '--artType',
                        metavar='type',
                        default='build',
                        choices=['build', 'grid'],
                        help='Run tests with the given art-type: build (default) or grid')
    parser.add_argument('-j', '--maxJobs',
                        metavar='N',
                        type=int,
                        default=1,
                        help='Run up to N tests in parallel (actual number may be limited by ART based on available CPUs and memory)')
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        help='Increase output verbosity')
    parser.add_argument('-d', '--dryRun',
                        action='store_true',
                        help='List tests which would be executed, but don\'t execute them')
 
    return parser
 
 

get_patterns()

def runTrigART.get_patterns

(

args

)

Definition at line 71 of file runTrigART.py.

def get_patterns(args):
    patterns = ['^test_', '(.sh|.py)$', package_prefix(args.package)]
    if args.testName:
        patterns.append(args.testName)
    if args.artType == 'grid':
        patterns.append('_grid.')
    else:
        patterns.append('_build.')
    if args.minimal:
        patterns.append(minimal_pattern(args.package))
 
    logging.debug("Searching for scripts with the following patterns: %s", patterns)
    return patterns
 
 

main()

def runTrigART.main

(

)

Definition at line 192 of file runTrigART.py.

def main():
    args = get_parser().parse_args()
    logging.basicConfig(stream=sys.stdout,
                        format='%(levelname)-8s %(message)s',
                        level=logging.DEBUG if args.verbose else logging.INFO)
 
    topdir = 'runTrigART'
    statusfile = 'results/{:s}-status.json'.format(topdir)
 
    # Load previous results
    if args.rerun_failed or args.summary:
        status_data = json.load(open(os.path.join(topdir,statusfile)))
        all_test_results = status_data[topdir]
        failed_tests = analyse_results(all_test_results)
        if args.summary:
            print_summary(all_test_results, failed_tests)
            return 0
 
    if args.rerun_failed:
        scripts = find_scripts(get_patterns(args)+['|'.join(failed_tests)])
    else:
        scripts = find_scripts(get_patterns(args))
 
    logging.info("The following %d tests will be executed: ", len(scripts))
    for filename in scripts:
        logging.info("    %s", os.path.basename(filename))
 
    if len(scripts) > 5*args.maxJobs:
        if args.maxJobs == 1:
            logging.warning("You are running %d tests in sequence. This may take "
                            "a long time, consider using -j N option.", len(scripts))
        else:
            logging.warning("You are running %d tests with %d parallel jobs. "
                            "This may take a long time.", len(scripts), args.maxJobs)
 
    if args.dryRun:
        return 0
 
    success = True
    with remember_cwd():
        prep_dirs(topdir, scripts)
        os.chdir(topdir)
        for script_path in scripts:
            target = 'test/' + os.path.basename(script_path)
            os.symlink(script_path, target)
 
        # Set up and run ART
        commands = [
            'export ATLAS_LOCAL_ROOT_BASE="${ATLAS_LOCAL_ROOT_BASE:-/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase}"',
            'source "${ATLAS_LOCAL_ROOT_BASE}"/user/atlasLocalSetup.sh --quiet',
            'lsetup -q art']
        art_cmd = 'art.py run --run-all-tests --max-jobs={:d} {:s} . results'.format(args.maxJobs, '-v' if args.verbose else '-q')
        commands.append(art_cmd)
        cmd = ' && '.join(commands)
        logging.info("Executing ART command: %s", art_cmd)
        subprocess.call(cmd, shell=True)
        logging.info("ART finished, analysing the results\n")
 
        # Read the result summary from JSON
        if not os.path.isfile(statusfile):
            logging.error("ART status.json file is missing - likely the ART runner failed!")
            exit(1)
        with open(statusfile, 'r') as f:
            status_data = json.load(f)
            all_test_results = status_data[topdir]
            if len(all_test_results) != len(scripts):
                logging.warning("Selected %d tests but ART executed only %d. Please check why some tests did not run!")
            failed_tests = analyse_results(all_test_results)
            print_summary(all_test_results, failed_tests)
            print_failed_ref_comp(all_test_results, failed_tests)
            if len(failed_tests) > 0:
                success = False
 
    if not success:
        exit(1)
    else:
        exit(0)
 
 

minimal_pattern()

def runTrigART.minimal_pattern

(

package

)

Definition at line 16 of file runTrigART.py.

def minimal_pattern(package):
    dict = {'TriggerTest':      'test_trig_data_v1Dev_build',
            'TrigP1Test':       'test_trigP1_v1Dev_decodeBS_build',
            'TrigAnalysisTest': 'test_trigAna_RDOtoRDOTrig_v1Dev_build'}
    if package == 'ALL':
        return '({})'.format('|'.join([v for v in dict.values() if v]))
    elif package in dict and dict[package] is not None:
        return dict[package]
    else:
        logging.error("Minimal set of tests for %s is not defined.", package)
        exit(1)
 
 

prep_dirs()

def runTrigART.prep_dirs	(		topdir,
			scripts
	)

Creates test result structure if missing, if present clears the area only for the tests to be run

Definition at line 180 of file runTrigART.py.

def prep_dirs(topdir, scripts):
    """ Creates test result structure if missing, if present clears the area only for the tests to be run"""
 
    shutil.rmtree(topdir+'/test', ignore_errors=True)
    os.makedirs(topdir+'/test', exist_ok=True)
 
    # clear results dir
    for script in scripts:
        toerase = topdir+'/results/runTrigART/'+os.path.splitext(os.path.basename(script))[0]
        shutil.rmtree(toerase, ignore_errors=True)
 
 

print_failed_ref_comp()

def runTrigART.print_failed_ref_comp	(		all_test_results,
			failed_tests
	)

Definition at line 154 of file runTrigART.py.

def print_failed_ref_comp(all_test_results, failed_tests):
    if len(failed_tests) == 0:
        return
 
    step_name = 'CountRefComp'
    for test_name in failed_tests:
        athena_failed = False
        refcomp_failed = False
        test_results = all_test_results[test_name]
        for step in test_results['result']:
            if step['name'] in ['athena', 'athenaHLT', 'Reco_tf', 'CheckLog'] and step['result'] != 0:
                athena_failed = True
            if step['name'] == step_name and step['result'] != 0:
                refcomp_failed = True
 
        # Print only if athena didn't fail, as a failed job will always give wrong counts
        if refcomp_failed and not athena_failed:
            log_path = 'results/runTrigART/{:s}/{:s}.log'.format(os.path.splitext(test_name)[0], step_name)
            if not os.path.isfile(log_path):
                continue
            logging.info("==================================================")
            logging.info('Printing output of failed CountRefComp for %s', test_name)
            subprocess.call('cat ' + log_path, shell=True)
            logging.info("==================================================")
 
 

print_summary()

def runTrigART.print_summary	(		all_test_results,
			failed_tests
	)

Definition at line 137 of file runTrigART.py.

def print_summary(all_test_results, failed_tests):
    if len(failed_tests) > 0:
        logging.info(
            "%d tests succeeded out of %d executed",
            len(all_test_results)-len(failed_tests),
            len(all_test_results))
        logging.error("==================================================")
        logging.error("The following %d tests failed:", len(failed_tests))
        for test_name in failed_tests:
            logging.error("    %s", test_name)
        logging.error("==================================================")
    else:
        logging.info("==================================================")
        logging.info("All %d executed tests succeeded", len(all_test_results))
        logging.info("==================================================")