check_log Namespace Reference

Functions
	get_parser ()
	main ()
	parseConfig (args)
	scanLogfile (args, logfile, ignorePattern=[])

Variables
dict	regexMap = {}
	builtins = dir(builtins)
list	builtinErrors = [b for b in builtins if 'Error' in b]
list	backtrace
list	fpeTracebackStart = [r'FPEAuditor.*INFO FPE stacktrace']
list	fpeTracebackCont

Detailed Description

Tool to check for error messages in a log file.

By default ERROR, FATAL and CRITICAL messages are considered.
The config file may be used to provide patterns of lines to exclude from this check
(known problems or false positives). If no config file is provided, all errors will be shown.

Function Documentation

get_parser()

get_parser

(

)

Definition at line 85 of file check_log.py.

def get_parser():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=
                                     lambda prog : argparse.HelpFormatter(
                                         prog, max_help_position=40, width=100))
 
    parser.add_argument('logfile', metavar='<logfile>', nargs='+',
                        help='log file(s) to scan')
    parser.add_argument('--config', metavar='<file>',
                        help='specify config file')
    parser.add_argument('--showexcludestats', action='store_true',
                        help='print summary table with number of matches for each exclude pattern')
    parser.add_argument('--printpatterns', action='store_true',
                        help='print the list of warning/error patterns being searched for')
    parser.add_argument('--warnings', action = 'store_true',
                        help='check for WARNING messages')
    parser.add_argument('--errors', action = 'store_true',
                        help='check for ERROR messages')
 
    return parser
 
 

main()

main

(

)

Definition at line 106 of file check_log.py.

def main():
    parser = get_parser()
 
    args = parser.parse_args()
    if not (args.errors or args.warnings):
        parser.error('at least one of --errors or --warnings must be enabled')
 
    ignorePattern = parseConfig(args) if args.config else []
    rc = 0
    for i, lf in enumerate(args.logfile):
        if i>0:
            print()
        rc += scanLogfile(args, lf, ignorePattern)
 
    return rc
 
 

parseConfig()

parseConfig

(

args

)

Parses the config file provided into a list (ignorePattern)

Definition at line 123 of file check_log.py.

def parseConfig(args):
    """Parses the config file provided into a list (ignorePattern)"""
    ignorePattern = []
 
    os.system(f"get_files -data -symlink {args.config} > /dev/null")
    with open(args.config) as f:
        print('Ignoring warnings/error patterns defined in ' + args.config)
        for aline in f:
            if 'ignore' in aline:
              line = aline.strip('ignore').strip()
              if line.startswith('\'') and line.endswith('\''):
                  line = line[1:-1]
              ignorePattern.append(line)
    return ignorePattern
 
 

scanLogfile()

scanLogfile	(		args,
			logfile,
			ignorePattern = [] )

Scan one log file and print report

Definition at line 139 of file check_log.py.

def scanLogfile(args, logfile, ignorePattern=[]):
    """Scan one log file and print report"""
    tPattern = re.compile('|'.join(backtrace))
    fpeStartPattern = re.compile('|'.join(fpeTracebackStart))
    fpeContPattern = re.compile('|'.join(fpeTracebackCont))
    ignoreDict = None
 
    categories = []
    if args.warnings is True:
        categories += ['warning']
    if args.errors is True:
        categories += ['error/fatal', 'prohibited', 'python error', 'fpe', 'backtrace']
 
    # if ignorePattern is empty, igLevels.search would match anything, protect by adding a pattern that matches nothing
    if not ignorePattern:
        ignorePattern.append('(?!)')
 
    igLevels = re.compile('|'.join(ignorePattern))
 
    patterns = {
        cat: re.compile('|'.join(regexMap[cat])) for cat in categories
    }
    resultsA = {cat:[] for cat in categories}
    with open(logfile, encoding='utf-8') as f:
        tracing = False
        fpeTracing = False
 
        try:
            for line in f:
                # First check if we need to start or continue following a trace
                # Tracing only makes sense for errors
                if args.errors:
                    if tPattern.search(line) and not igLevels.search(line):
                        tracing = True
                    elif fpeStartPattern.search(line) and not igLevels.search(line):
                        fpeTracing = True
                    elif line =='\n':
                        tracing = False
                        fpeTracing = False
 
                if tracing:
                    # Save all lines after a backtrace even if they don't belong to backtrace
                    resultsA['backtrace'].append(line)
                elif fpeTracing:
                    # Continue following FPE so long as recognised
                    if fpeStartPattern.search(line) or fpeContPattern.search(line):
                        resultsA['fpe'].append(line)
                    else:
                        fpeTracing = False
                else:
                    for cat in categories:
                        if patterns[cat].search(line):
                            resultsA[cat].append(line)
        except UnicodeDecodeError as e:
            print(f'ERROR: Exception raised processing log file: {e}\n')
 
    ignoreDict = {}
    results = {cat:[] for cat in categories}
    if args.config is None:
        results = resultsA
    else:
        if args.showexcludestats:
            separateIgnoreRegex = [re.compile(line) for line in ignorePattern]
            ignoreDict = {line:0 for line in ignorePattern} # stores counts of ignored errors/warnings
 
        # Filter messages
        for cat, messages in resultsA.items():
            for res in messages:
                if not igLevels.search(res):
                    results[cat].append(res)
                elif args.showexcludestats:
                    for i in range(len(separateIgnoreRegex)):
                        if separateIgnoreRegex[i].search(res):
                            ignoreDict[ignorePattern[i]] += 1
 
 
    # Report results
    found_bad_message = False
    for cat in categories:
 
        if args.printpatterns:
            print(f'check_log.py - Checking for {cat} messages with pattern: {str(patterns[cat])} in '+logfile+'\n')
        if len(results[cat]) > 0:
            print(f'Found {len(results[cat])} {cat} message(s) in {logfile}:')
            for msg in results[cat]: print(msg.strip('\n'))
            found_bad_message = True
 
    if ignoreDict:
        print('Ignored:')
        for s in ignoreDict:
            if ignoreDict[s] > 0:
                print(str(ignoreDict[s]) + "x " + s)
        print('\n')
 
    if found_bad_message:
        print(f'FAILURE : problematic message found in {logfile}')
        return 1
 
    print(f'No error/warning messages found in {logfile}')
    return 0
 
 

Variable Documentation

backtrace

list check_log.backtrace

Initial value:

=  [
    r'Traceback',
    r'Shortened traceback',
    r'stack trace',
    r'^Algorithm stack',
    r'^#\d+\s*0x\w+ in ',
]

Definition at line 59 of file check_log.py.

builtinErrors

list check_log.builtinErrors = [b for b in builtins if 'Error' in b]

Definition at line 55 of file check_log.py.

builtins

check_log.builtins = dir(builtins)

Definition at line 54 of file check_log.py.

fpeTracebackCont

list check_log.fpeTracebackCont

Initial value:

=  [
    '  in function : ',
    '  included from : ',
    '  in library : ',
]

Definition at line 70 of file check_log.py.

fpeTracebackStart

list check_log.fpeTracebackStart = [r'FPEAuditor.*INFO FPE stacktrace']

Definition at line 69 of file check_log.py.

regexMap

dict check_log.regexMap = {}

Definition at line 17 of file check_log.py.