trig-test-json Namespace Reference

Classes
class	LastUpdatedOrderedDict

Functions
def	get_num_from_checklog (filename)
def	get_num_histos (filename)
def	convert_to_megabytes (number, unit)
def	extract_mem_perfmon (line)
def	analyse_perfmon (filename)
def	mem_func (x_arr, x_trans, init_slope, exec_slope)
def	find_dmem_prmon (xdata, ydata, label, filename)
def	plot_prmon (xdata, ydata, name, filename, params=[])
def	analyse_prmon (filename)
def	main ()

Variables
	stream
	format
	level
	do_prmon
	mod_name

Function Documentation

analyse_perfmon()

def trig-test-json.analyse_perfmon

(

filename

)

Definition at line 102 of file trig-test-json.py.

def analyse_perfmon(filename):
    if not os.path.isfile(filename):
        logging.warning("Cannot open file {}".format(filename))
        return None
    with open(filename) as logfile:
        first_line = -1
        last_line = -1
        all_lines = logfile.readlines()
        for i, line in enumerate(all_lines):
            if first_line >= 0 and last_line >= 0:
                break
            if "=== [evt - slice] ===" in line:
                first_line = i
            elif "=== [fin - slice] ===" in line:
                last_line = i
        if first_line < 0 or last_line < 0:
            logging.warning("Cannot extract memory usage information from {}".format(filename))
            return None
        evt_mon_lines = all_lines[first_line:last_line]
        vmem_line = re.findall("^VMem.*$", '\n'.join(evt_mon_lines), re.MULTILINE)
        rss_line = re.findall("^RSS.*$", '\n'.join(evt_mon_lines), re.MULTILINE)
        logging.debug("vmem_line = {}".format(vmem_line))
        logging.debug("rss_line = {}".format(rss_line))
        if len(vmem_line) == 0:
            logging.warning("Cannot extract VMem information from {}".format(filename))
        if len(rss_line) == 0:
            logging.warning("Cannot extract RSS information from {}".format(filename))
        vmem, dvmem = extract_mem_perfmon(vmem_line)
        rss, drss = extract_mem_perfmon(rss_line)
        data = LastUpdatedOrderedDict()
        data['vmem'] = "{0:.3f}".format(vmem)
        data['delta-vmem'] = "{0:.3f}".format(dvmem)
        data['rss'] = "{0:.3f}".format(rss)
        data['delta-rss'] = "{0:.3f}".format(drss)
        return data
 
 

analyse_prmon()

def trig-test-json.analyse_prmon

(

filename

)

Definition at line 171 of file trig-test-json.py.

def analyse_prmon(filename):
    try:
        prmon_data = read_csv(filename, sep='\t')
    except IOError:
        logging.warning("Cannot open file {}".format(filename))
        return None
    time_v = prmon_data['wtime']
    pss_v = prmon_data['pss']
    rss_v = prmon_data['rss']
    vmem_v = prmon_data['vmem']
    data = LastUpdatedOrderedDict()
    data['vmem'] = "{0:.3f}".format(convert_to_megabytes(max(vmem_v), 'kB'))
    data['rss'] = "{0:.3f}".format(convert_to_megabytes(max(rss_v), 'kB'))
    data['pss'] = "{0:.3f}".format(convert_to_megabytes(max(pss_v), 'kB'))
    if len(time_v) < 80:
        logging.info('Not enough prmon data points, skipping memory slope fitting')
        # Save plots without fitting and return
        plot_prmon(time_v, pss_v, 'pss', filename)
        plot_prmon(time_v, rss_v, 'rss', filename)
        plot_prmon(time_v, vmem_v, 'vmem', filename)
        return data
    d_pss = find_dmem_prmon(time_v, pss_v, 'pss', filename)
    d_rss = find_dmem_prmon(time_v, rss_v, 'rss', filename)
    d_vmem = find_dmem_prmon(time_v, vmem_v, 'vmem', filename)
    data['delta-vmem'] = "{0:.3f}".format(convert_to_megabytes(d_vmem, 'kB'))
    data['delta-rss'] = "{0:.3f}".format(convert_to_megabytes(d_rss, 'kB'))
    data['delta-pss'] = "{0:.3f}".format(convert_to_megabytes(d_pss, 'kB'))
    return data
 
 

convert_to_megabytes()

def trig-test-json.convert_to_megabytes	(		number,
			unit
	)

Definition at line 74 of file trig-test-json.py.

def convert_to_megabytes(number, unit):
    multipliers = {
        'B': 1.0/(1024**2),
        'kB': 1.0/1024,
        'MB': 1,
        "GB": 1024,
        'TB': 1024**2
    }
    for unit_name, mult in multipliers.items():
        if unit_name == unit:
            return float(number)*mult
    logging.error("Unit conversion failed from {} to MB".format(unit))
    return None
 
 

extract_mem_perfmon()

def trig-test-json.extract_mem_perfmon

(

line

)

Definition at line 89 of file trig-test-json.py.

def extract_mem_perfmon(line):
    words = line[0].split()
    mem_end = words[5:7]
    logging.debug("mem_end = {}".format(mem_end))
    mem_delta = words[8:10]
    logging.debug("mem_delta = {}".format(mem_delta))
    mem_mb = convert_to_megabytes(mem_end[0], mem_end[1])
    logging.debug("mem_mb = {}".format(mem_mb))
    delta_mb = convert_to_megabytes(mem_delta[0], mem_delta[1])
    logging.debug("delta_mb = {}".format(delta_mb))
    return mem_mb, delta_mb
 
 

find_dmem_prmon()

def trig-test-json.find_dmem_prmon	(		xdata,
			ydata,
			label,
			filename
	)

Definition at line 149 of file trig-test-json.py.

def find_dmem_prmon(xdata, ydata, label, filename):
    popt, pcov = curve_fit(mem_func, xdata, ydata, bounds=(0, [0.9*max(xdata), np.inf, np.inf]))
    logging.debug("Fit result: %s", str(popt))
    plot_prmon(xdata, ydata, label, filename, popt)
    x_trans = popt[0]
    x_last = xdata.iloc[-1]
    dmem_v = mem_func([x_last, x_trans], popt[0], popt[1], popt[2])
    return dmem_v[0] - dmem_v[1]
 
 

get_num_from_checklog()

def trig-test-json.get_num_from_checklog

(

filename

)

Definition at line 46 of file trig-test-json.py.

def get_num_from_checklog(filename):
    if not os.path.isfile(filename):
        logging.warning(f"Cannot open file {filename}")
        return None
    with open(filename) as logfile:
        if re.search("No error/warning messages found in.*$", logfile.read(), re.MULTILINE) is not None:
            logging.debug("Number of errors/warnings: 0")
            return 0
        logfile.seek(0)
        match = re.search(r'Found (\d+) (\w+) message\‍(s\‍) in.*$', logfile.read(), re.MULTILINE)
        if match is None:
            logging.warning(f"Cannot extract number of messages from {filename}")
            return None
        logging.debug(f"Number of {match[2]}s: {match[1]}")
        return int(match[1])
 

get_num_histos()

def trig-test-json.get_num_histos

(

filename

)

Definition at line 62 of file trig-test-json.py.

def get_num_histos(filename):
    if not os.path.isfile(filename):
        logging.warning(f"Cannot open file {filename}")
        return None
    with open(filename) as logfile:
        match = re.search(r"Total histograms:.* (\d+)$", logfile.read(), re.MULTILINE)
        if match is None:
            logging.warning("Cannot extract number of histograms from {}".format(filename))
            return None
        logging.debug(f"Found {match[1]} histograms")
        return int(match[1])
 

main()

def trig-test-json.main

(

)

Definition at line 201 of file trig-test-json.py.

def main():
    data = LastUpdatedOrderedDict()
 
    # Get number of errors
    checklog_log = first_existing_file(['checklog.log', 'CheckLog.log'])
    ne = get_num_from_checklog(checklog_log) if checklog_log else None
    logging.debug("ne: {}".format(ne))
    if ne is None:
        logging.warning("Failed to read number of errors from the log")
        data['num-errors'] = 'n/a'
    else:
        data['num-errors'] = ne
 
    # Get number of warnings
    warnings_log = first_existing_file(['warnings.log', 'Warnings.log'])
    nw = get_num_from_checklog(warnings_log) if warnings_log else None
    logging.debug("nw: {}".format(nw))
    if nw is None:
        logging.warning("Failed to read number of warnings from the log")
        data['num-warnings'] = 'n/a'
    else:
        data['num-warnings'] = nw
 
    # Get number of histograms
    histcount_log = first_existing_file(['histSizes.log', 'HistCount.log'])
    nh = get_num_histos(histcount_log) if histcount_log else None
    logging.debug("nh: {}".format(nh))
    if nh is None:
        logging.warning("Failed to read number of histograms from the log")
        data['num-histograms'] = 'n/a'
    else:
        data['num-histograms'] = nh
 
    # Get memory usage information from prmon
    if do_prmon:
        prmon_log = newest_file(r'prmon\..*\.txt')
        if not prmon_log:
            prmon_log = first_existing_file(['prmon.full.RDOtoRDOTrigger', 'prmon.full.RAWtoESD', 'prmon.full.ESDtoAOD', 'prmon.full.RAWtoALL'])
        if not prmon_log:
            logging.info("No prmon output found, the result will be empty")
            data['prmon'] = 'n/a'
        else:
            logging.info("Analysing prmon output from %s", prmon_log)
            prmon_data = analyse_prmon(prmon_log)
            if prmon_data is None:
                logging.warning("Could not analyse prmon output, the result will be empty")
                data['prmon'] = 'n/a'
            else:
                data['prmon'] = prmon_data
 
    # Get memory usage information from PerfMon
    perfmon_log = newest_file(r'.*perfmon\.summary\.txt')
    if not perfmon_log:
        logging.info("No PerfMon output found, the result will be empty")
        data['memory-usage'] = 'n/a'
    else:
        logging.info("Analysing PerfMon output from %s", perfmon_log)
        perfmon_data = analyse_perfmon(perfmon_log)
        if perfmon_data is None:
            logging.warning("Could not analyse PerfMon output, the result will be empty")
            data['memory-usage'] = 'n/a'
        else:
            data['memory-usage'] = perfmon_data
 
    # Save data to JSON file
    with open('extra-results.json', 'w') as outfile:
        json.dump(data, outfile, indent=4)
 
 

mem_func()

def trig-test-json.mem_func	(		x_arr,
			x_trans,
			init_slope,
			exec_slope
	)

Definition at line 139 of file trig-test-json.py.

def mem_func(x_arr, x_trans, init_slope, exec_slope):
    retval = []
    for x in x_arr:
        if x < x_trans:
            retval.append(init_slope * x)
        else:
            retval.append(exec_slope * x + (init_slope - exec_slope) * x_trans)
    return retval
 
 

plot_prmon()

def trig-test-json.plot_prmon	(		xdata,
			ydata,
			name,
			filename,
			params = []
	)

Definition at line 159 of file trig-test-json.py.

def plot_prmon(xdata, ydata, name, filename, params=[]):
    plt.plot(xdata, ydata, 'b-', label=name)
    if len(params)>0:
        plt.plot(xdata, mem_func(xdata, *params), 'r-', label='{:s} fit, exec slope={:.2f} kB/s'.format(name, params[2]))
    plt.xlabel('wtime [s]')
    plt.ylabel(name+' [kB]')
    plt.legend()
    plt.title('{:s} from {:s}'.format(name, filename))
    plt.savefig('prmon_{:s}.pdf'.format(name), bbox_inches='tight')
    plt.clf()
 
 

Variable Documentation

do_prmon

trig-test-json.do_prmon

Definition at line 21 of file trig-test-json.py.

format

trig-test-json.format

Definition at line 19 of file trig-test-json.py.

level

trig-test-json.level

Definition at line 20 of file trig-test-json.py.

mod_name

trig-test-json.mod_name

Definition at line 22 of file trig-test-json.py.

stream

trig-test-json.stream

Definition at line 18 of file trig-test-json.py.