15 from collections
import OrderedDict
16 from TrigValTools.TrigARTUtils
import first_existing_file, newest_file
18 logging.basicConfig(stream=sys.stdout,
19 format=
'%(levelname)-8s %(message)s',
27 from pandas
import read_csv
28 mod_name =
'scipy.optimize'
29 from scipy.optimize
import curve_fit
30 mod_name =
'matplotlib'
33 import matplotlib.pyplot
as plt
34 except Exception
as e:
36 logging.warning(
'Failed to import module %s, prmon output analysis will be skipped! %s', mod_name, e)
39 'Store items in the order the keys were last added'
44 OrderedDict.__setitem__(self, key, value)
48 if not os.path.isfile(filename):
49 logging.warning(
"Cannot open file {}".
format(filename))
51 with open(filename)
as logfile:
52 lines = re.findall(
"{}.*$".
format(pattern), logfile.read(), re.MULTILINE)
54 logging.warning(
"Could not find pattern \"{}\" in file {}".
format(pattern, filename))
62 logging.warning(
"Cannot extract number of messages from {}".
format(filename))
64 logging.debug(
"line: {}".
format(line))
65 m = re.search(
r'\((.+?)\):', line)
72 logging.warning(
"Cannot extract number of histograms from {}".
format(filename))
74 logging.debug(
"line: {}".
format(line))
75 return line.split()[-1]
86 for unit_name, mult
in multipliers.items():
88 return float(number)*mult
89 logging.error(
"Unit conversion failed from {} to MB".
format(unit))
94 words = line[0].
split()
96 logging.debug(
"mem_end = {}".
format(mem_end))
97 mem_delta = words[8:10]
98 logging.debug(
"mem_delta = {}".
format(mem_delta))
100 logging.debug(
"mem_mb = {}".
format(mem_mb))
102 logging.debug(
"delta_mb = {}".
format(delta_mb))
103 return mem_mb, delta_mb
107 if not os.path.isfile(filename):
108 logging.warning(
"Cannot open file {}".
format(filename))
110 with open(filename)
as logfile:
113 all_lines = logfile.readlines()
114 for i, line
in enumerate(all_lines):
115 if first_line >= 0
and last_line >= 0:
117 if "=== [evt - slice] ===" in line:
119 elif "=== [fin - slice] ===" in line:
121 if first_line < 0
or last_line < 0:
122 logging.warning(
"Cannot extract memory usage information from {}".
format(filename))
124 evt_mon_lines = all_lines[first_line:last_line]
125 vmem_line = re.findall(
"^VMem.*$",
'\n'.
join(evt_mon_lines), re.MULTILINE)
126 rss_line = re.findall(
"^RSS.*$",
'\n'.
join(evt_mon_lines), re.MULTILINE)
127 logging.debug(
"vmem_line = {}".
format(vmem_line))
128 logging.debug(
"rss_line = {}".
format(rss_line))
129 if len(vmem_line) == 0:
130 logging.warning(
"Cannot extract VMem information from {}".
format(filename))
131 if len(rss_line) == 0:
132 logging.warning(
"Cannot extract RSS information from {}".
format(filename))
136 data[
'vmem'] =
"{0:.3f}".
format(vmem)
137 data[
'delta-vmem'] =
"{0:.3f}".
format(dvmem)
138 data[
'rss'] =
"{0:.3f}".
format(rss)
139 data[
'delta-rss'] =
"{0:.3f}".
format(drss)
143 def mem_func(x_arr, x_trans, init_slope, exec_slope):
147 retval.append(init_slope * x)
149 retval.append(exec_slope * x + (init_slope - exec_slope) * x_trans)
154 popt, pcov = curve_fit(mem_func, xdata, ydata, bounds=(0, [0.9*
max(xdata), np.inf, np.inf]))
155 logging.debug(
"Fit result: %s",
str(popt))
156 plot_prmon(xdata, ydata, label, filename, popt)
158 x_last = xdata.iloc[-1]
159 dmem_v =
mem_func([x_last, x_trans], popt[0], popt[1], popt[2])
160 return dmem_v[0] - dmem_v[1]
164 plt.plot(xdata, ydata,
'b-', label=name)
166 plt.plot(xdata,
mem_func(xdata, *params),
'r-', label=
'{:s} fit, exec slope={:.2f} kB/s'.
format(name, params[2]))
167 plt.xlabel(
'wtime [s]')
168 plt.ylabel(name+
' [kB]')
170 plt.title(
'{:s} from {:s}'.
format(name, filename))
171 plt.savefig(
'prmon_{:s}.pdf'.
format(name), bbox_inches=
'tight')
177 prmon_data = read_csv(filename, sep=
'\t')
179 logging.warning(
"Cannot open file {}".
format(filename))
181 time_v = prmon_data[
'wtime']
182 pss_v = prmon_data[
'pss']
183 rss_v = prmon_data[
'rss']
184 vmem_v = prmon_data[
'vmem']
190 logging.info(
'Not enough prmon data points, skipping memory slope fitting')
211 logging.debug(
"ne: {}".
format(ne))
213 logging.warning(
"Failed to read number of errors from the log")
214 data[
'num-errors'] =
'n/a'
216 data[
'num-errors'] = ne
221 logging.debug(
"nw: {}".
format(nw))
223 logging.warning(
"Failed to read number of warnings from the log")
224 data[
'num-warnings'] =
'n/a'
226 data[
'num-warnings'] = nw
231 logging.debug(
"nh: {}".
format(nh))
233 logging.warning(
"Failed to read number of histograms from the log")
234 data[
'num-histograms'] =
'n/a'
236 data[
'num-histograms'] = nh
242 prmon_log =
first_existing_file([
'prmon.full.RDOtoRDOTrigger',
'prmon.full.RAWtoESD',
'prmon.full.ESDtoAOD',
'prmon.full.RAWtoALL'])
244 logging.info(
"No prmon output found, the result will be empty")
245 data[
'prmon'] =
'n/a'
247 logging.info(
"Analysing prmon output from %s", prmon_log)
249 if prmon_data
is None:
250 logging.warning(
"Could not analyse prmon output, the result will be empty")
251 data[
'prmon'] =
'n/a'
253 data[
'prmon'] = prmon_data
256 perfmon_log =
newest_file(
r'.*perfmon\.summary\.txt')
258 logging.info(
"No PerfMon output found, the result will be empty")
259 data[
'memory-usage'] =
'n/a'
261 logging.info(
"Analysing PerfMon output from %s", perfmon_log)
263 if perfmon_data
is None:
264 logging.warning(
"Could not analyse PerfMon output, the result will be empty")
265 data[
'memory-usage'] =
'n/a'
267 data[
'memory-usage'] = perfmon_data
270 with open(
'extra-results.json',
'w')
as outfile:
271 json.dump(data, outfile, indent=4)
274 if "__main__" in __name__: