ATLAS Offline Software
Loading...
Searching...
No Matches
TRTCalib_last_Skeleton.py
Go to the documentation of this file.
1# Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
2
3import sys, os, glob, subprocess, tarfile, fnmatch, smtplib
4from email.mime.multipart import MIMEMultipart
5from email.mime.text import MIMEText
6
7
8def nextstep(text):
9 print("\n"+"#"*100)
10 print("#")
11 print("# %s" % (text))
12 print("#")
13 print("#"*100,"\n")
14
15def tryError(command, error):
16 try:
17 print(" Running: %s\n" % (command))
18 stdout, stderr = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
19 print("OUTPUT: \n%s" % (stdout.decode('ascii')))
20 print("ERRORS: %s" % ("NONE" if stderr.decode('ascii')=='' else "\n"+stderr.decode('ascii')))
21 if stderr:
22 exit(1)
23 except OSError as e:
24 print(error,e)
25 sys.exit(e.errno)
26
27def send_statusmail(itera, runNumber, mto, outdir, a, b, c, d, e, f, g, h):
28
29 thres_Res_Ar = 140
30 thres_Res_Xe = 130
31 res_tolerance = 10
32 thres_Res_time = 0
33 res_time_tolerance = 0.5
34
35 mserver = 'cernmx.cern.ch'
36 mfrom = 'no-reply@cern.ch'
37 msubject = "TRT CALIB TIER0 - Exit Status for Run %d" % runNumber
38
39 def color(cond):
40 if cond:
41 return '#90EE90'
42 else:
43 return '#FF5733'
44
45 # Build HTML message body
46 mbody_html = f"""
47 <html>
48 <head>
49 <style>
50 table, th, td {{
51 border: 1px solid white;
52 border-collapse: collapse;
53 font-weight:bold;
54 margin-left:20px
55 }}
56 </style>
57 </head>
58 <body>
59 <p><b>Calibration job finished</b> for run <b>{runNumber}</b>, iteration: <b>{itera}</b></p>
60 <p>Here are the residuals obtained at detector level with currently used constants:</p>
61
62 <table>
63 <tr>
64 <td>Residual-Barrel A: </td>
65 <td style="background-color:{color(abs(a-thres_Res_Ar)<res_tolerance)};">{a:.1f}μm</td>
66 <td>&nbsp&nbsp</td>
67 <td>Time-Residual-Barrel A: </td>
68 <td style="background-color:{color(abs(b-thres_Res_time)<res_time_tolerance)};">{b:.2f}ns</td>
69 </tr>
70 <tr>
71 <td>Residual-Barrel C: </td>
72 <td style="background-color:{color(abs(c-thres_Res_Ar)<res_tolerance)};"> {c:.1f}μm</td>
73 <td>&nbsp&nbsp</td>
74 <td>Time-Residual-Barrel C: </td>
75 <td style="background-color:{color(abs(d-thres_Res_time)<res_time_tolerance)};"> {d:.2f}ns</td>
76 </tr>
77 <tr>
78 <td>Residual-Endcap A: </td>
79 <td style="background-color:{color(abs(e-thres_Res_Xe)<res_tolerance)};"> {e:.1f}μm</td>
80 <td>&nbsp&nbsp</td>
81 <td>Time-Residual-Endcap A: </td>
82 <td style="background-color:{color(abs(f-thres_Res_time)<res_time_tolerance)};"> {f:.2f}ns</td>
83 </tr>
84 <tr>
85 <td>Residual-Endcap C: </td>
86 <td style="background-color:{color(abs(g-thres_Res_Xe)<res_tolerance)};">{g:.1f}μm</td>
87 <td>&nbsp&nbsp</td>
88 <td>Time-Residual-Endcap C: </td>
89 <td style="background-color:{color(abs(h-thres_Res_time)<res_time_tolerance)};">{h:.2f}ns</td>
90 </tr>
91 </table>
92
93 <p><b>Expect:</b> Residual 140 ± 10 μm in the barrel (Ar) and 130 ± 10 μm in EC (Xe). Time-Residual 0.0 ± 0.5 ns.</p>
94 <p><span style="color:{color(True)}; font-weight:bold;">Green</span> is within the threshold, <span style="color:{color(False)}; font-weight:bold;">Red</span> otherwise</p>
95 <p>Please check the histograms of the current run to decide whether to upload new constants.<br>
96 Histograms can be found on AFS, directory: <b>{outdir}</b></p>
97 </body>
98 </html>
99 """
100
101 # Create MIME email
102 msg = MIMEMultipart("alternative")
103 msg["Subject"] = msubject
104 msg["From"] = mfrom
105 msg["To"] = ", ".join(mto) if isinstance(mto, list) else mto
106
107 msg.attach(MIMEText(mbody_html, "html"))
108
109 print("Email body (HTML):\n\n", mbody_html)
110 print("Email sent to:")
111
112 try:
113 with smtplib.SMTP(mserver) as con:
114 if isinstance(mto, str):
115 print(f"\t- {mto}")
116 con.sendmail(mfrom, mto, msg.as_string())
117 elif isinstance(mto, list):
118 for onemto in mto:
119 print(f"\t- {onemto}")
120 con.sendmail(mfrom, onemto, msg.as_string())
121 except OSError as e:
122 print("ERROR: Failed sending email notification\n", e)
123 exit(e.errno)
124
125
126
127def fromRunArgs(runArgs):
128
129
130 nextstep("UNTAR files")
131
132
133 print("Uncompressing files:")
134 try:
135 for file in runArgs.inputTARFile:
136 print("\t-",file)
137 tarfile.open(file).extractall(".")
138 except OSError as e:
139 print("ERROR: Failed uncompressing TAR file\n",e)
140 sys.exit(e.errno)
141
142
143 nextstep("Filtering files")
144
145
146 # RT files (nor merging barrel)
147 files_list_rt =[item for item in glob.glob("*_rt.txt") if "barrel" not in item]
148 files_list_t0 =[item for item in glob.glob("*_t0.txt") if "barrel" not in item]
149 files_list_cal=[item for item in glob.glob("*calibout.root") if "barrel" not in item]
150 # tracks
151 files_list_trk=glob.glob("*tracktuple.root")
152 # straws
153 files_list_stw=glob.glob("*merged.straw.txt")
154 # constants
155 files_list_ctn=glob.glob("*calib_constants_out.txt")
156
157 def listFiles(v,txt=""):
158 print("Number of files for %s: %i" % (txt,len(v)))
159
160 if not v:
161 print("ERROR: list of files for %s is empty" % (txt) )
162 sys.exit(1)
163
164 for i in v:
165 print("\t-%s: %s" % (txt,i))
166
167 listFiles(files_list_rt ,"RT")
168 listFiles(files_list_t0 ,"T0")
169 listFiles(files_list_cal,"Cal")
170 listFiles(files_list_trk,"Track")
171 listFiles(files_list_stw,"Straws")
172 listFiles(files_list_ctn,"Constants")
173
174
175 nextstep("Merging Rt and t0 files")
176
177
178 command = "touch calibout.txt ;"
179 command += " echo '# Fileformat=2' >> calibout.txt ; "
180 command += " echo '# RtRelation' >> calibout.txt ; "
181
182 # Merging Rt files
183 for _f in files_list_rt :
184 command += "cat %s >> calibout.txt ; " % (_f)
185
186 command += " echo '# StrawT0' >> calibout.txt ; "
187
188 # Merging t0 files
189 for _f in files_list_t0 :
190 command += "cat %s >> calibout.txt ; " % (_f)
191
192 # add footer
193 command += " echo '#GLOBALOFFSET 0.0000' >> calibout.txt ; "
194
195 if os.path.isfile("calibout.txt"):
196 print("calibout.txt.. already exists. Removed.")
197 os.remove("calibout.txt")
198
199 tryError(command,"ERROR: Not able to merge the Rt and t0 files\n")
200
201
202
203 nextstep("Merging constants files")
204
205
206 command = " touch oldt0s.txt ;"
207 for _f in files_list_ctn :
208 command += "cat %s >> oldt0s.txt ; " % (_f)
209
210 if os.path.isfile("oldt0s.txt"):
211 print("oldt0s.txt.. already exists. Removed.")
212 os.remove("oldt0s.txt")
213
214 tryError(command,"ERROR: Not able to merge the Rt and t0 files\n")
215
216
217
218 nextstep("Shift text file")
219
220
221 command = 'TRTCalib_cfilter.py calibout.txt calibout.txt oldt0s.txt shiftrt'
222
223 tryError(command,"ERROR: Failed in process TRTCalib_cfilter\n")
224
225
226 nextstep("Run hadd to merge root files")
227
228
229 # Removing the output file if it exists to avoid problems
230 if os.path.isfile("merge.root"):
231 print("merge.root.. already exists. Removed.")
232 os.remove("merge.root")
233
234 command = 'hadd merge.root '
235 # merge all the files
236 for _f in files_list_cal :
237 command += "%s " % (_f)
238 # and track tuple, we just need the first one since they are all the same
239 command += "%s " % files_list_trk[0]
240
241 tryError(command,"ERROR: Failed in process hadd\n")
242
243
244 nextstep("Rename root files")
245
246
247 outputFile = runArgs.outputTAR_MERGEDFile
248 runNumber = int(runArgs.inputTARFile[0].split('.')[1])
249
250 # Rename calibout.txt and merge.root
251 command = "mv -v calibout.txt %s.calibout.txt ; " % outputFile
252 command += "mv -v merge.root %s.merge.root ; " % outputFile
253 # only one straw file is needed
254 command += "mv -v %s straws.%d.txt ; " % (files_list_stw[0], runNumber)
255 # copy also db const
256 command += "mv -v dbconst.txt %s.dbconst.txt" % outputFile
257
258 tryError(command,"ERROR: Failed in process hadd\n")
259
260
261 nextstep("Make all plots")
262
263
264 # generating the .ps file
265 command = "TRTCalib_makeplots itersum %s.merge.root %s/lastconfigfile" % (outputFile, os.path.abspath('.'))
266
267 tryError(command,"ERROR: Failed in creating plots (itersum.ps file)\n")
268
269
270 nextstep("converting ps to pdf")
271
272
273 command = "ps2pdf itersum.ps %s.itersum.pdf" % (outputFile)
274
275 tryError(command,"ERROR: Failed in creating itersum.pdf from itersum.ps)\n")
276
277
278 nextstep("Straw status Report")
279
280
281 command = "mkdir -p -v output/ ; TRTCalib_StrawStatusReport %d" % (runNumber)
282
283 tryError(command,"ERROR: Failed running TRTCalib_StrawStatusReport.cxx\n")
284
285
286 nextstep("Straw status plots (.cxx)")
287
288
289 command = "TRTCalib_StrawStatusPlots"
290
291 tryError(command,"ERROR: Failed running root macro TRTCalib_StrawStatusPlots\n")
292
293
294 nextstep("TAR'ing files")
295
296
297 try:
298 # Getting list of files to be compressed
299 files_list=glob.glob(outputFile+".*")
300 # Compressing
301 tar = tarfile.open(outputFile, "w:gz")
302 print("\nCompressing files in %s output file:" % outputFile)
303 for file in files_list:
304 print("\t-",file)
305 tar.add(file)
306 tar.close()
307 except OSError as e:
308 print("ERROR: Failed compressing the output files\n",e)
309 sys.exit(e.errno)
310
311
312
313 nextstep("Copying files to AFS Directory")
314
315
316 outDIR = "%s/run_%d" % (runArgs.attrtcal_dir, runNumber)
317
318 # Extracting iteration from Tier0 and for emails
319 outputFile_split = outputFile.split('.')
320 itera = '9999' # default iteration for testing only
321 if len(outputFile_split) > 5:
322 if "iter" in outputFile_split[5]:
323 itera = (outputFile_split[5])[-1:] # e.g. data14_cos.00247236.express_express.trtcal.TXT.iter1 --> '1'
324
325 command = "mkdir -p -v %s ; " % (outDIR)
326 command += "cp -v %s.merge.root %s/trtcalib_0%s_histograms.root ; " % (outputFile, outDIR, itera)
327 command += "cp -v %s.calibout.txt %s/calibout.%d.NoShifted.txt ; " % (outputFile, outDIR, runNumber)
328 command += "cp -v %s.itersum.pdf %s/Plots.%d.pdf ; " % (outputFile, outDIR, runNumber)
329 command += "cp -v %s.dbconst.txt %s/dbconst.%d.txt ; " % (outputFile, outDIR, runNumber)
330 command += "cp -v %s.dbconst.txt %s/calibout.%d.txt ; " % (outputFile, outDIR, runNumber)
331 command += "cp -v straws.%d.txt %s/straws.%d.txt; " % (runNumber, outDIR, runNumber)
332 command += "cp -v TRT_StrawStatusReport.txt %s/TRT_StrawStatusReport.%d.txt; " % (outDIR, runNumber)
333 command += "cp -v allPlots.pdf %s/TRT_StrawStatusReport.%d.pdf; " % (outDIR, runNumber)
334
335 tryError(command,"ERROR: Files cannot be copied to the chosen directory\n")
336
337
338
339
340 nextstep("Extractor information")
341
342
343 command = "python -m TRT_CalibAlgs.TRTCalib_Extractor %s/trtcalib_0%s_histograms.root > %s/extraction.txt" % (outDIR, itera, outDIR)
344
345 tryError(command,"ERROR: Extracting information for email notification\n")
346
347
348 nextstep("Email notification")
349
350
351 res_ba = 0 ; tres_ba = 0
352 res_bc = 0 ; tres_bc = 0
353 res_ea = 0 ; tres_ea = 0
354 res_ec = 0 ; tres_ec = 0
355
356 try:
357 with open("%s/extraction.txt" % (outDIR)) as exfile:
358 for line in exfile :
359 line = line.strip()
360 if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part 1*') :
361 res_ba = float(line.split()[6])*1000
362 if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part 1*') :
363 tres_ba = float(line.split()[6])
364 if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part -1*') :
365 res_bc = float(line.split()[6])*1000
366 if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part -1*') :
367 tres_bc = float(line.split()[6])
368 if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part 2*') :
369 res_ea = float(line.split()[6])*1000
370 if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part 2*') :
371 tres_ea = float(line.split()[6])
372 if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part -2*') :
373 res_ec = float(line.split()[6])*1000
374 if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part -2*') :
375 tres_ec = float(line.split()[6])
376 except OSError as e:
377 print("ERROR: Failed reading %s/extraction.txt file\n" % (outDIR) ,e)
378 sys.exit(e.errno)
379
380 if runArgs.sendNotification and runArgs.emailList :
381 send_statusmail(itera, runNumber, runArgs.emailList, outDIR, res_ba, tres_ba, res_bc, tres_bc, res_ea, tres_ea, res_ec, tres_ec)
382 else:
383 print("INFO: No email notification sent since --sendNotification=%r or empty --emailList=" % (runArgs.sendNotification), runArgs.emailList)
384
print
void print(char *figname, TCanvas *c1)
Definition TRTCalib_StrawStatusPlots.cxx:26
split
std::vector< std::string > split(const std::string &s, const std::string &t=":")
Definition hcg.cxx:177
TRTCalib_last_Skeleton.tryError
tryError(command, error)
Definition TRTCalib_last_Skeleton.py:15
TRTCalib_last_Skeleton.fromRunArgs
fromRunArgs(runArgs)
Definition TRTCalib_last_Skeleton.py:127
TRTCalib_last_Skeleton.nextstep
nextstep(text)
Definition TRTCalib_last_Skeleton.py:8
TRTCalib_last_Skeleton.send_statusmail
send_statusmail(itera, runNumber, mto, outdir, a, b, c, d, e, f, g, h)
Definition TRTCalib_last_Skeleton.py:27
Generated on for ATLAS Offline Software by doxygen 1.14.0