ATLAS Offline Software
TRTCalib_last_Skeleton.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
2 
3 import sys, os, glob, subprocess, tarfile, fnmatch, smtplib
4 from email.mime.multipart import MIMEMultipart
5 from email.mime.text import MIMEText
6 
7 
8 def nextstep(text):
9  print("\n"+"#"*100)
10  print("#")
11  print("# %s" % (text))
12  print("#")
13  print("#"*100,"\n")
14 
15 def tryError(command, error):
16  try:
17  print(" Running: %s\n" % (command))
18  stdout, stderr = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
19  print("OUTPUT: \n%s" % (stdout.decode('ascii')))
20  print("ERRORS: %s" % ("NONE" if stderr.decode('ascii')=='' else "\n"+stderr.decode('ascii')))
21  if stderr:
22  exit(1)
23  except OSError as e:
24  print(error,e)
25  sys.exit(e.errno)
26 
27 def send_statusmail(itera, runNumber, mto, outdir, a, b, c, d, e, f, g, h):
28 
29  thres_Res_Ar = 140
30  thres_Res_Xe = 130
31  res_tolerance = 10
32  thres_Res_time = 0
33  res_time_tolerance = 0.5
34 
35  mserver = 'cernmx.cern.ch'
36  mfrom = 'no-reply@cern.ch'
37  msubject = "TRT CALIB TIER0 - Exit Status for Run %d" % runNumber
38 
39  def color(cond):
40  if cond:
41  return '#90EE90'
42  else:
43  return '#FF5733'
44 
45  # Build HTML message body
46  mbody_html = f"""
47  <html>
48  <head>
49  <style>
50  table, th, td {{
51  border: 1px solid white;
52  border-collapse: collapse;
53  font-weight:bold;
54  margin-left:20px
55  }}
56  </style>
57  </head>
58  <body>
59  <p><b>Calibration job finished</b> for run <b>{runNumber}</b>, iteration: <b>{itera}</b></p>
60  <p>Here are the residuals obtained at detector level with currently used constants:</p>
61 
62  <table>
63  <tr>
64  <td>Residual-Barrel A: </td>
65  <td style="background-color:{color(abs(a-thres_Res_Ar)<res_tolerance)};">{a:.1f}μm</td>
66  <td>&nbsp&nbsp</td>
67  <td>Time-Residual-Barrel A: </td>
68  <td style="background-color:{color(abs(b-thres_Res_time)<res_time_tolerance)};">{b:.2f}ns</td>
69  </tr>
70  <tr>
71  <td>Residual-Barrel C: </td>
72  <td style="background-color:{color(abs(c-thres_Res_Ar)<res_tolerance)};"> {c:.1f}μm</td>
73  <td>&nbsp&nbsp</td>
74  <td>Time-Residual-Barrel C: </td>
75  <td style="background-color:{color(abs(d-thres_Res_time)<res_time_tolerance)};"> {d:.2f}ns</td>
76  </tr>
77  <tr>
78  <td>Residual-Endcap A: </td>
79  <td style="background-color:{color(abs(e-thres_Res_Xe)<res_tolerance)};"> {e:.1f}μm</td>
80  <td>&nbsp&nbsp</td>
81  <td>Time-Residual-Endcap A: </td>
82  <td style="background-color:{color(abs(f-thres_Res_time)<res_time_tolerance)};"> {f:.2f}ns</td>
83  </tr>
84  <tr>
85  <td>Residual-Endcap C: </td>
86  <td style="background-color:{color(abs(g-thres_Res_Xe)<res_tolerance)};">{g:.1f}μm</td>
87  <td>&nbsp&nbsp</td>
88  <td>Time-Residual-Endcap C: </td>
89  <td style="background-color:{color(abs(h-thres_Res_time)<res_time_tolerance)};">{h:.2f}ns</td>
90  </tr>
91  </table>
92 
93  <p><b>Expect:</b> Residual 140 ± 10 μm in the barrel (Ar) and 130 ± 10 μm in EC (Xe). Time-Residual 0.0 ± 0.5 ns.</p>
94  <p><span style="color:{color(True)}; font-weight:bold;">Green</span> is within the threshold, <span style="color:{color(False)}; font-weight:bold;">Red</span> otherwise</p>
95  <p>Please check the histograms of the current run to decide whether to upload new constants.<br>
96  Histograms can be found on AFS, directory: <b>{outdir}</b></p>
97  </body>
98  </html>
99  """
100 
101  # Create MIME email
102  msg = MIMEMultipart("alternative")
103  msg["Subject"] = msubject
104  msg["From"] = mfrom
105  msg["To"] = ", ".join(mto) if isinstance(mto, list) else mto
106 
107  msg.attach(MIMEText(mbody_html, "html"))
108 
109  print("Email body (HTML):\n\n", mbody_html)
110  print("Email sent to:")
111 
112  try:
113  with smtplib.SMTP(mserver) as con:
114  if isinstance(mto, str):
115  print(f"\t- {mto}")
116  con.sendmail(mfrom, mto, msg.as_string())
117  elif isinstance(mto, list):
118  for onemto in mto:
119  print(f"\t- {onemto}")
120  con.sendmail(mfrom, onemto, msg.as_string())
121  except OSError as e:
122  print("ERROR: Failed sending email notification\n", e)
123  exit(e.errno)
124 
125 
126 
127 def fromRunArgs(runArgs):
128 
129 
130  nextstep("UNTAR files")
131 
132 
133  print("Uncompressing files:")
134  try:
135  for file in runArgs.inputTARFile:
136  print("\t-",file)
137  tarfile.open(file).extractall(".")
138  except OSError as e:
139  print("ERROR: Failed uncompressing TAR file\n",e)
140  sys.exit(e.errno)
141 
142 
143  nextstep("Filtering files")
144 
145 
146  # RT files (nor merging barrel)
147  files_list_rt =[item for item in glob.glob("*_rt.txt") if "barrel" not in item]
148  files_list_t0 =[item for item in glob.glob("*_t0.txt") if "barrel" not in item]
149  files_list_cal=[item for item in glob.glob("*calibout.root") if "barrel" not in item]
150  # tracks
151  files_list_trk=glob.glob("*tracktuple.root")
152  # straws
153  files_list_stw=glob.glob("*merged.straw.txt")
154  # constants
155  files_list_ctn=glob.glob("*calib_constants_out.txt")
156 
157  def listFiles(v,txt=""):
158  print("Number of files for %s: %i" % (txt,len(v)))
159 
160  if not v:
161  print("ERROR: list of files for %s is empty" % (txt) )
162  sys.exit(1)
163 
164  for i in v:
165  print("\t-%s: %s" % (txt,i))
166 
167  listFiles(files_list_rt ,"RT")
168  listFiles(files_list_t0 ,"T0")
169  listFiles(files_list_cal,"Cal")
170  listFiles(files_list_trk,"Track")
171  listFiles(files_list_stw,"Straws")
172  listFiles(files_list_ctn,"Constants")
173 
174 
175  nextstep("Merging Rt and t0 files")
176 
177 
178  command = "touch calibout.txt ;"
179  command += " echo '# Fileformat=2' >> calibout.txt ; "
180  command += " echo '# RtRelation' >> calibout.txt ; "
181 
182  # Merging Rt files
183  for _f in files_list_rt :
184  command += "cat %s >> calibout.txt ; " % (_f)
185 
186  command += " echo '# StrawT0' >> calibout.txt ; "
187 
188  # Merging t0 files
189  for _f in files_list_t0 :
190  command += "cat %s >> calibout.txt ; " % (_f)
191 
192  # add footer
193  command += " echo '#GLOBALOFFSET 0.0000' >> calibout.txt ; "
194 
195  if os.path.isfile("calibout.txt"):
196  print("calibout.txt.. already exists. Removed.")
197  os.remove("calibout.txt")
198 
199  tryError(command,"ERROR: Not able to merge the Rt and t0 files\n")
200 
201 
202 
203  nextstep("Merging constants files")
204 
205 
206  command = " touch oldt0s.txt ;"
207  for _f in files_list_ctn :
208  command += "cat %s >> oldt0s.txt ; " % (_f)
209 
210  if os.path.isfile("oldt0s.txt"):
211  print("oldt0s.txt.. already exists. Removed.")
212  os.remove("oldt0s.txt")
213 
214  tryError(command,"ERROR: Not able to merge the Rt and t0 files\n")
215 
216 
217 
218  nextstep("Shift text file")
219 
220 
221  command = 'TRTCalib_cfilter.py calibout.txt calibout.txt oldt0s.txt shiftrt'
222 
223  tryError(command,"ERROR: Failed in process TRTCalib_cfilter\n")
224 
225 
226  nextstep("Run hadd to merge root files")
227 
228 
229  # Removing the output file if it exists to avoid problems
230  if os.path.isfile("merge.root"):
231  print("merge.root.. already exists. Removed.")
232  os.remove("merge.root")
233 
234  command = 'hadd merge.root '
235  # merge all the files
236  for _f in files_list_cal :
237  command += "%s " % (_f)
238  # and track tuple, we just need the first one since they are all the same
239  command += "%s " % files_list_trk[0]
240 
241  tryError(command,"ERROR: Failed in process hadd\n")
242 
243 
244  nextstep("Rename root files")
245 
246 
247  outputFile = runArgs.outputTAR_MERGEDFile
248  runNumber = int(runArgs.inputTARFile[0].split('.')[1])
249 
250  # Rename calibout.txt and merge.root
251  command = "mv -v calibout.txt %s.calibout.txt ; " % outputFile
252  command += "mv -v merge.root %s.merge.root ; " % outputFile
253  # only one straw file is needed
254  command += "mv -v %s straws.%d.txt ; " % (files_list_stw[0], runNumber)
255  # copy also db const
256  command += "mv -v dbconst.txt %s.dbconst.txt" % outputFile
257 
258  tryError(command,"ERROR: Failed in process hadd\n")
259 
260 
261  nextstep("Make all plots")
262 
263 
264  # generating the .ps file
265  command = "TRTCalib_makeplots itersum %s.merge.root %s/lastconfigfile" % (outputFile, os.path.abspath('.'))
266 
267  tryError(command,"ERROR: Failed in creating plots (itersum.ps file)\n")
268 
269 
270  nextstep("converting ps to pdf")
271 
272 
273  command = "ps2pdf itersum.ps %s.itersum.pdf" % (outputFile)
274 
275  tryError(command,"ERROR: Failed in creating itersum.pdf from itersum.ps)\n")
276 
277 
278  nextstep("Straw status Report")
279 
280 
281  command = "mkdir -p -v output/ ; TRTCalib_StrawStatusReport %d" % (runNumber)
282 
283  tryError(command,"ERROR: Failed running TRTCalib_StrawStatusReport.cxx\n")
284 
285 
286  nextstep("Straw status plots (.cxx)")
287 
288 
289  command = "TRTCalib_StrawStatusPlots"
290 
291  tryError(command,"ERROR: Failed running root macro TRTCalib_StrawStatusPlots\n")
292 
293 
294  nextstep("TAR'ing files")
295 
296 
297  try:
298  # Getting list of files to be compressed
299  files_list=glob.glob(outputFile+".*")
300  # Compressing
301  tar = tarfile.open(outputFile, "w:gz")
302  print("\nCompressing files in %s output file:" % outputFile)
303  for file in files_list:
304  print("\t-",file)
305  tar.add(file)
306  tar.close()
307  except OSError as e:
308  print("ERROR: Failed compressing the output files\n",e)
309  sys.exit(e.errno)
310 
311 
312 
313  nextstep("Copying files to AFS Directory")
314 
315 
316  outDIR = "%s/run_%d" % (runArgs.attrtcal_dir, runNumber)
317 
318  # Extracting iteration from Tier0 and for emails
319  outputFile_split = outputFile.split('.')
320  itera = '9999' # default iteration for testing only
321  if len(outputFile_split) > 5:
322  if "iter" in outputFile_split[5]:
323  itera = (outputFile_split[5])[-1:] # e.g. data14_cos.00247236.express_express.trtcal.TXT.iter1 --> '1'
324 
325  command = "mkdir -p -v %s ; " % (outDIR)
326  command += "cp -v %s.merge.root %s/trtcalib_0%s_histograms.root ; " % (outputFile, outDIR, itera)
327  command += "cp -v %s.calibout.txt %s/calibout.%d.NoShifted.txt ; " % (outputFile, outDIR, runNumber)
328  command += "cp -v %s.itersum.pdf %s/Plots.%d.pdf ; " % (outputFile, outDIR, runNumber)
329  command += "cp -v %s.dbconst.txt %s/dbconst.%d.txt ; " % (outputFile, outDIR, runNumber)
330  command += "cp -v %s.dbconst.txt %s/calibout.%d.txt ; " % (outputFile, outDIR, runNumber)
331  command += "cp -v straws.%d.txt %s/straws.%d.txt; " % (runNumber, outDIR, runNumber)
332  command += "cp -v TRT_StrawStatusReport.txt %s/TRT_StrawStatusReport.%d.txt; " % (outDIR, runNumber)
333  command += "cp -v allPlots.pdf %s/TRT_StrawStatusReport.%d.pdf; " % (outDIR, runNumber)
334 
335  tryError(command,"ERROR: Files cannot be copied to the chosen directory\n")
336 
337 
338 
339 
340  nextstep("Extractor information")
341 
342 
343  command = "python -m TRT_CalibAlgs.TRTCalib_Extractor %s/trtcalib_0%s_histograms.root > %s/extraction.txt" % (outDIR, itera, outDIR)
344 
345  tryError(command,"ERROR: Extracting information for email notification\n")
346 
347 
348  nextstep("Email notification")
349 
350 
351  res_ba = 0 ; tres_ba = 0
352  res_bc = 0 ; tres_bc = 0
353  res_ea = 0 ; tres_ea = 0
354  res_ec = 0 ; tres_ec = 0
355 
356  try:
357  with open("%s/extraction.txt" % (outDIR)) as exfile:
358  for line in exfile :
359  line = line.strip()
360  if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part 1*') :
361  res_ba = float(line.split()[6])*1000
362  if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part 1*') :
363  tres_ba = float(line.split()[6])
364  if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part -1*') :
365  res_bc = float(line.split()[6])*1000
366  if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part -1*') :
367  tres_bc = float(line.split()[6])
368  if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part 2*') :
369  res_ea = float(line.split()[6])*1000
370  if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part 2*') :
371  tres_ea = float(line.split()[6])
372  if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part -2*') :
373  res_ec = float(line.split()[6])*1000
374  if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part -2*') :
375  tres_ec = float(line.split()[6])
376  except OSError as e:
377  print("ERROR: Failed reading %s/extraction.txt file\n" % (outDIR) ,e)
378  sys.exit(e.errno)
379 
380  if runArgs.sendNotification and runArgs.emailList :
381  send_statusmail(itera, runNumber, runArgs.emailList, outDIR, res_ba, tres_ba, res_bc, tres_bc, res_ea, tres_ea, res_ec, tres_ec)
382  else:
383  print("INFO: No email notification sent since --sendNotification=%r or empty --emailList=" % (runArgs.sendNotification), runArgs.emailList)
384 
TRTCalib_last_Skeleton.nextstep
def nextstep(text)
Definition: TRTCalib_last_Skeleton.py:8
TRTCalib_last_Skeleton.fromRunArgs
def fromRunArgs(runArgs)
Definition: TRTCalib_last_Skeleton.py:127
TRTCalib_last_Skeleton.tryError
def tryError(command, error)
Definition: TRTCalib_last_Skeleton.py:15
color
color
Definition: BinsDiffFromStripMedian.h:18
calibdata.exit
exit
Definition: calibdata.py:235
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:26
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
Trk::open
@ open
Definition: BinningType.h:40
python.CaloAddPedShiftConfig.int
int
Definition: CaloAddPedShiftConfig.py:45
TRTCalib_last_Skeleton.send_statusmail
def send_statusmail(itera, runNumber, mto, outdir, a, b, c, d, e, f, g, h)
Definition: TRTCalib_last_Skeleton.py:27
Trk::split
@ split
Definition: LayerMaterialProperties.h:38
python.LArMinBiasAlgConfig.float
float
Definition: LArMinBiasAlgConfig.py:65
Generated on Mon Sep 29 2025 21:22:49 for ATLAS Offline Software by   doxygen 1.8.18