ATLAS Offline Software
DeMoGenerateWWW.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 # Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
3 # Author : Benjamin Trocme (CNRS/IN2P3 - LPSC Grenoble) - 2022 - 2023
4 #
5 # Script used to generate to https://atlasdqm.web.cern.ch/atlasdqm/DeMo/ webpage
6 #
7 # Documentation: https://twiki.cern.ch/twiki/bin/viewauth/Atlas/DataQualityDemo
8 
9 
10 import sys, socket, pathlib, errno, re, subprocess, os
11 import argparse
12 
13 from DeMoLib import retrieveYearTagProperties,returnPeriod
14 
15 maindir = "/afs/cern.ch/user/a/atlasdqm/www/DeMo"
16 libdir = os.path.dirname(__file__)
17 
18 weeklyYear = "2023"
19 weeklyTag = "AtlasReady"
20 
21 global run3_yt
22 run3_yt = {"2022":["AtlasReady_BestLumi_HEAD","AtlasReady_BestLumi","GRL_M"],
23  "2023":["AtlasReady"]}
24 
25 # run3_yt_description is retrieved from the DeMoConfig.dat files. See below
26 global run3_yt_description
27 run3_yt_description = {}
28 
29 global subsystems
30 subsystems = {"Inner detector":["Pixel","SCT","TRT"],
31  "Calorimeters":["LAr","Tile"],
32  "Muon":["RPC","MDT","TGC","CSC"],
33  "Trigger/Lumi/Global":["Trig_L1","Trig_HLT","Lumi","Global"],
34  "Forward": ["ALFA","AFP","LUCID","ZDC"],
35  "Combined perf.":["IDGlobal","BTag","CaloCP","MuonCP"]}
36 global subsystemName
37 subsystemName = {"Pixel":"Pixel",
38  "SCT":"SCT",
39  "TRT":"TRT",
40  "LAr":"LAr",
41  "Tile":"Tile",
42  "RPC":"RPC",
43  "MDT":"MDT",
44  "TGC":"TGC",
45  "CSC":"CSC",
46  "Trig_L1":"L1 trigger",
47  "Trig_HLT":"HLT trigger",
48  "Lumi":"Luminosity",
49  "Global":"Global",
50  "ALFA":"ALFA",
51  "AFP":"AFP",
52  "LUCID":"LUCID",
53  "ZDC":"ZDC",
54  "IDGlobal":"ID global",
55  "BTag":"Btagging",
56  "CaloCP":"Calorimeter combined",
57  "MuonCP":"Muon combined"}
58 
59 subsystemNotInRun2 = {}
60 subsystemNotInRun3 = ["CSC"]
61 
62 runListDir = "./YearStats-common"
63 
64 
67 def addLink(l):
68  htmlSplitted = l.split("https")
69 
70  if len(htmlSplitted)>1: # Url found -> Direct link added
71  urlAlreadyReplaced = []
72  for iUrl in range(1,len(htmlSplitted)):
73  url = "https%s"%(htmlSplitted[iUrl].split(" ")[0])
74  if (url not in urlAlreadyReplaced):
75  l = l.replace(url,'''<a href="%s" target="_blank"> %s </a>'''%(url,url))
76  urlAlreadyReplaced.append(url)
77 
78  return l
79 
80 
81 
82 def addHeader(page,year):
83 
84  run3 = (year != "Run2")
85 
86  page.write('''<!doctype html>\n''')
87  page.write('''<html lang=''>\n''')
88  page.write('''<head>\n''')
89  page.write(''' <meta charset='utf-8'>\n''')
90  page.write(''' <meta http-equiv="X-UA-Compatible" content="IE=edge">\n''')
91  page.write(''' <meta name="viewport" content="width=device-width, initial-scale=1">\n''')
92  page.write(''' <link rel="stylesheet" href="styles.css">\n''')
93  page.write(''' <script src="http://code.jquery.com/jquery-latest.min.js" type="text/javascript"></script>\n''')
94  page.write(''' <script src="script.js"></script>\n''')
95  page.write(''' <title>DeMo outputs</title>\n''')
96  page.write('''</head> \n''')
97  page.write('''<body>\n''')
98 
99 
100  page.write('''<div id='cssmenu'>\n''')
101  page.write('''<ul>\n''')
102  if (year == "Run2"):
103  page.write(''' <li><a href='index-Run2.html'><span><b>Run 2</b></span></a></li>\n''')
104  else:
105  page.write(''' <li><a href='index-Run2.html'><span>Run 2</span></a></li>\n''')
106  for irun3_year in run3_yt.keys():
107  if (irun3_year == year):
108  page.write(''' <li><a href='index-%s.html'><span><b>%s</b></span></a></li>\n'''%(irun3_year,irun3_year))
109  else:
110  page.write(''' <li><a href='index-%s.html'><span>%s</span></a></li>\n'''%(irun3_year,irun3_year))
111  page.write('''</ul>\n''')
112  page.write('''</div>\n''')
113 
114 
115  page.write('''<div id='cssmenu'>\n''')
116  page.write('''<ul>\n''')
117 
118 
119  page.write(''' <li class='active has-sub'><a><span>Overview</span></a>\n''')
120  page.write(''' <ul>\n''')
121  if (run3):
122  if (year == weeklyYear):
123  page.write(''' <li><a href='Weekly.html'><span>Weekly</span></a></li>\n''')
124 
125  for iTag in run3_yt[year]:
126  page.write(''' <li><hr><a href='YearStats-%s-%s.html'><span>Year stats - %s </span></a></li>\n'''%(year,iTag,run3_yt_description["%s_%s"%(year,iTag)]))
127  page.write(''' <li><a href='recapDefectsHighlights-%s-%s.html'><span>Defect highlights (beta) - %s </span></a></li>\n'''%(year,iTag,run3_yt_description["%s_%s"%(year,iTag)]))
128  else:
129  page.write(''' <li><a href='YearStats-2018-DQPaper_2018.html'><span>2018 overview - DQ Paper</span></a></li>\n''')
130  page.write(''' <li><a href='YearStats-2017-DQPaper_2017.html'><span>2017 overview - DQ Paper</span></a></li>\n''')
131  page.write(''' <li><a href='YearStats-2016-DQPaper_2016.html'><span>2016 overview - DQ Paper</span></a></li>\n''')
132  page.write(''' <li><a href='YearStats-2015-DQPaper_2015.html'><span>2015 overview - DQ Paper</span></a></li>\n''')
133 
134  page.write(''' </ul>\n''')
135  page.write(''' </li>\n''')
136 
137  for iSystem in subsystems.keys():
138  page.write(''' <li class='active has-sub'><a><span>%s</span></a>'''%iSystem)
139  page.write(''' <ul>\n''')
140 
141  for iSystem in subsystems[iSystem]:
142  if ((run3) and (iSystem not in subsystemNotInRun3)) or ((not run3) and (iSystem not in subsystemNotInRun2)):
143  page.write('''<li class='has-sub'><a ><span>%s</span></a>'''%subsystemName[iSystem])
144  page.write(''' <ul>\n''')
145  else:
146  continue
147 
148  if (run3) and (iSystem not in subsystemNotInRun3):
149  if (year == weeklyYear):
150  page.write(''' <li><a href='%s-Weekly.html'><span>Weekly</span></a></li>\n'''%(iSystem))
151  for iTag in run3_yt[year]:
152  page.write(''' <li><hr><a href='%s-YearStats-%s-%s.html'><span>Year stats - %s </span></a></li>\n'''%(iSystem,year,iTag,run3_yt_description["%s_%s"%(year,iTag)]))
153  page.write(''' <li><a href='%s-recapDefects-%s-%s.html'><span>Defect recap - %s </span></a></li>\n'''%(iSystem,year,iTag,run3_yt_description["%s_%s"%(year,iTag)]))
154  elif (iSystem not in subsystemNotInRun2):
155  page.write(''' <li><hr><a href='%s-YearStats-DQPaper_.html'><span>Year stats - Run2</span></a></li>\n'''%(iSystem))
156  page.write(''' <li class='last'><a href='%s-recapDefects-DQPaper_.html'><span>Defect recap - Run2</span></a></li>\n'''%(iSystem))
157  page.write(''' </ul>\n''')
158  page.write('''</li>\n''')
159  page.write('''</ul>\n''')
160  page.write('''</li>\n''')
161 
162  page.write('''</ul>\n''')
163  page.write('''</div>\n''')
164 
165  page.write('''<br>\n''')
166 
167  return
168 
169 
170 def addFooter(page):
171  page.write('''<div style="text-align:left" class="rectangle">\n''')
172  page.write('''Link to the <a href="logs/DeMoCron.log"> cron job output </a> - Documentation available <a href="https://twiki.cern.ch/twiki/bin/view/Atlas/DataQualityDemo"> here </a> <br>\n''')
173  page.write('''Original developer: Benjamin Trocm&eacute (LPSC Grenoble) - Maintenance: Benjamin Trocm&eacute (LPSC Grenoble)\n''')
174  page.write('''</div></body>\n''')
175  page.write('''<html>\n''')
176  return
177 
178 
179 def addHeaderWeekly(page):
180  page.write('''<div style="text-align:left" class="rectangle">\n''')
181  page.write('''Considered dataset: <br>\n''')
182  page.write('''- all ATLAS-ready runs acquired in the last 7 days <br>\n''')
183  page.write('''- all ATLAS-ready runs fully signed off in the last 7 days <br>\n''')
184  page.write('''- all runs considered for the upcoming wednesday formal signoff (list available <a href="%s/%s/runlist-weekly.dat" target="_blank"> here </a>)<br>'''%(runListDir,weeklyYear))
185  page.write('''Plots updated daily at midnight<br>\n''')
186  page.write('''Status meaning (based on UNCHECKED defect status):<br>\n''')
187  page.write('''-EXPR. : no assessment yet<br>\n''')
188  page.write('''-BULK : express assessment completed<br>\n''')
189  page.write('''-DONE : express and bulk assessment completed (but no final signoff - relevant only for LAr)<br>\n''')
190  page.write('''-FINAL OK: assessment completed<br>\n''')
191  page.write('''The spade symbol after the status indicates that this run is supposed to be signed off at the upcoming DQ wednesday meeting <br>\n''')
192  page.write('''</div>\n''')
193  return
194 
195 
196 def addHeaderYearStatsRecap(page,year="",tag="",system="",type="YearStats"):
197  page.write('''<div style="text-align:left" class="rectangle">\n''')
198 
199  if (system != ""):
200  if (type == "YearStats"):
201  page.write('''<b>%s - %s</b> - Switch to the <a href="%s-recapDefects-%s-%s.html"> defect recap page</a><br>\n'''%(year,run3_yt_description["%s_%s"%(year,tag)],system,year,tag))
202  else:
203  page.write('''<b>%s - %s</b> - Switch to the <a href="%s-YearStats-%s-%s.html"> year stats page</a><br>\n'''%(year,run3_yt_description["%s_%s"%(year,tag)],system,year,tag))
204  else:
205  page.write('''<b>%s - %s</b><br>\n'''%(year,run3_yt_description["%s_%s"%(year,tag)]))
206 
207  if "AtlasReady" in tag: # Year tag named AtlasReady are usually devoted to monitor all runs
208  page.write('''All runs with ATLAS ready considered.''')
209  if "GRL" in tag:
210  page.write('''List of runs derived from the <a href="https://twiki.cern.ch/twiki/bin/view/AtlasProtected/GoodRunListsForAnalysisRun3" target="_blank">GRL twiki </a>.''')
211  page.write('''<br> Direct link to the <a href="%s/%s/runlist-%s-%s.dat" target="_blank"> list. </a> <br>\n'''%(runListDir,year,year,tag))
212 
213  # Scan the DeMoLib.py file to extract the monitored defects and the database tags
214  DeMoLibFile = open("%s/DeMoLib.py"%libdir,"r")
215  offLumiTag = ""
216  offLumiAcctTag = ""
217  defectTag = ""
218  if (system != ""):
219  listOfDefects = {"partIntol":"","partTol":"","globIntol":"","globTol":""}
220 
221  for line in DeMoLibFile:
222  if (system != ""):
223  defectSearch = "%s system"%system
224  for iType in listOfDefects.keys():
225  if re.search(defectSearch, line) and iType in line:
226  listOfDefects[iType] = (((line.split(" = ")[1]).split("# %s system"%system)[0]).replace(" ","")).replace(",",", ")
227 
228  DeMoLibFile.close()
229 
230  if (system != ""):
231  page.write("<hr>")
232  page.write('''<b>Monitored defects </b><br>\n''')
233  if len(listOfDefects["globIntol"])>2:
234  page.write('''Global intolerable defects: %s <br>'''%listOfDefects["globIntol"])
235  if len(listOfDefects["partIntol"])>2:
236  page.write('''Partition intolerable defects: %s <br>'''%listOfDefects["partIntol"])
237  if len(listOfDefects["globTol"])>2:
238  page.write('''Global tolerable defects : %s <br>'''%listOfDefects["globTol"])
239  if len(listOfDefects["partTol"])>2:
240  page.write('''Partition tolerable defects : %s <br>'''%listOfDefects["partTol"])
241 
242  # Add the database tags
243  page.write('''<b>Database tags </b><br>\n''')
244  yearTagProperties = retrieveYearTagProperties(year,tag)
245  for iyt in yearTagProperties.keys():
246  if (iyt == "Defect tag") or (iyt == "OflLumi tag") or (iyt == "OflLumiAcct tag"):
247  page.write('''%s: %s <br>\n'''%(iyt,yearTagProperties[iyt]))
248  if (iyt == "Veto tag") and (system == "LAr"):
249  page.write('''%s: %s <br>\n'''%(iyt,yearTagProperties[iyt]))
250 
251  if (system != ""):
252  page.write('''<hr><a href="YearStats-%s/%s/%s/runs-notYetSignedOff.dat" target="_blank"> Runs not yet signed off </a> - \n'''%(system,year,tag))
253  if (type == "YearStats"):
254  page.write('''<a href="YearStats-%s/%s/%s/daemon-grl.out" target="_blank"> daemon output </a> - \n'''%(system,year,tag))
255  else:
256  page.write('''<a href="YearStats-%s/%s/%s/daemon-grl.out" target="_blank"> daemon output </a> - \n'''%(system,year,tag))
257  # Add the time of the last reset
258  if (os.path.exists("YearStats-%s/%s/%s/lastResetYS.dat"%(system,year,tag))):
259  lastResetFile = open("YearStats-%s/%s/%s/lastResetYS.dat"%(system,year,tag),"r")
260  for iline in lastResetFile.readlines():
261  page.write("Last year-stats reset: %s"%iline)
262 
263  # Look for remaining tokens
264  p = subprocess.Popen("ls",stdout=subprocess.PIPE)
265  (output, err) = p.communicate()
266  remainingToken = ""
267  for iLine in (output.decode('ascii').split("\n")):
268  if (system != ""):
269  if "%s-%s-%s.token"%(system,year,tag) in iLine:
270  remainingToken = "%s<br>"%iLine
271  else:
272  if "%s-%s.token"%(year,tag) in iLine:
273  remainingToken = remainingToken + "%s<br>"%iLine
274  if remainingToken != "":
275  page.write('''<hr><b>Remaining tokens</b>\n''')
276  page.write('''<br>There are some remaining tokens in ~atlasdqm/w1/DeMo: %s <br> This may indicate a problem in processing. Please check the daemon ouputs...'''%remainingToken)
277 
278  if (system == ""):
279  page.write('''</div>\n''')
280  else:
281  page.write("<hr>\n")
282 
283  return
284 
285 
286 def addHeaderRun2RecapDefects(page):
287  page.write('''<div style="text-align:left" class="rectangle">\n''')
288  page.write('''Considered dataset: all "stable beam" runs signed off.\n''')
289  page.write('''<br><a href='#2018' target='_self'> 2018 dataset </a>\n''')
290  page.write('''<br><a href='#2017' target='_self'> 2017 dataset </a>\n''')
291  page.write('''<br><a href='#2016' target='_self'> 2016 dataset </a>\n''')
292  page.write('''<br><a href='#2015' target='_self'> 2015 dataset </a>\n''')
293  page.write('''</div>\n''')
294  return
295 
296 
297 def createRun2LegacyPages():
298 
299  for iYear in ["2015","2016","2017","2018"]:
300  page = open("%s/YearStats-%s-DQPaper_%s.html"%(maindir,iYear,iYear),"w")
301  addHeader(page,"Run2")
302  for iSystem in subsystemName.keys():
303  if (iSystem not in subsystemNotInRun2):
304  page.write('''<div style="text-align:left" class="rectangle"> <b>%s</b>'''%iSystem)
305  page.write('''<figure> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/DQPaper_%s/grl-defects.png" alt="" width="750" content="no-cache" /></figure>\n'''%(iSystem,iYear,iYear))
306  if iSystem == "LAr":
307  page.write('''<figure> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/DQPaper_%s/grl-veto.png" alt="" width="750" content="no-cache" /></figure>\n'''%(iSystem,iYear,iYear))
308  page.write('''</div>\n''')
309  addFooter(page)
310  page.close()
311 
312 
313  for iSystem in subsystemName.keys():
314  if (iSystem not in subsystemNotInRun3):
315  page = open("%s/%s-YearStats-DQPaper_.html"%(maindir,iSystem),"w")
316  addHeader(page,"Run2")
317  for iYear in ["2015","2016","2017","2018"]:
318  page.write('''<div style="text-align:left" class="rectangle"><b>%s</b>\n'''%(iYear))
319  page.write('''<figure> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-LAr/%s/DQPaper_%s/grl-defects.png" alt="" width="750" content="no-cache" /></figure>\n'''%(iYear,iYear))
320  if iSystem == "LAr":
321  page.write('''<figure> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-LAr/%s/DQPaper_%s/grl-veto.png" alt="" width="750" content="no-cache" /></figure>\n'''%(iYear,iYear))
322  page.write('''<figure> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-LAr/%s/DQPaper_%s/grl-lumi.png" alt="" width="750" content="no-cache" /></figure>\n'''%(iYear,iYear))
323  page.write('''</div>\n''')
324  addFooter(page)
325  page.close()
326 
327 
328  for iSystem in subsystemName.keys():
329  if (iSystem not in subsystemNotInRun2):
330  page = open("%s/%s-recapDefects-DQPaper_.html"%(maindir,iSystem),"w")
331  addHeader(page,"Run2")
332  addHeaderRun2RecapDefects(page)
333  for iYear in ["2015","2016","2017","2018"]:
334  page.write('''<hr>\n''')
335  page.write('''<a id='%s'> %s dataset </a>\n'''%(iYear,iYear))
336  readFile = open("YearStats-%s/%s/DQPaper_%s/recapDefects.html"%(iSystem,iYear,iYear))
337  for line in readFile:
338  page.write(line)
339  addFooter(page)
340  page.close()
341 
342 
343 def createWeeklyOverview(weeklyYear,weeklyTag):
344  weekly = open("%s/Weekly.html"%maindir,"w")
345  addHeader(weekly,weeklyYear)
346  addHeaderWeekly(weekly)
347  for iSystem in subsystemName.keys():
348  if (iSystem not in subsystemNotInRun3):
349  weekly.write('''<div style="text-align:left" class="rectangle"> <b>%s</b>'''%subsystemName[iSystem])
350  weekly.write('''<figure> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/Weekly/summary-0.png" alt="" width="750" content="no-cache"/></figure>\n'''%(iSystem,weeklyYear,weeklyTag))
351  weekly.write('''<a> Link to the <a href="%s-Weekly.html" > weekly plots.</a>\n'''%(iSystem))
352  weekly.write('''</div>\n''')
353  addFooter(weekly)
354  weekly.close()
355 
356 
357 def createWeeklySystem(weeklyYear,weeklyTag,system):
358  page = open("%s/%s-Weekly.html"%(maindir,system),"w")
359  addHeader(page,weeklyYear)
360  addHeaderWeekly(page)
361  page.write('''<div style="text-align:left" class="rectangle"><b>%s</b> - <a href="YearStats-%s/%s/%s/daemon-weekly.out" target="_blank"> Daemon output </a>\n'''%(subsystemName[system],system,weeklyYear,weeklyTag))
362  page.write('''<figure> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/Weekly/defects--Run--%s.png" alt="" width="750" content="no-cache" /></figure>\n'''%(system,weeklyYear,weeklyTag,weeklyTag))
363  if system == "LAr" :
364  page.write('''<figure> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/Weekly/veto--Run--%s.png" alt="" width="750" content="no-cache" /></figure>\n'''%(system,weeklyYear,weeklyTag,weeklyTag))
365  page.write('''<figure> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/Weekly/noiseBurst_veto_evol.png" alt="" width="750" content="no-cache" /></figure>\n'''%(system,weeklyYear,weeklyTag))
366  page.write('''<figure> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/Weekly/summary-0.png" alt="" width="750" content="no-cache" /></figure>\n'''%(system,weeklyYear,weeklyTag))
367  addFooter(page)
368  page.close()
369 
370 
371 def createYearStatsSystem(year,tag,system):
372  yearTag = "%s_%s"%(year,tag)
373  page = open("%s/%s-YearStats-%s-%s.html"%(maindir,system,year,tag),"w")
374  addHeader(page,year)
375  addHeaderYearStatsRecap(page,year,tag,system,"YearStats")
376  page.write('''<div style="text-align:left" class="rectangle"><figure> <a href="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/grl-defects.png" target="_blank"> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/grl-defects.png" alt="" width="750" content="no-cache" /> </a></figure>\n'''%(system,year,tag,system,year,tag))
377  if system == "LAr" :
378  page.write('''<figure> <a href="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/grl-veto.png" target="_blank"><img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/grl-veto.png" alt="" width="750" content="no-cache" /></a></figure>\n'''%(system,year,tag,system,year,tag))
379  page.write('''<figure> <a href="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/grl-lumi.png" target="_blank"> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/grl-lumi.png" alt="" width="750" content="no-cache" /></a></figure>\n'''%(system,year,tag,system,year,tag))
380  page.write('''</div>\n''')
381  page.write('''</div>\n''')
382  addFooter(page)
383  page.close()
384 
385 
386 def createYearStatsOverview(year,tag):
387  yearTag = "%s_%s"%(year,tag)
388  page = open("%s/YearStats-%s-%s.html"%(maindir,year,tag),"w")
389  addHeader(page,year)
390  addHeaderYearStatsRecap(page,year,tag)
391 
392  page.write('''<div style="text-align:left" class="rectangle"> <b>ATLAS </b><br>\n''')
393  page.write('''Within each system, the overlaps between defects are treated.<br>\n''')
394  page.write('''For the ATLAS inefficiency, the overlaps between systems are treated.<br>\n''')
395  page.write('''<figure> <a href="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-Global/%s/%s/DataLoss.png" target="_blank"> <img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-Global/%s/%s/DataLoss.png" alt="" width="750" content="no-cache" /></figure> </a>\n'''%(year,tag,year,tag))
396  page.write('''</div>\n''')
397 
398  for iSystem in subsystemName.keys():
399  if (iSystem not in subsystemNotInRun3):
400  page.write('''<div style="text-align:left" class="rectangle"> <b>%s</b>'''%subsystemName[iSystem])
401  page.write('''<figure> <a href="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/grl-defects.png" target="_blank"><img src="https://atlasdqm.web.cern.ch/atlasdqm/DeMo/YearStats-%s/%s/%s/grl-defects.png" alt="" width="750" content="no-cache" /></a></figure>\n'''%(iSystem,year,tag,iSystem,year,tag))
402  page.write('''</div>\n''')
403 
404  addFooter(page)
405  page.close()
406 
407 
408 def createDefectRecapSystem(year,tag,system):
409  yearTag = "%s_%s"%(year,tag)
410  page = open("%s/%s-recapDefects-%s-%s.html"%(maindir,system,year,tag),"w")
411  addHeader(page,year)
412  addHeaderYearStatsRecap(page,year,tag,system,"Recap")
413  #page.write('''<div style="text-align:left" class="rectangle"><b>%s - %s</b> - <a href="YearStats-%s/%s/%s/daemon-recapDefects.out" target="_blank"> daemon output </a> - <a href="YearStats-%s/%s/%s/runs-notYetSignedOff.dat" target="_blank"> Runs not yet signed off </a> - Switch to the <a href="%s-YearStats-%s-%s.html"> year stats page</a> </div>\n'''%(year,run3_yt_description[tag],system,year,tag,system,year,tag,system,year,tag))
414 
415  if (os.path.exists("YearStats-%s/%s/%s/recapDefects.html"%(system,year,tag))):
416  readFile = open("YearStats-%s/%s/%s/recapDefects.html"%(system,year,tag))
417  for line in readFile: # NB: the recapDefects.html files contain a single line for the whole table.
418  page.write(addLink(line))
419 
420  page.write('''</div>\n''')
421  addFooter(page)
422  page.close()
423 
424 
425 def createDefectRecapHighlights(year,tag):
426  yearTag = "%s_%s"%(year,tag)
427  page = open("%s/recapDefectsHighlights-%s-%s.html"%(maindir,year,tag),"w")
428  addHeader(page,year)
429  addHeaderYearStatsRecap(page,year,tag)
430 
431  defectsToKeep = {"Pixel":["DISABLED"],
432  "TRT":["DISABLED"],
433  "RPC":["DISABLED"],
434  "MDT":["DISABLED"],
435  "TGC":["DISABLED"],
436  "Global":["TOROID_OFF","NOTCONSIDERED"]
437  }
438 
439  runHighlights = {}
440 
441  for iSystem in subsystemName.keys():
442  if (iSystem in defectsToKeep.keys()) and (iSystem not in subsystemNotInRun3):
443  if (os.path.exists("YearStats-%s/%s/%s/recapDefects.txt"%(iSystem,year,tag))):
444  readFile = open("YearStats-%s/%s/%s/recapDefects.txt"%(iSystem,year,tag))
445 
446  currentDefect = ""
447  currentRun = ""
448 
449  for line in readFile:
450  if line.startswith("===== Recap for "):
451  currentDefect = ((line.split("===== Recap for ")[1]).split("===")[0]).replace(" ","")
452  elif line.startswith("Description:") or line.startswith(" Run") or (currentDefect not in defectsToKeep[iSystem]):
453  continue
454  else:
455  iovLine = line.split("|")
456  if (len(iovLine) == 8): # The defect to keep should be on a single line (i.e. not multiple iov in a given run)
457  runFound = iovLine[0]
458  if (runFound not in runHighlights.keys()):
459  runHighlights[runFound] = {}
460  runHighlights[runFound]["TotLumi"] = iovLine[2]
461  if (iovLine[3] == runHighlights[runFound]["TotLumi"]):
462  runHighlights[runFound][iSystem] = "<th>Whole run </th><th>%s : %s</th>"%(currentDefect,addLink(iovLine[7]))
463  else:
464  runHighlights[runFound][iSystem] = "<th>%s </th><th>%s : %s</th>"%(iovLine[3],currentDefect,addLink(iovLine[7]))
465 
466  page.write('''<div style="text-align:left" class="rectangle">''')
467 
468  page.write('''<table class="report"><tr class="out0"> <th width="100pix"></th><th width="100pix"></th><th width="100pix"></th><th width="150pix"></th><th width="100pix"></th></th><th width="1200pix"></th></tr>\n''')
469  page.write('''<tr class="out0"><th> Run </th><th> Period </th> <th> Total lumi </th><th> Affected system </th> <th> Lost lumi </th> <th> Comment </th> </tr> \n''')
470 
471  listOfRuns = list(runHighlights.keys())
472  listOfRuns.sort(reverse=True)
473 
474  for iRun in listOfRuns:
475  newRun = True
476  for iSystem in runHighlights[iRun].keys():
477  if iSystem == "TotLumi":
478  continue
479  if newRun:
480  page.write('''<tr class="out1"><th> %s </th> <th> %s </th> <th> %s </th> <th> %s </th> %s </tr> </tr>\n'''%(iRun,returnPeriod(int(iRun),"Pixel",year,"AtlasReady"),runHighlights[iRun]["TotLumi"],iSystem,runHighlights[iRun][iSystem]))
481  newRun = False
482  else:
483  page.write('''<tr class="out1"><th> </th> <th> </th> <th> </th> <th> %s </th> %s </tr></tr>\n'''%(iSystem,runHighlights[iRun][iSystem]))
484 
485  page.write('''</table>\n''')
486 
487 
488  page.write('''</div>''')
489  addFooter(page)
490  page.close()
491 
492 
493 
496 
497 parser = argparse.ArgumentParser(description='')
498 parser.add_argument('-y','--year',dest='parser_year',default = "2023",help='Year: Run2, 2022,2023...[Default: 2023].',action='store')
499 
500 args = parser.parse_args()
501 parser.print_help()
502 
503 year = args.parser_year
504 
505 # run3_yt_description is retrieved from the DeMoConfig.dat file
506 run3_yt_description = {}
507 for iYear in run3_yt.keys():
508  for iTag in run3_yt[iYear]:
509  DeMoConfigFile = open("%s/%s/DeMoConfig-%s-%s.dat"%(runListDir,iYear,iYear,iTag),"r")
510  for iline in DeMoConfigFile:
511  if "Description: " in iline:
512  run3_yt_description["%s_%s"%(iYear,iTag)] = (iline.split("Description: ")[1]).replace("\n","")
513 
514 homepage = open("%s/index-%s.html"%(maindir,year),"w")
515 addHeader(homepage,year)
516 addFooter(homepage)
517 homepage.close()
518 
519 if (year == "Run2"):
520 
521  createRun2LegacyPages()
522 else:
523 
524  createWeeklyOverview(weeklyYear,weeklyTag)
525 
526 
527  for iSystem in subsystemName.keys():
528  if iSystem not in subsystemNotInRun3:
529  createWeeklySystem(weeklyYear,weeklyTag,iSystem)
530 
531 
532  for iTag in run3_yt[year]:
533  createYearStatsOverview(year,iTag)
534 
535 
536  for iTag in run3_yt[year]:
537  for iSystem in subsystemName.keys():
538  if (iSystem not in subsystemNotInRun3):
539  createYearStatsSystem(year,iTag,iSystem)
540 
541 
542  for iTag in run3_yt[year]:
543  for iSystem in subsystemName.keys():
544  if (iSystem not in subsystemNotInRun3):
545  createDefectRecapSystem(year,iTag,iSystem)
546 
547 
548  for iTag in run3_yt[year]:
549  createDefectRecapHighlights(year,iTag)
replace
std::string replace(std::string s, const std::string &s2, const std::string &s3)
Definition: hcg.cxx:307
DeMoGenerateWWW.addHeader
def addHeader(page, year)
Definition: DeMoGenerateWWW.py:82
CaloCellPos2Ntuple.int
int
Definition: CaloCellPos2Ntuple.py:24
DeMoGenerateWWW.createYearStatsOverview
def createYearStatsOverview(year, tag)
Definition: DeMoGenerateWWW.py:386
DeMoGenerateWWW.addFooter
def addFooter(page)
Definition: DeMoGenerateWWW.py:170
DeMoGenerateWWW.addLink
def addLink(l)
When https is found in a character line, add a link in html If none, simply return the line unchanged...
Definition: DeMoGenerateWWW.py:67
DeMoGenerateWWW.createDefectRecapHighlights
def createDefectRecapHighlights(year, tag)
Definition: DeMoGenerateWWW.py:425
DeMoGenerateWWW.createWeeklySystem
def createWeeklySystem(weeklyYear, weeklyTag, system)
Definition: DeMoGenerateWWW.py:357
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:195
histSizes.list
def list(name, path='/')
Definition: histSizes.py:38
DeMoGenerateWWW.addHeaderRun2RecapDefects
def addHeaderRun2RecapDefects(page)
Definition: DeMoGenerateWWW.py:286
DeMoLib.retrieveYearTagProperties
def retrieveYearTagProperties(year, tag)
Return the year/tag properties (defect/veto/lumi tags...) stored in YearStats-common/DeMoConfig-[year...
Definition: DeMoLib.py:68
DeMoGenerateWWW.addHeaderWeekly
def addHeaderWeekly(page)
Definition: DeMoGenerateWWW.py:179
DeMoGenerateWWW.createWeeklyOverview
def createWeeklyOverview(weeklyYear, weeklyTag)
Definition: DeMoGenerateWWW.py:343
Trk::open
@ open
Definition: BinningType.h:40
DeMoGenerateWWW.createRun2LegacyPages
def createRun2LegacyPages()
Definition: DeMoGenerateWWW.py:297
DeMoGenerateWWW.addHeaderYearStatsRecap
def addHeaderYearStatsRecap(page, year="", tag="", system="", type="YearStats")
Definition: DeMoGenerateWWW.py:196
DeMoLib.returnPeriod
def returnPeriod(runNb, system, year, tag)
Return the data period of a run based on the information stored in YearStats-[system]/[year]/[tag]/ru...
Definition: DeMoLib.py:56
python.Bindings.keys
keys
Definition: Control/AthenaPython/python/Bindings.py:790
DeMoGenerateWWW.createYearStatsSystem
def createYearStatsSystem(year, tag, system)
Definition: DeMoGenerateWWW.py:371
Trk::split
@ split
Definition: LayerMaterialProperties.h:38
DeMoGenerateWWW.createDefectRecapSystem
def createDefectRecapSystem(year, tag, system)
Definition: DeMoGenerateWWW.py:408
Generated on Thu Jul 4 2024 21:13:59 for ATLAS Offline Software by   doxygen 1.8.18