44def runDQ(args):
45
46 runNumbers=args.runNumbers
47 fromDate=args.since
48 toDate=args.until
49 output=args.output
50 commentFile=args.commentFile
51 doWeb=args.web
52
53 runsFromFiles = {}
54
55 if runNumbers==[]:
56 if len(args.filesInput)>0:
57
58 for f in args.filesInput:
59 rf = ROOT.TFile.Open(f)
60 for k in rf.GetListOfKeys():
61 if k.GetName().startswith("run_"):
62 runsFromFiles[k.GetName()[4:]] = f
63 rf.Close()
64 else:
65
66 print(
"Available runs from",fromDate,
"to",toDate,
":")
67 for projName,projPattern in [("pp","data2*eV"),("cosmic","data2*_cos"),("hi","data2*_hi")]:
68 print(f
" {projName}:")
69 for runNum,file in getFiles(project=projPattern,runNumbers=runNumbers,since=datetime.datetime.combine(fromDate, datetime.datetime.min.time()),until=datetime.datetime.combine(toDate, datetime.datetime.min.time())).items():
71
72 exit(0)
73
74 if not os.path.exists(commentFile):
75 print(
"ERROR: Comment file",commentFile,
"does not exist")
76 exit(1)
77
78 hcfg = os.path.expandvars("$BuildArea/$CMTCONFIG/data/DataQualityConfigurations/collisions_run.hcfg")
79 if not os.path.exists("./hanResults"): os.mkdir("./hanResults")
80
81 if "previous" in runNumbers:
82
83
84 previousRuns = []
85 for f in glob.glob(f"./hanResults/{args.stream}*.root"):
86 if os.path.getmtime(f)>=datetime.datetime.combine(args.since, datetime.datetime.min.time()).timestamp():
87 previousRuns += [f.split(
"/")[-1].
split(
".")[1].
split(
"_")[1]]
88 print(
"Including previous runs from this week:",previousRuns)
89 newRunList = []
90 for r in runNumbers:
91 if r=="previous":
92 newRunList+=previousRuns
93 else:
94 newRunList+=[r]
95 runNumbers = newRunList
96
97 histResults = {}
98 allRuns = []
99 allMeta = {}
100
101 spotDetections = {}
102
103 import base64
104
105 ROOT.gROOT.SetBatch(True)
106
107 tmpFileName = "/tmp/plt.png"
108 i=0
109 while os.path.exists(tmpFileName):
110 tmpFileName = f"/tmp/plt{i}.png"
111 i+=1
112
113
114 firstDraw = True
115 for runNum,file in (pbar := tqdm.tqdm(runsFromFiles.items() if len(runsFromFiles)>0 else getFiles(runNumbers=runNumbers,since=datetime.datetime.combine(fromDate, datetime.datetime.min.time()),until=datetime.datetime.combine(toDate, datetime.datetime.min.time()),stream=args.stream).items(),desc="Collating DQ results",unit='run')):
116 if len(runNumbers) and runNum not in runNumbers and "*" not in runNumbers: continue
117 allRuns += [runNum]
119 allMeta[runNum]["project"] = file.split("/")[5]
120 hanFiles = glob.glob(f"./hanResults/{args.stream}.run_{runNum}_han*.root")
121 if len(hanFiles)==0:
122
123
124 if doWeb:
125 import random
126 r = random.randrange(100000,999999)
127 cmdStr = f"DQWebDisplay.py {file} TestDisplay \"{r}\""
128 import subprocess
129 print(
"Executing:",cmdStr,
"... Please be patient ...")
130 process = subprocess.Popen(cmdStr, shell=True)
131 process.wait()
132 if process.returncode==0:
133
134 import shutil
135
136 shutil.copy(f"/afs/cern.ch/user/a/atlasdqm/dqmdisk/han_results/test/{r}/{args.stream}/run_{runNum}/run_{runNum}_han.root",f"./hanResults/{args.stream}.run_{runNum}_han.{r}.root")
137 hanFiles = [f"./hanResults/{args.stream}.run_{runNum}_han.{r}.root"]
138 else:
139 print(
"WARNING: Failed to run DQWebDisplay ... please report this!")
140 input("Press Enter to continue (will run DQ algorithms locally)...")
141 if len(hanFiles)==0:
142 ROOT.dqutils.MonitoringFile(file).getHanResults("./hanResults",file,hcfg,"","")
143 os.rename(f"./hanResults/run_{runNum}_han.root",f"./hanResults/{args.stream}.run_{runNum}_han.root")
144 hanFiles = [f"./hanResults/{args.stream}.run_{runNum}_han.root"]
145 print(
"Saved results of DQ algorithms to:",hanFiles[0])
146 allMeta[runNum][
"hanFile"] = hanFiles[0].
split(
"/")[-1]
147 f = ROOT.TFile(hanFiles[0])
148 ROOT.gStyle.SetOptStat(False)
149 ROOT.gStyle.SetPadRightMargin(0.15)
150 ROOT.gErrorIgnoreLevel = ROOT.kWarning
151 def findHists(d,firstDraw,runNum):
152 for k in d.GetListOfKeys():
153 if k.IsFolder():
154 if k.GetName().endswith("_"): continue
155 findHists(k.ReadObj(),firstDraw)
156 elif k.ReadObj().InheritsFrom("TH1") or k.ReadObj().InheritsFrom("TEfficiency"):
157
158
159
160 res = d.Get(k.GetName() + "_/Results")
161 if res is None:
162 print(d.GetPath()+
"/"+k.GetName(),
"ERROR:" + k.GetName())
163 else:
164 histPath = d.GetPath().
split(
":",1)[-1] +
"/"+k.GetName()
165 if histPath not in histResults: histResults[histPath] = {}
166 histResults[histPath][runNum] = eval(res.GetString().
Data())
167 conf = d.Get(k.GetName() + "_/Config")
168 dispOpt = []
169 h = k.ReadObj()
170 if conf:
171 confDict = eval(conf.GetString().
Data())
172 if "name" in confDict:
173 histResults[histPath][runNum]["algorithm"] = confDict["name"]
174 if confDict["name"]=="L1Calo_BinsDiffFromStripMedian" and "detail/" not in histPath:
175
176 if histPath not in spotDetections:
177 spotDetections[histPath] = h.Clone("spotDet_"+h.GetName())
178 spotDetections[histPath].Reset()
179 spotDetections[histPath].SetDirectory(0)
180 for kk,vv
in eval(res.GetString().
Data()).items():
181 if any([a in kk for a in ["_Hot(","_Cold(","_Dead("]]):
182
184 loc = kk.split(
"(")[1].
split(
")")[0].
split(
",")
185 loc = (int(loc[0]),int(loc[1]))
186 binx = spotDetections[histPath].GetXaxis().GetBinCenter(loc[0])
187 biny = spotDetections[histPath].GetYaxis().GetBinCenter(loc[1])
188 spotDetections[histPath].Fill(binx,biny,1 if "_Hot" in kk else -1)
189
190 if "annotations" in confDict and "display" in confDict["annotations"]:
191 dispOpt = confDict[
"annotations"][
"display"].
split(
",")
192
193
194 if h.InheritsFrom(
"TEfficiency")
and h.GetTotalHistogram().
GetEntries()==0:
195
196 h = h.GetTotalHistogram()
197 dString = ""
198 for o in dispOpt:
199 if o.startswith("Draw="): dString = o[5:]
200 if dString=="" and h.InheritsFrom("TH1") and h.GetDimension()==2: dString = "COLZ"
201
202 extraHist = None
203 if h.InheritsFrom("TH2") and "COL" in dString:
204 allVals = []
205 for i in range(1,h.GetNbinsX()+1):
206 for j in range(1,h.GetNbinsY()+1):
207 if h.GetBinContent(i,j)!=0:
208 allVals += [h.GetBinContent(i,j)]
209 allVals.sort()
210 if len(allVals)>0:
211 h.SetMaximum(allVals[int(len(allVals)*0.95)]*1.1+10)
212
213 if any([v>h.GetMaximum() for v in allVals]):
214 extraHist = h.Clone("extraHist")
215 extraHist.Reset()
216 isProf = h.InheritsFrom("TProfile2D")
217 for i in range(h.GetNbinsX()+1):
218 for j in range(h.GetNbinsY()+1):
219 if isProf and h.GetBinContent(i,j)>h.GetMaximum(): extraHist.SetBinEntries(h.GetBin(i,j),1)
220 extraHist.SetBinContent(i,j,h.GetMaximum() if h.GetBinContent(i,j)>h.GetMaximum() else 0)
221 pbar.set_description("Drawing " + h.GetName() + ("(Please be patient, the first draw is the slowest)" if firstDraw else ""))
222 firstDraw=False
223 h.Draw(dString)
224 for o in dispOpt:
225 if o.startswith("SetPalette("): ROOT.gStyle.SetPalette(int(re.findall(r'\d+', o)[0]))
226 elif o.startswith("LogX"): ROOT.gPad.SetLogx(True)
227 elif o.startswith("LogY"): ROOT.gPad.SetLogy(True)
228 elif o.startswith("LogZ"): ROOT.gPad.SetLogz(True)
229 elif o.startswith("SetGridx"): ROOT.gPad.SetGridx(True)
230 elif o.startswith("SetGridy"): ROOT.gPad.SetGridy(True)
231
232 if h.InheritsFrom(
"TH1")
and h.GetXaxis().GetTitle().
strip()
in [
"LB",
"LBN",
"Lumi Block"]:
233
234 r = []
235 for i in range(h.GetNbinsX()+1):
236 filled=False
237 if h.GetDimension()==2:
238 for j in range(h.GetNbinsY()+1):
239 if h.GetBinContent(i,j)!=0:
240 filled=True
241 break
242 elif h.GetDimension()==1:
243 filled = (h.GetBinContent(i)!=0)
244 if filled:
245 if len(r)==0:
246 r = [i-1,i+1]
247 else:
248 r[1] = i+1
249 if len(r)>0: h.GetXaxis().SetRange(r[0],r[1])
250 if extraHist is not None:
251 extraHist.SetBit(ROOT.kCanDelete)
252 extraHist.SetFillColor(ROOT.kPink+6)
253 extraHist.SetFillStyle(1001)
254 extraHist.SetLineWidth(0)
255 extraHist.Draw("BOX same")
256
257 ROOT.gPad.SaveAs(tmpFileName)
258 ROOT.gPad.SetFillColor(ROOT.kWhite)
259 ROOT.gStyle.SetPalette(ROOT.kBird)
260 ROOT.gPad.SetLogx(False);ROOT.gPad.SetLogy(False);ROOT.gPad.SetLogz(False)
261 ROOT.gPad.SetGridx(False);ROOT.gPad.SetGridy(False)
262
263 pltCode = base64.b64encode(open(tmpFileName,
'rb').
read()).decode(
'utf-8').
replace(
'\n',
'')
264 os.remove(tmpFileName)
265 histResults[histPath][runNum]["imageCode"] = pltCode
266
267 findHists(f,firstDraw,runNum)
268
269 allRuns.sort()
270
271 if len(allRuns)==0:
272 print(
"No runs for the report")
273 exit(1)
274
275
276 with open(commentFile,'r') as file:
277 for line in file:
278 if not line.strip(): continue
279 try:
280 path,runs,comment = line.split(":",2)
281 except ValueError as e:
282 print(
"Comment line does not match required format:",line)
283 raise e
284 if path!="": path = "/L1Calo/Expert/"+path
285 elif path not in histResults: histResults[path] = {}
286 if path not in histResults:
287 print(
"WARNING: Unknown histogram path",path,
"- cannot add comment")
288 continue
289 for run in runs.split(","):
290 if (run=="" or path=="") and run not in histResults[path]: histResults[path][run] = {}
291 if run not in histResults[path]:
292 print(
"WARNING: Unknown run",run,
"- cannot add comment")
293 continue
294 if "comment" in histResults[path][run]: histResults[path][run]["comment"] += ";" + comment
295 else: histResults[path][run]["comment"] = comment
296
297 from DQDefects import DefectsDB
298 ddb = DefectsDB()
299
300 with open(output,'w') as outFile:
301 outFile.write("""
302<html><head><style>
303thead {
304 background-color: white;
305 position: sticky;
306 top: 0;
307}
308</style></head><body>
309 """)
310 if args.stream!="express_express":
311 outFile.write(f"<h1>L1Calo Validation DQ Report - {fromDate} to {toDate} - {args.stream} - Author: {os.getlogin()}</h1>\n")
312 else:
313 outFile.write(f"<h1>L1Calo DQ Report - {fromDate} to {toDate} - Author: {os.getlogin()}</h1>\n")
314 outFile.write("<table cellspacing=0><thead><tr><td rowspan=\"5\" valign=top><a href=\"#ByteSreamDecoders\">ByteSreamDecoders</a><br><a href=\"#Efficiency\">Efficiency</a><br><a href=\"#Inputs\">Inputs</a><br><a href=\"#Outputs\">Outputs</a><br><a href=\"#PpmTrex\">PpmTrex</a><br><a href=\"#Sim\">Sim</a></td><td align=right>RunNumber:</td>\n")
315 for r in allRuns: outFile.write(f"<td align=center><a href='https://atlas-runquery.cern.ch/query.py?q=find+run+{r}+%2F+show+all' target='_blank'>{r}</a></td>\n")
316 outFile.write("</td><tr><td align=right>Athena Release:</td>")
317 for r in allRuns: outFile.write(f"<td align=center>{allMeta[r]['release']}</td>\n")
318 outFile.write("</td><tr><td align=right>Project:</td>")
319 for r in allRuns: outFile.write(f"<td align=center>{allMeta[r]['project']}</td>\n")
320 outFile.write("</td><tr><td align=right>Approx Lumi (incl. unstable)/fb<sup>-1</sup>:</td>")
321 for r in allRuns: outFile.write(f"<td align=center>{allMeta[r]['lumi']:.3f}</td>\n")
322 outFile.write("</tr><tr><td align=right>Defects:</td>\n")
323
324 for r in tqdm.tqdm(allRuns,desc="Accessing defects ..."):
325 defects = ddb.retrieve(since=(int(r),0), until=(int(r),999999), primary_only=True)
326 dStr = ""
327 for d in defects:
328 if not d.channel.startswith("TRIG_L1_CAL"): continue
329 dStr += f"{d.since.lumi}-{d.until.lumi}:{d.channel.split('_',3)[-1]}<br>"
330 outFile.write(f" <td align=center>{dStr}</td>\n")
331 outFile.write("</tr>\n")
332 if "" in histResults:
333
334 outFile.write("<tr><td align=right>Overall DQ Comments:</td>\n")
335 for r in allRuns:
336 cStr = histResults[''][r]["comment"] if r in histResults[''] else ""
337 outFile.write(f" <td align=center><font size=1>{cStr}</font></td>\n")
338 outFile.write("</tr>\n")
339 outFile.write("</thead><tbody>\n")
340 currentDir = ""
341 for k in sorted(histResults.keys(),key=lambda x: x.replace("/detail","/zzzdetail")):
342 if "Expert/" not in k: continue
343 v = histResults[k]
344
345 if currentDir != k.split("/")[3]:
346 if currentDir != "":
347 outFile.write(f"<tr><td colspan=\"{len(allRuns)+2}\"><hr width=100%><div id=\"{k.split('/')[3]}\"></div></td></tr>\n")
348 currentDir = k.split("/")[3]
349
350 allUndefined = all([m["Status"]=="Undefined" for rr,m in v.items() if rr in allRuns])
351 if "detail/" not in k and allUndefined: continue
352 anyRed = any([m["Status"]=="Red" for rr,m in v.items() if rr in allRuns])
353 anyYellow = any([m["Status"]=="Yellow" for rr,m in v.items() if rr in allRuns])
354 if allUndefined: col = "#cccccc"
355 elif anyRed: col = "#fc9797" if "detail/" in k else "#ff0000"
356 elif anyYellow: col = "#fcd283" if "detail/" in k else "#ffa500"
357 else: col = "#b3e8b3" if "detail/" in k else "#00dd00"
358 bgColorStr = ' bgcolor=\"#eeeeee\"' if 'detail' in k else ''
359 outFile.write(f"<tr{bgColorStr}><td colspan=\"2\" align=right><div style=\"color:{col}\">{k.replace('/L1Calo/Expert/','')}</div><br><font size=1>{v['']['comment'] if '' in v else ''}</font></td>\n")
360 for r in allRuns:
361 if r not in v:
362 outFile.write("<td align=center>N/A</td>\n")
363 else:
364 if v[r]["Status"]=="Undefined": col = "#cccccc"
365 elif v[r]["Status"]=="Red": col = "#fc9797" if "detail/" in k else "#ff0000"
366 elif v[r]["Status"]=="Yellow": col = "#fcd283" if "detail/" in k else "#ffa500"
367 else: col = "#b3e8b3" if "detail/" in k else "#00dd00"
368 altText = ""
369 if "algorithm" in v[r]: altText += f"Algorithm:{v[r]['algorithm']}

"
370 for kk,vv in v[r].items():
371 if kk not in ["imageCode","Status","name","comment","algorithm"]: altText += f"{kk}:{vv}
"
372 linkUrl = f"https://atlasdqm.cern.ch/webdisplay/tier0/1/{allMeta[r]['hanFile'].split('.')[0]}/run_{r}"
373 if len(allMeta[r][
"hanFile"].
split(
"."))==4:
374 webDisplayNum = allMeta[r][
"hanFile"].
split(
".")[-2]
375 linkUrl = f"https://atlasdqm.cern.ch/webdisplay/test/{webDisplayNum}/{allMeta[r]['hanFile'].split('.')[0]}/run_{r}"
376 outFile.write(f"<td valign=top align=center width=150><img title=\"{altText}\" src=\"data:image/png;base64,{v[r]['imageCode']}\" width=150 style='border:2px solid {col}' onclick=\"window.open('{linkUrl}/run{k}','_blank');\" ondblclick=\"window.open(this.src, '_blank');\"/><br><font size='1'>{v[r].get('comment','')}</font></td>")
377
378 outFile.write("</tr>\n")
379 outFile.write("</tbody></table>\n")
380 outFile.write("<h4>Spot Detection</h4><br>+1 for each run where location is Hot, -1 where it is Cold/Dead<br>")
381 ROOT.gStyle.SetPalette(ROOT.kRainBow)
382 for k,v in spotDetections.items():
383 if "Expert/" not in k: continue
384 outFile.write(k+"<br>")
385 v.Draw("COL1Z")
386 ROOT.gPad.SaveAs(tmpFileName)
387 pltCode = base64.b64encode(open(tmpFileName,
'rb').
read()).decode(
'utf-8').
replace(
'\n',
'')
388 os.remove(tmpFileName)
389 outFile.write(f"<img src=\"data:image/png;base64,{pltCode}\" width=400><br>")
390
391 outFile.write("</body></html>\n")
392
393 print(
"DQ Report Created @",output)
394
395
396 return histResults
397
TGraphErrors * GetEntries(TH2F *histo)
std::string replace(std::string s, const std::string &s2, const std::string &s3)
IovVectorMap_t read(const Folder &theFolder, const SelectionCriterion &choice, const unsigned int limit=10)