552 def makeHTML(cls, dic, dicsum, doPickle=True, doDQSummary=True, doDQPlots=True, dqsumGRL="PHYS_StandardGRL_All_Good_25ns", dbbTag=(
"HEAD",
"HEAD")):
553 """ method returns a string (unicode is fine) of html code, with out tag <div>...</div> """
564 with timer(
"DQSummary"):
570 livetrigger =
'L1_EM30'
571 for r,run
in enumerate(dic[DataKey(
'Run')]):
572 ptag = dic[DataKey(
'Project tag')][r][0].value
575 livetrigger =
'L1_EM5'
576 elif 'data16_hip5TeV' in ptag
or 'data16_hip8TeV' in ptag:
578 livetrigger =
'L1_EM12'
579 elif 'data15' in ptag
or 'data16' in ptag
or 'data17' in ptag
or 'data18' in ptag:
581 livetrigger =
'L1_EM12'
586 warning +=
'<font color="red"> WARNING (DQSummary): The defect database tag "%s" must be a 2-element sequence. Will use "HEAD" tag instead!</font><br>' %(dbbTag)
587 dbbTag = (
"HEAD",
"HEAD")
591 if dbbTag[0]
not in [
'HEAD'] + ddb.defects_tags:
592 warning +=
'<font color="red"> WARNING (DQSummary): The defined defect tag "%s" is not defined in defect database. Will use "HEAD" tag instead!</font><br>' %(dbbTag[0])
593 dbbTag = (
"HEAD", dbbTag[1])
595 if dbbTag[1]
not in [
'HEAD'] + ddb.logics_tags:
596 warning +=
'<font color="red"> WARNING (DQSummary): The defined logic tag "%s" is not defined in defect database. Will use "HEAD" tag instead!</font><br>' %(dbbTag[1])
597 dbbTag = (dbbTag[0],
"HEAD")
599 ddb = DefectsDB(tag=(dbbTag[0], dbbTag[1]))
601 if dqsumGRL
in ddb.virtual_defect_logics.keys():
602 defects_primary_grl = ddb.resolve_primary_defects(ddb._resolve_evaluation_order([dqsumGRL]))
604 warning +=
'<font color="red"> WARNING (DQSummary): The defined virtual defect or GRL "%s" is not defined in logic. Will use no GRL for calculation!</font><br>' %(dqsumGRL)
609 if 'Cosmics' in dqsumGRL:
620 dicsum[DataKey(
'TotalLumi')] = [0.,0.]
623 dicsum[DataKey(
'TotalNotReady')] = 0.
624 dicsum[DataKey(
'TotalBusy')] = 0.
627 detectors = [
'ALFA',
'LCD',
'ZDC',
'BCM',
'PIXEL',
'SCT',
'TRT',
'LAR',
'TILE',
628 'MS_CSC',
'MS_RPC',
'MS_TGC',
'MS_MDT',
'MBTS',
'TRIG',
'GLOBAL']
629 detectors_color = {
'ALFA':kYellow-9,
'LCD':kYellow-7,
'ZDC':kYellow-4,
630 'BCM':kOrange+1,
'PIXEL':kOrange-3,
'SCT':kOrange+7,
'TRT':kRed-3,
631 'LAR':kRed+2,
'TILE':kPink-7,
632 'MS_CSC':kBlue+4,
'MS_RPC':kBlue-2,
'MS_MDT':kAzure+3,
'MS_TGC':kBlue-9,
'MBTS':kBlue-6,
633 'TRIG':kGreen-3,
'GLOBAL':kGreen+3,
'_TOTAL':TColor.GetColor(
"#585858")}
634 detectors_lumiloss = {}
635 detectors_affectedLBs = {}
638 detectors_lumiloss[d]={}
639 detectors_affectedLBs[d]={}
642 detectors_lumiloss[
'_TOTAL']={}
643 detectors_affectedLBs[
'_TOTAL']={}
646 performances = [
'LUMI',
'ID',
'CALO',
'TAU',
'EGAMMA',
'JET',
'MET',
'MCP',
'BTAG']
647 performances_color = {
'LUMI':kYellow-9,
'ID':kOrange+7,
'CALO':kRed+2,
'TAU':kBlue+2,
648 'EGAMMA':kGreen-3,
'JET':kAzure+3,
'MET':kGreen-6,
'MCP':kBlue-3,
649 'BTAG':kPink,
'_TOTAL':TColor.GetColor(
"#585858")}
650 performances_lumiloss = {}
651 performances_affectedLBs = {}
653 for p
in performances:
654 performances_lumiloss[p]={}
655 performances_affectedLBs[p]={}
658 performances_lumiloss[
'_TOTAL']={}
659 performances_affectedLBs[
'_TOTAL']={}
664 columns = [
'Run Info',
'ES1',
'BLK']
665 global_systems = [
'Calo',
'Tracking',
'Muon',
'Trigger & Lumi']
666 columns += global_systems
669 with timer(
'Create summary results table'):
671 summarytable =
'<div><table id="resulttable" class="resulttable" style="margin-left: 13px; margin-right: 13px;" >'
673 summarytable +=
'<tr>'
674 summarytable +=
'<th colspan="1"></th>'
677 summarytable +=
'<th colspan="2">Missing Sign-Off</th>'
679 summarytable +=
'<th colspan="%i">Defects in ATLAS Ready<br>'%len(global_systems)
680 summarytable +=
'<div style="font-size: xx-small; cursor: pointer;" onclick="toggle_dq(this)">[show tolerable]</div></th>'
681 summarytable +=
'</tr>'
684 summarytable +=
'<tr>'
687 summarytable +=
'<th>%s</th>'%item
688 summarytable +=
'</tr>'
690 totalNumberOfReadyLB = 0
695 for r,run
in enumerate(dic[DataKey(
'Run')]):
696 tot_nlb += dic[DataKey(
'#LB')][r][0]
698 runsWithoutReadyForPhysics = []
699 runsWithoutOfflineLumiInfo = []
702 for r,run
in enumerate(dic[DataKey(
'Run')]):
707 nlb = dic[DataKey(
'#LB')][r][0]
710 readylb =
range(1,nlb+1)
714 totalNumberOfReadyLB += len(readylb)
717 if len(readylb) == 0:
718 runsWithoutReadyForPhysics += [
str(run) ]
724 lumiperlb = dict(zip(readylb, [100./tot_nlb] * nlb))
725 livefrac = dict(zip(readylb, [1] * nlb))
730 if len(lumiperlb) == 0:
731 runsWithoutOfflineLumiInfo += [
str(run) ]
736 for lbl
in lumiperlb:
738 if lbl
not in readylb:
740 if lbl
not in livefrac:
744 lumiperlb[lbl]*=livefrac[lbl]
749 for i,item
in enumerate(columns):
754 ptag = dic[DataKey(
'Project tag')][r][0].value
755 year =
'20'+ptag.split(
'_')[0][4:]
756 DP = ARQ_COMA.get_periods_for_run(run)
759 content[
'Run Info'] =
'<b>%s</b> <font style="font-size:10px;">data period <b>%s</b></font><br>'%(
str(run),dp)
762 tobeignored = [
'ALFA',
'ZDC',
'LCD']
764 if re.match(
'data1[3-9]', ptag[0:6]):
768 target=
'target="blank"'
769 somelinks =
'<font style="font-size:10px;color:#0000FF">'
770 somelinks +=
'<a href="https://atlas-tagservices.cern.ch/tagservices/RunBrowser/runBrowserReport/runBrowserReport.php?runs=%s&pn=%s&fnt=%s" %s>COMA</a>'%(run,dp,ptag,target)
771 somelinks +=
', <a href="https://atlasdqm.web.cern.ch/atlasdqm/DQBrowser/makeMatrix.php?selection=All&run=%s&runup=&dbinstance=CONDBR2_DCS&tag=DEFAULT" %s>DCS</a>'%(run,target)
772 somelinks +=
', <a href="https://atlas.web.cern.ch/Atlas/GROUPS/DATAPREPARATION/DataSummary/%s/run.py?run=%s" %s>DS</a>'%(year,run,target)
773 somelinks +=
', <a href="https://atlasdqm.cern.ch/defectentry/?run=%s" %s>defects</a>'%(run,target)
774 somelinks +=
'<br><a href="https://atlasdqm.cern.ch/dqsignoff/%s/" %s>DQlogbook</a>'%(run,target)
775 somelinks +=
', <a href="https://atlasdqm.cern.ch/webdisplay/tier0?lowrun=%s&highrun=%s&show_all=on" %s>DQWebDisplay</a>'%(run,run,target)
776 somelinks +=
'</font><br>'
778 content[
'Run Info'] += somelinks
781 rundate = dic[DataKey(
'Start and endtime')][r].
split(
",")
782 tstart = time.strftime(
"%a, %d %b %Y %H:%M:%S",time.localtime(
float(rundate[0])))
783 tend = time.strftime(
"%a, %d %b %Y %H:%M:%S",time.localtime(
float(rundate[1])))
784 content[
'Run Info'] +=
'<font style="font-size:10px;">'
785 content[
'Run Info'] +=
'<br>Start: <font style="color:#4C7D7E;">%s</font>'%tstart
786 content[
'Run Info'] +=
'<br>End: <font style="color:#4C7D7E;">%s</font>'%tend
787 content[
'Run Info'] +=
'</font>'
795 lhctable =
'<br><font style="font-size:10px;">'
796 lhctable +=
'<table><tr><td>'
797 lhctable +=
'<b>Stable Beam</b><br>%s'%
'<br>'.
join([
'%s-%s: <font style="color:#4C7D7E">%s</font>'%(b.startlb,b.endlb,b.value)
for b
in dic[DataKey(
'lhc:stablebeams')][r]])
798 lhctable +=
'</td><td>'
799 lhctable +=
'<b>ATLAS Ready</b><br>%s'%
'<br>'.
join([
'%s-%s: <font style="color:#4C7D7E">%s</font>'%(b.startlb,b.endlb,b.value)
for b
in dic[DataKey(
'Ready for physics')][r]])
800 lhctable +=
'</td></tr></table>'
801 lhctable +=
'</font>'
802 content[
'Run Info'] += lhctable
806 dicsum[DataKey(
'TotalLumi')][0]+= lumitot
807 content[
'Run Info'] +=
'<font style="font-size:10px;">'
808 content[
'Run Info'] +=
'Ready Recorded: <b>%.2f</b> %s'%(lumitot,unit)
809 content[
'Run Info'] +=
'</font><br>'
812 defects = dic[DataKey(
'DQ')][r]
814 defects_primary = [d
for d
in defects
if d.value.primary]
819 detectors_lumiloss[d][run]= 0.
820 detectors_affectedLBs[d][run]=[]
821 for p
in performances:
822 performances_lumiloss[p][run]=0.
823 performances_affectedLBs[p][run]=[]
826 detectors_lumiloss[
'_TOTAL'][run]= 0.
827 detectors_affectedLBs[
'_TOTAL'][run]=[]
828 performances_lumiloss[
'_TOTAL'][run]= 0.
829 performances_affectedLBs[
'_TOTAL'][run]=[]
832 total_affectedLBs = []
837 ListOfIntolerableDefects = {}
838 ListOfTolerableDefects = {}
839 ListOfIgnoredDefects = {}
842 for item
in defects_primary:
843 defect = item.value.defect
844 comment = item.value.comment
845 user = item.value.user
846 startlb = item.startlb
848 tolerable = item.value.tolerable
853 if not dqsumGRL ==
'no' and not tolerable:
854 if defect
not in defects_primary_grl:
858 if 'GLOBAL_NOTREADY' in defect:
859 GlobalNotReady += [ lb
for lb
in range(startlb,endlb)]
861 if 'GLOBAL_BUSY' in defect:
862 GlobalBusy += [ lb
for lb
in range(startlb,endlb)
if lb
in readylb]
867 if 'UNCHECKED_FINAL' in defect:
871 if 'BULK_UNCHECKED' in defect:
872 short = defect.split(
'_UNCHECKED')[0]
873 content[
'BLK']+=
'<font style="font-size:8px;font-weight:bold;">%s</font><br>'%short
876 if 'UNCHECKED' in defect:
877 short = defect.split(
'_UNCHECKED')[0]
878 content[
'ES1']+=
'<font style="font-size:8px;font-weight:bold;">%s</font><br>'%short
883 print (
'run %s: this defect is fishy %s '%(run,defect))
887 word = defect.split(
'_')
891 if cpdet
not in detectors
and cpdet
not in performances:
892 print (
'This system is not included: %s (%s)'%(cpdet,defect))
897 RangeDefect = [ lb
for lb
in range(startlb,endlb)
if lb
in readylb ]
898 if len(RangeDefect)>0:
900 if defect
not in ListOfIntolerableDefects:
901 ListOfIntolerableDefects[defect]= [[],
'']
902 ListOfIntolerableDefects[defect][0] = RangeDefect
903 ListOfIntolerableDefects[defect][1] = user+
':'+comment
905 ListOfIntolerableDefects[defect][0]+= RangeDefect
906 if comment
not in ListOfIntolerableDefects[defect][1]:
907 ListOfIntolerableDefects[defect][1]+=
' '+user+
':'+comment
910 if cpdet
in tobeignored:
914 if cpdet
in detectors:
915 detectors_affectedLBs[cpdet][run]+=RangeDefect
916 if cpdet
in performances:
917 performances_affectedLBs[cpdet][run]+=RangeDefect
918 total_affectedLBs += RangeDefect
921 if defect
not in ListOfIgnoredDefects:
922 ListOfIgnoredDefects[defect]= [[],
'']
923 ListOfIgnoredDefects[defect][0]= RangeDefect
924 ListOfIgnoredDefects[defect][1]=
'[%s]:%s '%(user,comment)
926 ListOfIgnoredDefects[defect][0]+= RangeDefect
927 if comment
not in ListOfIgnoredDefects[defect][1]:
928 ListOfIgnoredDefects[defect][1] +=
'[%s]:%s '%(user,comment)
931 RangeDefect = [ lb
for lb
in range(startlb,endlb)
if lb
in readylb]
932 if len(RangeDefect)>0:
933 if defect
not in ListOfTolerableDefects:
934 ListOfTolerableDefects[defect]= [[],
'']
935 ListOfTolerableDefects[defect][0]= RangeDefect
936 ListOfTolerableDefects[defect][1]=
'[%s]:%s '%(user,comment)
938 ListOfTolerableDefects[defect][0]+= RangeDefect
939 if comment
not in ListOfTolerableDefects[defect][1]:
940 ListOfTolerableDefects[defect][1] +=
'[%s]:%s '%(user,comment)
944 for item
in global_systems:
945 content[item]=
'<table class="dqdefects">'
947 for item
in ListOfIntolerableDefects:
949 if system
not in global_systems:
951 lbs =
listify(ListOfIntolerableDefects[item][0])
952 tip = ListOfIntolerableDefects[item][1]
953 tipkey =
"dqcomment_int_%s_%s"%(item,run)
954 Run.addGlobalToolTip(tipkey,tip)
955 content[system]+=
'<tr class="showTip %s" >'%(tipkey)
956 content[system]+=
'<td class="intolerable">%s</td>'%(item)
957 content[system]+=
'<td class="lb">%s</td>'%(lbs)
958 content[system]+=
'</tr>'
960 for item
in ListOfIgnoredDefects:
962 if system
not in global_systems:
964 lbs =
listify(ListOfIgnoredDefects[item][0])
965 tip = ListOfIgnoredDefects[item][1]
966 tipkey =
"dqcomment_ign_%s_%s"%(item,run)
967 Run.addGlobalToolTip(tipkey,tip)
968 content[system]+=
'<tr class="showTip %s" >'%(tipkey)
969 content[system]+=
'<td class="ignored">%s</td>'%(item)
970 content[system]+=
'<td class="lb">%s</td>'%(lbs)
971 content[system]+=
'</tr>'
973 for item
in ListOfTolerableDefects:
975 if system
not in global_systems:
977 lbs =
listify(ListOfTolerableDefects[item][0])
978 tip = ListOfTolerableDefects[item][1]
979 tipkey =
"dqcomment_tol_%s_%s"%(item,run)
980 Run.addGlobalToolTip(tipkey,tip)
981 content[system]+=
'<tr class="showTip %s tolerable" style="visibility:collapse;">'%(tipkey)
982 content[system]+=
'<td>%s</td>'%(item)
983 content[system]+=
'<td class="lb">%s</td>'%(lbs)
984 content[system]+=
'</tr>'
987 for item
in global_systems:
988 content[item]+=
'</table><br>'
993 for gsys
in global_systems:
997 for defect
in ListOfTolerableDefects:
999 word = defect.split(
'_')
1002 cpdet +=
"_"+word[1]
1003 if cpdet
in tobeignored:
1006 tol[defect]=ListOfTolerableDefects[defect][0]
1007 for defect
in ListOfIntolerableDefects:
1009 word = defect.split(
'_')
1012 cpdet +=
"_"+word[1]
1013 if cpdet
in tobeignored:
1016 int[defect]=ListOfIntolerableDefects[defect][0]
1017 for defect
in ListOfIgnoredDefects:
1019 word = defect.split(
'_')
1022 cpdet +=
"_"+word[1]
1023 if cpdet
in tobeignored:
1026 ign[defect]=ListOfIgnoredDefects[defect][0]
1032 title =
"Click to zoom"
1033 wincontent =
"<img src="%s" height=%s/>"%(h,imgsize)
1034 openwin =
"javascript:openLargeWindow('Print1','"
1035 openwin +=
"<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">"
1036 openwin +=
"<html xmlns:"my"><head><title>Defects for run %s</title></head>"%run
1037 openwin +=
"<body style="background-color:#ffffff">%s</body></html>"%wincontent
1039 content[gsys]+=
'<a title="%s" href="%s" ><img src="%s" height=%s/></a>'%(title,openwin,h,thumbsize)
1043 if not GlobalNotReady:
1044 for lb
in GlobalNotReady:
1045 _lblumi += lumiperlb[lb]
1046 dicsum[DataKey(
'TotalNotReady')] += _lblumi
1047 content[
'Run Info'] +=
'<font style="font-size:10px;">'
1048 content[
'Run Info'] +=
'Global Not Ready: <b>%.2f</b> %s'%(_lblumi,unit)
1049 content[
'Run Info'] +=
'</font><br>'
1053 for lb
in GlobalBusy:
1054 _lblumi += lumiperlb[lb]
1055 dicsum[DataKey(
'TotalBusy')]+= _lblumi
1056 content[
'Run Info'] +=
'<font style="font-size:10px;">'
1057 content[
'Run Info'] +=
'Global Busy: <b>%.2f</b> %s'%(_lblumi,unit)
1058 content[
'Run Info'] +=
'</font><br>'
1062 if len(detectors_affectedLBs[d][run]) == 0:
1065 dll =
list(
set(detectors_affectedLBs[d][run]))
1066 detectors_affectedLBs[
'_TOTAL'][run]+= dll
1068 detectors_lumiloss[d][run]+= lumiperlb[lb]
1069 for p
in performances:
1070 if len(performances_affectedLBs[p][run]) == 0:
1073 pll =
list(
set(performances_affectedLBs[p][run]))
1074 performances_affectedLBs[
'_TOTAL'][run]+= pll
1076 performances_lumiloss[p][run]+= lumiperlb[lb]
1080 detectors_affectedLBs[
'_TOTAL'][run]=
list(
set(detectors_affectedLBs[
'_TOTAL'][run]))
1081 performances_affectedLBs[
'_TOTAL'][run]=
list(
set(performances_affectedLBs[
'_TOTAL'][run]))
1082 total_affectedLBs =
list(
set(total_affectedLBs))
1083 totallossperrun = 0.
1085 for lb
in total_affectedLBs:
1086 totallossperrun += lumiperlb[lb]
1087 dicsum[DataKey(
'TotalLumi')][1] += totallossperrun
1089 for lb
in detectors_affectedLBs[
'_TOTAL'][run]:
1090 detectors_lumiloss[
'_TOTAL'][run] += lumiperlb[lb]
1092 for lb
in performances_affectedLBs[
'_TOTAL'][run]:
1093 performances_lumiloss[
'_TOTAL'][run] += lumiperlb[lb]
1096 content[
'Run Info'] +=
'<font style="font-size:10px;">'
1097 content[
'Run Info'] +=
'DQ Lumi Loss: <b>%.2f</b> %s'%(totallossperrun,unit)
1098 content[
'Run Info'] +=
'</font><br>'
1101 summarytable+=
'<tr class="out2">'
1102 for item
in columns:
1103 summarytable+=
'<td>%s</td>'%content[item]
1104 summarytable+=
'</tr>'
1107 summarytable +=
'</table></div><br>'
1109 if runsWithoutReadyForPhysics:
1110 warning +=
'<div style="margin-left: 13px; color: darkred; width: 90%%;">WARNING! The following runs have no lumi block with ATLAS Ready: %s</div><br>' %
", ".
join(runsWithoutReadyForPhysics)
1111 if runsWithoutOfflineLumiInfo:
1112 warning +=
'<div style="margin-left: 13px; color: darkred; width: 90%%;">WARNING! The following runs have no offline lumi info: %s</div><br>' %
", ".
join(runsWithoutOfflineLumiInfo)
1122 if totalNumberOfReadyLB > 0:
1130 title =
"Click to zoom"
1133 summaryplot +=
'<table align="center" width="90%">'
1134 summaryplot +=
'<th>'
1138 wincontent =
"<img src="%s" width=%s/>"%(p,imgsize)
1141 openwincmd =
"javascript:openWindow('Print%i','"%(run)
1142 openwincmd +=
"<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">"
1143 openwincmd +=
"<html xmlns:"my"><head><title>DQ Summary Plot - Run %s</title></head>"%(run)
1144 openwincmd +=
"<body style="background-color:#ffffff">%s</body></html>"%wincontent
1147 summaryplot+=
'<td><a title="%s" href="%s" ><img src="%s" height=%s/></a></td>'%(title,openwincmd,p,thumbsize)
1150 summaryplot +=
'</th></table>'
1161 totalLumiRecorded = dicsum[DataKey(
'TotalLumi')][0]
1162 totalLumiLoss = dicsum[DataKey(
'TotalLumi')][1]
1163 lumiLossFraction = (100. * totalLumiLoss / totalLumiRecorded)
if totalLumiRecorded > 0
else 0
1166 print (
'+++++++++++++++++++++++++ Summary +++++++++++++++++++')
1167 print (
' Total Ready Recorded Luminosity: %.2f %s-1' % ( totalLumiRecorded, unit ))
1168 print (
' Total Luminosity Loss (ATLAS Ready) : %.2f %s-1' % ( totalLumiLoss, unit ))
1169 print (
' Total Global Not Ready (Stable Beams): %.2f %s-1'%(dicsum[DataKey(
'TotalNotReady')],unit))
1170 print (
' Total Global Busy (Stable Beams): %.2f %s-1'%(dicsum[DataKey(
'TotalBusy')],unit))
1171 print (
'+++++++++++++++++++++++++++++++++++++++++++++++++++++')
1175 if totalNumberOfReadyLB>0:
1176 summaryinfo =
'<table align="center" style="font-size:80%;"><tr>'
1177 summaryinfo +=
'<td> Total Luminosity Loss: <b>%.2f %s</b> (%.2f%%)' % ( totalLumiLoss, unit, lumiLossFraction )
1178 summaryinfo +=
'<br>Excluded Systems: %s</td></tr>' % tobeignored
1180 summaryinfo +=
'<tr><td style="font-size:70%%">using %s // %s</td></tr>'%(lumifolder.split(
':')[2],livetrigger)
1181 summaryinfo +=
'</table>'
1183 summaryinfo =
'<table align="center" style="font-size:80%;"><tr>'
1184 summaryinfo +=
'<td> No ready lumi block </td>'
1185 summaryinfo +=
'</tr></table>'
1191 pagecontent = summaryplot +
'<br>'
1192 pagecontent += summaryinfo +
'<br>'
1194 pagecontent += summarytable
1195 pagecontent += warning