551 def makeHTML(cls, dic, dicsum, doPickle=True, doDQSummary=True, doDQPlots=True, dqsumGRL="PHYS_StandardGRL_All_Good_25ns", dbbTag=(
"HEAD",
"HEAD")):
552 """ method returns a string (unicode is fine) of html code, with out tag <div>...</div> """
563 with timer(
"DQSummary"):
569 livetrigger =
'L1_EM30'
570 for r,run
in enumerate(dic[DataKey(
'Run')]):
571 ptag = dic[DataKey(
'Project tag')][r][0].value
574 livetrigger =
'L1_EM5'
575 elif 'data16_hip5TeV' in ptag
or 'data16_hip8TeV' in ptag:
577 livetrigger =
'L1_EM12'
578 elif 'data15' in ptag
or 'data16' in ptag
or 'data17' in ptag
or 'data18' in ptag:
580 livetrigger =
'L1_EM12'
585 warning +=
'<font color="red"> WARNING (DQSummary): The defect database tag "%s" must be a 2-element sequence. Will use "HEAD" tag instead!</font><br>' %(dbbTag)
586 dbbTag = (
"HEAD",
"HEAD")
590 if dbbTag[0]
not in [
'HEAD'] + ddb.defects_tags:
591 warning +=
'<font color="red"> WARNING (DQSummary): The defined defect tag "%s" is not defined in defect database. Will use "HEAD" tag instead!</font><br>' %(dbbTag[0])
592 dbbTag = (
"HEAD", dbbTag[1])
594 if dbbTag[1]
not in [
'HEAD'] + ddb.logics_tags:
595 warning +=
'<font color="red"> WARNING (DQSummary): The defined logic tag "%s" is not defined in defect database. Will use "HEAD" tag instead!</font><br>' %(dbbTag[1])
596 dbbTag = (dbbTag[0],
"HEAD")
598 ddb = DefectsDB(tag=(dbbTag[0], dbbTag[1]))
600 if dqsumGRL
in ddb.virtual_defect_logics.keys():
601 defects_primary_grl = ddb.resolve_primary_defects(ddb._resolve_evaluation_order([dqsumGRL]))
603 warning +=
'<font color="red"> WARNING (DQSummary): The defined virtual defect or GRL "%s" is not defined in logic. Will use no GRL for calculation!</font><br>' %(dqsumGRL)
608 if 'Cosmics' in dqsumGRL:
619 dicsum[DataKey(
'TotalLumi')] = [0.,0.]
622 dicsum[DataKey(
'TotalNotReady')] = 0.
623 dicsum[DataKey(
'TotalBusy')] = 0.
626 detectors = [
'ALFA',
'LCD',
'ZDC',
'BCM',
'PIXEL',
'SCT',
'TRT',
'LAR',
'TILE',
627 'MS_CSC',
'MS_RPC',
'MS_TGC',
'MS_MDT',
'MBTS',
'TRIG',
'GLOBAL']
628 detectors_color = {
'ALFA':kYellow-9,
'LCD':kYellow-7,
'ZDC':kYellow-4,
629 'BCM':kOrange+1,
'PIXEL':kOrange-3,
'SCT':kOrange+7,
'TRT':kRed-3,
630 'LAR':kRed+2,
'TILE':kPink-7,
631 'MS_CSC':kBlue+4,
'MS_RPC':kBlue-2,
'MS_MDT':kAzure+3,
'MS_TGC':kBlue-9,
'MBTS':kBlue-6,
632 'TRIG':kGreen-3,
'GLOBAL':kGreen+3,
'_TOTAL':TColor.GetColor(
"#585858")}
633 detectors_lumiloss = {}
634 detectors_affectedLBs = {}
637 detectors_lumiloss[d]={}
638 detectors_affectedLBs[d]={}
641 detectors_lumiloss[
'_TOTAL']={}
642 detectors_affectedLBs[
'_TOTAL']={}
645 performances = [
'LUMI',
'ID',
'CALO',
'TAU',
'EGAMMA',
'JET',
'MET',
'MCP',
'BTAG']
646 performances_color = {
'LUMI':kYellow-9,
'ID':kOrange+7,
'CALO':kRed+2,
'TAU':kBlue+2,
647 'EGAMMA':kGreen-3,
'JET':kAzure+3,
'MET':kGreen-6,
'MCP':kBlue-3,
648 'BTAG':kPink,
'_TOTAL':TColor.GetColor(
"#585858")}
649 performances_lumiloss = {}
650 performances_affectedLBs = {}
652 for p
in performances:
653 performances_lumiloss[p]={}
654 performances_affectedLBs[p]={}
657 performances_lumiloss[
'_TOTAL']={}
658 performances_affectedLBs[
'_TOTAL']={}
663 columns = [
'Run Info',
'ES1',
'BLK']
664 global_systems = [
'Calo',
'Tracking',
'Muon',
'Trigger & Lumi']
665 columns += global_systems
668 with timer(
'Create summary results table'):
670 summarytable =
'<div><table id="resulttable" class="resulttable" style="margin-left: 13px; margin-right: 13px;" >'
672 summarytable +=
'<tr>'
673 summarytable +=
'<th colspan="1"></th>'
676 summarytable +=
'<th colspan="2">Missing Sign-Off</th>'
678 summarytable +=
'<th colspan="%i">Defects in ATLAS Ready<br>'%len(global_systems)
679 summarytable +=
'<div style="font-size: xx-small; cursor: pointer;" onclick="toggle_dq(this)">[show tolerable]</div></th>'
680 summarytable +=
'</tr>'
683 summarytable +=
'<tr>'
686 summarytable +=
'<th>%s</th>'%item
687 summarytable +=
'</tr>'
689 totalNumberOfReadyLB = 0
694 for r,run
in enumerate(dic[DataKey(
'Run')]):
695 tot_nlb += dic[DataKey(
'#LB')][r][0]
697 runsWithoutReadyForPhysics = []
698 runsWithoutOfflineLumiInfo = []
701 for r,run
in enumerate(dic[DataKey(
'Run')]):
706 nlb = dic[DataKey(
'#LB')][r][0]
709 readylb =
range(1,nlb+1)
713 totalNumberOfReadyLB += len(readylb)
716 if len(readylb) == 0:
717 runsWithoutReadyForPhysics += [
str(run) ]
723 lumiperlb = dict(zip(readylb, [100./tot_nlb] * nlb))
724 livefrac = dict(zip(readylb, [1] * nlb))
729 if len(lumiperlb) == 0:
730 runsWithoutOfflineLumiInfo += [
str(run) ]
735 for lbl
in lumiperlb:
737 if lbl
not in readylb:
739 if lbl
not in livefrac:
743 lumiperlb[lbl]*=livefrac[lbl]
748 for i,item
in enumerate(columns):
753 ptag = dic[DataKey(
'Project tag')][r][0].value
754 year =
'20'+ptag.split(
'_')[0][4:]
755 DP = ARQ_COMA.get_periods_for_run(run)
758 content[
'Run Info'] =
'<b>%s</b> <font style="font-size:10px;">data period <b>%s</b></font><br>'%(
str(run),dp)
761 tobeignored = [
'ALFA',
'ZDC',
'LCD']
763 if re.match(
'data1[3-9]', ptag[0:6]):
767 target=
'target="blank"'
768 somelinks =
'<font style="font-size:10px;color:#0000FF">'
769 somelinks +=
'<a href="https://atlas-tagservices.cern.ch/tagservices/RunBrowser/runBrowserReport/runBrowserReport.php?runs=%s&pn=%s&fnt=%s" %s>COMA</a>'%(run,dp,ptag,target)
770 somelinks +=
', <a href="https://atlasdqm.web.cern.ch/atlasdqm/DQBrowser/makeMatrix.php?selection=All&run=%s&runup=&dbinstance=CONDBR2_DCS&tag=DEFAULT" %s>DCS</a>'%(run,target)
771 somelinks +=
', <a href="https://atlas.web.cern.ch/Atlas/GROUPS/DATAPREPARATION/DataSummary/%s/run.py?run=%s" %s>DS</a>'%(year,run,target)
772 somelinks +=
', <a href="https://atlasdqm.cern.ch/defectentry/?run=%s" %s>defects</a>'%(run,target)
773 somelinks +=
'<br><a href="https://atlasdqm.cern.ch/dqsignoff/%s/" %s>DQlogbook</a>'%(run,target)
774 somelinks +=
', <a href="https://atlasdqm.cern.ch/webdisplay/tier0?lowrun=%s&highrun=%s&show_all=on" %s>DQWebDisplay</a>'%(run,run,target)
775 somelinks +=
'</font><br>'
777 content[
'Run Info'] += somelinks
780 rundate = dic[DataKey(
'Start and endtime')][r].
split(
",")
781 tstart = time.strftime(
"%a, %d %b %Y %H:%M:%S",time.localtime(
float(rundate[0])))
782 tend = time.strftime(
"%a, %d %b %Y %H:%M:%S",time.localtime(
float(rundate[1])))
783 content[
'Run Info'] +=
'<font style="font-size:10px;">'
784 content[
'Run Info'] +=
'<br>Start: <font style="color:#4C7D7E;">%s</font>'%tstart
785 content[
'Run Info'] +=
'<br>End: <font style="color:#4C7D7E;">%s</font>'%tend
786 content[
'Run Info'] +=
'</font>'
794 lhctable =
'<br><font style="font-size:10px;">'
795 lhctable +=
'<table><tr><td>'
796 lhctable +=
'<b>Stable Beam</b><br>%s'%
'<br>'.
join([
'%s-%s: <font style="color:#4C7D7E">%s</font>'%(b.startlb,b.endlb,b.value)
for b
in dic[DataKey(
'lhc:stablebeams')][r]])
797 lhctable +=
'</td><td>'
798 lhctable +=
'<b>ATLAS Ready</b><br>%s'%
'<br>'.
join([
'%s-%s: <font style="color:#4C7D7E">%s</font>'%(b.startlb,b.endlb,b.value)
for b
in dic[DataKey(
'Ready for physics')][r]])
799 lhctable +=
'</td></tr></table>'
800 lhctable +=
'</font>'
801 content[
'Run Info'] += lhctable
805 dicsum[DataKey(
'TotalLumi')][0]+= lumitot
806 content[
'Run Info'] +=
'<font style="font-size:10px;">'
807 content[
'Run Info'] +=
'Ready Recorded: <b>%.2f</b> %s'%(lumitot,unit)
808 content[
'Run Info'] +=
'</font><br>'
811 defects = dic[DataKey(
'DQ')][r]
813 defects_primary = [d
for d
in defects
if d.value.primary]
818 detectors_lumiloss[d][run]= 0.
819 detectors_affectedLBs[d][run]=[]
820 for p
in performances:
821 performances_lumiloss[p][run]=0.
822 performances_affectedLBs[p][run]=[]
825 detectors_lumiloss[
'_TOTAL'][run]= 0.
826 detectors_affectedLBs[
'_TOTAL'][run]=[]
827 performances_lumiloss[
'_TOTAL'][run]= 0.
828 performances_affectedLBs[
'_TOTAL'][run]=[]
831 total_affectedLBs = []
836 ListOfIntolerableDefects = {}
837 ListOfTolerableDefects = {}
838 ListOfIgnoredDefects = {}
841 for item
in defects_primary:
842 defect = item.value.defect
843 comment = item.value.comment
844 user = item.value.user
845 startlb = item.startlb
847 tolerable = item.value.tolerable
852 if not dqsumGRL ==
'no' and not tolerable:
853 if defect
not in defects_primary_grl:
857 if 'GLOBAL_NOTREADY' in defect:
858 GlobalNotReady += [ lb
for lb
in range(startlb,endlb)]
860 if 'GLOBAL_BUSY' in defect:
861 GlobalBusy += [ lb
for lb
in range(startlb,endlb)
if lb
in readylb]
866 if 'UNCHECKED_FINAL' in defect:
870 if 'BULK_UNCHECKED' in defect:
871 short = defect.split(
'_UNCHECKED')[0]
872 content[
'BLK']+=
'<font style="font-size:8px;font-weight:bold;">%s</font><br>'%short
875 if 'UNCHECKED' in defect:
876 short = defect.split(
'_UNCHECKED')[0]
877 content[
'ES1']+=
'<font style="font-size:8px;font-weight:bold;">%s</font><br>'%short
882 print (
'run %s: this defect is fishy %s '%(run,defect))
886 word = defect.split(
'_')
890 if cpdet
not in detectors
and cpdet
not in performances:
891 print (
'This system is not included: %s (%s)'%(cpdet,defect))
896 RangeDefect = [ lb
for lb
in range(startlb,endlb)
if lb
in readylb ]
897 if len(RangeDefect)>0:
899 if defect
not in ListOfIntolerableDefects:
900 ListOfIntolerableDefects[defect]= [[],
'']
901 ListOfIntolerableDefects[defect][0] = RangeDefect
902 ListOfIntolerableDefects[defect][1] = user+
':'+comment
904 ListOfIntolerableDefects[defect][0]+= RangeDefect
905 if comment
not in ListOfIntolerableDefects[defect][1]:
906 ListOfIntolerableDefects[defect][1]+=
' '+user+
':'+comment
909 if cpdet
in tobeignored:
913 if cpdet
in detectors:
914 detectors_affectedLBs[cpdet][run]+=RangeDefect
915 if cpdet
in performances:
916 performances_affectedLBs[cpdet][run]+=RangeDefect
917 total_affectedLBs += RangeDefect
920 if defect
not in ListOfIgnoredDefects:
921 ListOfIgnoredDefects[defect]= [[],
'']
922 ListOfIgnoredDefects[defect][0]= RangeDefect
923 ListOfIgnoredDefects[defect][1]=
'[%s]:%s '%(user,comment)
925 ListOfIgnoredDefects[defect][0]+= RangeDefect
926 if comment
not in ListOfIgnoredDefects[defect][1]:
927 ListOfIgnoredDefects[defect][1] +=
'[%s]:%s '%(user,comment)
930 RangeDefect = [ lb
for lb
in range(startlb,endlb)
if lb
in readylb]
931 if len(RangeDefect)>0:
932 if defect
not in ListOfTolerableDefects:
933 ListOfTolerableDefects[defect]= [[],
'']
934 ListOfTolerableDefects[defect][0]= RangeDefect
935 ListOfTolerableDefects[defect][1]=
'[%s]:%s '%(user,comment)
937 ListOfTolerableDefects[defect][0]+= RangeDefect
938 if comment
not in ListOfTolerableDefects[defect][1]:
939 ListOfTolerableDefects[defect][1] +=
'[%s]:%s '%(user,comment)
943 for item
in global_systems:
944 content[item]=
'<table class="dqdefects">'
946 for item
in ListOfIntolerableDefects:
948 if system
not in global_systems:
950 lbs =
listify(ListOfIntolerableDefects[item][0])
951 tip = ListOfIntolerableDefects[item][1]
952 tipkey =
"dqcomment_int_%s_%s"%(item,run)
953 Run.addGlobalToolTip(tipkey,tip)
954 content[system]+=
'<tr class="showTip %s" >'%(tipkey)
955 content[system]+=
'<td class="intolerable">%s</td>'%(item)
956 content[system]+=
'<td class="lb">%s</td>'%(lbs)
957 content[system]+=
'</tr>'
959 for item
in ListOfIgnoredDefects:
961 if system
not in global_systems:
963 lbs =
listify(ListOfIgnoredDefects[item][0])
964 tip = ListOfIgnoredDefects[item][1]
965 tipkey =
"dqcomment_ign_%s_%s"%(item,run)
966 Run.addGlobalToolTip(tipkey,tip)
967 content[system]+=
'<tr class="showTip %s" >'%(tipkey)
968 content[system]+=
'<td class="ignored">%s</td>'%(item)
969 content[system]+=
'<td class="lb">%s</td>'%(lbs)
970 content[system]+=
'</tr>'
972 for item
in ListOfTolerableDefects:
974 if system
not in global_systems:
976 lbs =
listify(ListOfTolerableDefects[item][0])
977 tip = ListOfTolerableDefects[item][1]
978 tipkey =
"dqcomment_tol_%s_%s"%(item,run)
979 Run.addGlobalToolTip(tipkey,tip)
980 content[system]+=
'<tr class="showTip %s tolerable" style="visibility:collapse;">'%(tipkey)
981 content[system]+=
'<td>%s</td>'%(item)
982 content[system]+=
'<td class="lb">%s</td>'%(lbs)
983 content[system]+=
'</tr>'
986 for item
in global_systems:
987 content[item]+=
'</table><br>'
992 for gsys
in global_systems:
996 for defect
in ListOfTolerableDefects:
998 word = defect.split(
'_')
1001 cpdet +=
"_"+word[1]
1002 if cpdet
in tobeignored:
1005 tol[defect]=ListOfTolerableDefects[defect][0]
1006 for defect
in ListOfIntolerableDefects:
1008 word = defect.split(
'_')
1011 cpdet +=
"_"+word[1]
1012 if cpdet
in tobeignored:
1015 int[defect]=ListOfIntolerableDefects[defect][0]
1016 for defect
in ListOfIgnoredDefects:
1018 word = defect.split(
'_')
1021 cpdet +=
"_"+word[1]
1022 if cpdet
in tobeignored:
1025 ign[defect]=ListOfIgnoredDefects[defect][0]
1031 title =
"Click to zoom"
1032 wincontent =
"<img src="%s" height=%s/>"%(h,imgsize)
1033 openwin =
"javascript:openLargeWindow('Print1','"
1034 openwin +=
"<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">"
1035 openwin +=
"<html xmlns:"my"><head><title>Defects for run %s</title></head>"%run
1036 openwin +=
"<body style="background-color:#ffffff">%s</body></html>"%wincontent
1038 content[gsys]+=
'<a title="%s" href="%s" ><img src="%s" height=%s/></a>'%(title,openwin,h,thumbsize)
1042 if not GlobalNotReady:
1043 for lb
in GlobalNotReady:
1044 _lblumi += lumiperlb[lb]
1045 dicsum[DataKey(
'TotalNotReady')] += _lblumi
1046 content[
'Run Info'] +=
'<font style="font-size:10px;">'
1047 content[
'Run Info'] +=
'Global Not Ready: <b>%.2f</b> %s'%(_lblumi,unit)
1048 content[
'Run Info'] +=
'</font><br>'
1052 for lb
in GlobalBusy:
1053 _lblumi += lumiperlb[lb]
1054 dicsum[DataKey(
'TotalBusy')]+= _lblumi
1055 content[
'Run Info'] +=
'<font style="font-size:10px;">'
1056 content[
'Run Info'] +=
'Global Busy: <b>%.2f</b> %s'%(_lblumi,unit)
1057 content[
'Run Info'] +=
'</font><br>'
1061 if len(detectors_affectedLBs[d][run]) == 0:
1064 dll =
list(
set(detectors_affectedLBs[d][run]))
1065 detectors_affectedLBs[
'_TOTAL'][run]+= dll
1067 detectors_lumiloss[d][run]+= lumiperlb[lb]
1068 for p
in performances:
1069 if len(performances_affectedLBs[p][run]) == 0:
1072 pll =
list(
set(performances_affectedLBs[p][run]))
1073 performances_affectedLBs[
'_TOTAL'][run]+= pll
1075 performances_lumiloss[p][run]+= lumiperlb[lb]
1079 detectors_affectedLBs[
'_TOTAL'][run]=
list(
set(detectors_affectedLBs[
'_TOTAL'][run]))
1080 performances_affectedLBs[
'_TOTAL'][run]=
list(
set(performances_affectedLBs[
'_TOTAL'][run]))
1081 total_affectedLBs =
list(
set(total_affectedLBs))
1082 totallossperrun = 0.
1084 for lb
in total_affectedLBs:
1085 totallossperrun += lumiperlb[lb]
1086 dicsum[DataKey(
'TotalLumi')][1] += totallossperrun
1088 for lb
in detectors_affectedLBs[
'_TOTAL'][run]:
1089 detectors_lumiloss[
'_TOTAL'][run] += lumiperlb[lb]
1091 for lb
in performances_affectedLBs[
'_TOTAL'][run]:
1092 performances_lumiloss[
'_TOTAL'][run] += lumiperlb[lb]
1095 content[
'Run Info'] +=
'<font style="font-size:10px;">'
1096 content[
'Run Info'] +=
'DQ Lumi Loss: <b>%.2f</b> %s'%(totallossperrun,unit)
1097 content[
'Run Info'] +=
'</font><br>'
1100 summarytable+=
'<tr class="out2">'
1101 for item
in columns:
1102 summarytable+=
'<td>%s</td>'%content[item]
1103 summarytable+=
'</tr>'
1106 summarytable +=
'</table></div><br>'
1108 if runsWithoutReadyForPhysics:
1109 warning +=
'<div style="margin-left: 13px; color: darkred; width: 90%%;">WARNING! The following runs have no lumi block with ATLAS Ready: %s</div><br>' %
", ".
join(runsWithoutReadyForPhysics)
1110 if runsWithoutOfflineLumiInfo:
1111 warning +=
'<div style="margin-left: 13px; color: darkred; width: 90%%;">WARNING! The following runs have no offline lumi info: %s</div><br>' %
", ".
join(runsWithoutOfflineLumiInfo)
1121 if totalNumberOfReadyLB > 0:
1129 title =
"Click to zoom"
1132 summaryplot +=
'<table align="center" width="90%">'
1133 summaryplot +=
'<th>'
1137 wincontent =
"<img src="%s" width=%s/>"%(p,imgsize)
1140 openwincmd =
"javascript:openWindow('Print%i','"%(run)
1141 openwincmd +=
"<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">"
1142 openwincmd +=
"<html xmlns:"my"><head><title>DQ Summary Plot - Run %s</title></head>"%(run)
1143 openwincmd +=
"<body style="background-color:#ffffff">%s</body></html>"%wincontent
1146 summaryplot+=
'<td><a title="%s" href="%s" ><img src="%s" height=%s/></a></td>'%(title,openwincmd,p,thumbsize)
1149 summaryplot +=
'</th></table>'
1160 totalLumiRecorded = dicsum[DataKey(
'TotalLumi')][0]
1161 totalLumiLoss = dicsum[DataKey(
'TotalLumi')][1]
1162 lumiLossFraction = (100. * totalLumiLoss / totalLumiRecorded)
if totalLumiRecorded > 0
else 0
1165 print (
'+++++++++++++++++++++++++ Summary +++++++++++++++++++')
1166 print (
' Total Ready Recorded Luminosity: %.2f %s-1' % ( totalLumiRecorded, unit ))
1167 print (
' Total Luminosity Loss (ATLAS Ready) : %.2f %s-1' % ( totalLumiLoss, unit ))
1168 print (
' Total Global Not Ready (Stable Beams): %.2f %s-1'%(dicsum[DataKey(
'TotalNotReady')],unit))
1169 print (
' Total Global Busy (Stable Beams): %.2f %s-1'%(dicsum[DataKey(
'TotalBusy')],unit))
1170 print (
'+++++++++++++++++++++++++++++++++++++++++++++++++++++')
1174 if totalNumberOfReadyLB>0:
1175 summaryinfo =
'<table align="center" style="font-size:80%;"><tr>'
1176 summaryinfo +=
'<td> Total Luminosity Loss: <b>%.2f %s</b> (%.2f%%)' % ( totalLumiLoss, unit, lumiLossFraction )
1177 summaryinfo +=
'<br>Excluded Systems: %s</td></tr>' % tobeignored
1179 summaryinfo +=
'<tr><td style="font-size:70%%">using %s // %s</td></tr>'%(lumifolder.split(
':')[2],livetrigger)
1180 summaryinfo +=
'</table>'
1182 summaryinfo =
'<table align="center" style="font-size:80%;"><tr>'
1183 summaryinfo +=
'<td> No ready lumi block </td>'
1184 summaryinfo +=
'</tr></table>'
1190 pagecontent = summaryplot +
'<br>'
1191 pagecontent += summaryinfo +
'<br>'
1193 pagecontent += summarytable
1194 pagecontent += warning