540 def makeHTML(cls, dic, dicsum, doPickle=True, doDQSummary=True, doDQPlots=True, dqsumGRL="PHYS_StandardGRL_All_Good_25ns", dbbTag=(
"HEAD",
"HEAD")):
541 """ method returns a string (unicode is fine) of html code, with out tag <div>...</div> """
552 with timer(
"DQSummary"):
558 livetrigger =
'L1_EM30'
559 for r,run
in enumerate(dic[DataKey(
'Run')]):
560 ptag = dic[DataKey(
'Project tag')][r][0].value
563 livetrigger =
'L1_EM5'
564 elif 'data16_hip5TeV' in ptag
or 'data16_hip8TeV' in ptag:
566 livetrigger =
'L1_EM12'
567 elif 'data15' in ptag
or 'data16' in ptag
or 'data17' in ptag
or 'data18' in ptag:
569 livetrigger =
'L1_EM12'
574 warning +=
'<font color="red"> WARNING (DQSummary): The defect database tag "%s" must be a 2-element sequence. Will use "HEAD" tag instead!</font><br>' %(dbbTag)
575 dbbTag = (
"HEAD",
"HEAD")
579 if dbbTag[0]
not in [
'HEAD'] + ddb.defects_tags:
580 warning +=
'<font color="red"> WARNING (DQSummary): The defined defect tag "%s" is not defined in defect database. Will use "HEAD" tag instead!</font><br>' %(dbbTag[0])
581 dbbTag = (
"HEAD", dbbTag[1])
583 if dbbTag[1]
not in [
'HEAD'] + ddb.logics_tags:
584 warning +=
'<font color="red"> WARNING (DQSummary): The defined logic tag "%s" is not defined in defect database. Will use "HEAD" tag instead!</font><br>' %(dbbTag[1])
585 dbbTag = (dbbTag[0],
"HEAD")
587 ddb = DefectsDB(tag=(dbbTag[0], dbbTag[1]))
589 if dqsumGRL
in ddb.virtual_defect_logics.keys():
590 defects_primary_grl = ddb.resolve_primary_defects(ddb._resolve_evaluation_order([dqsumGRL]))
592 warning +=
'<font color="red"> WARNING (DQSummary): The defined virtual defect or GRL "%s" is not defined in logic. Will use no GRL for calculation!</font><br>' %(dqsumGRL)
597 if 'Cosmics' in dqsumGRL:
608 dicsum[DataKey(
'TotalLumi')] = [0.,0.]
611 dicsum[DataKey(
'TotalNotReady')] = 0.
612 dicsum[DataKey(
'TotalBusy')] = 0.
615 detectors = [
'ALFA',
'LCD',
'ZDC',
'BCM',
'PIXEL',
'SCT',
'TRT',
'LAR',
'TILE',
616 'MS_CSC',
'MS_RPC',
'MS_TGC',
'MS_MDT',
'MBTS',
'TRIG',
'GLOBAL']
617 detectors_color = {
'ALFA':kYellow-9,
'LCD':kYellow-7,
'ZDC':kYellow-4,
618 'BCM':kOrange+1,
'PIXEL':kOrange-3,
'SCT':kOrange+7,
'TRT':kRed-3,
619 'LAR':kRed+2,
'TILE':kPink-7,
620 'MS_CSC':kBlue+4,
'MS_RPC':kBlue-2,
'MS_MDT':kAzure+3,
'MS_TGC':kBlue-9,
'MBTS':kBlue-6,
621 'TRIG':kGreen-3,
'GLOBAL':kGreen+3,
'_TOTAL':TColor.GetColor(
"#585858")}
622 detectors_lumiloss = {}
623 detectors_affectedLBs = {}
626 detectors_lumiloss[d]={}
627 detectors_affectedLBs[d]={}
630 detectors_lumiloss[
'_TOTAL']={}
631 detectors_affectedLBs[
'_TOTAL']={}
634 performances = [
'LUMI',
'ID',
'CALO',
'TAU',
'EGAMMA',
'JET',
'MET',
'MCP',
'BTAG']
635 performances_color = {
'LUMI':kYellow-9,
'ID':kOrange+7,
'CALO':kRed+2,
'TAU':kBlue+2,
636 'EGAMMA':kGreen-3,
'JET':kAzure+3,
'MET':kGreen-6,
'MCP':kBlue-3,
637 'BTAG':kPink,
'_TOTAL':TColor.GetColor(
"#585858")}
638 performances_lumiloss = {}
639 performances_affectedLBs = {}
641 for p
in performances:
642 performances_lumiloss[p]={}
643 performances_affectedLBs[p]={}
646 performances_lumiloss[
'_TOTAL']={}
647 performances_affectedLBs[
'_TOTAL']={}
652 columns = [
'Run Info',
'ES1',
'BLK']
653 global_systems = [
'Calo',
'Tracking',
'Muon',
'Trigger & Lumi']
654 columns += global_systems
657 with timer(
'Create summary results table'):
659 summarytable =
'<div><table id="resulttable" class="resulttable" style="margin-left: 13px; margin-right: 13px;" >'
661 summarytable +=
'<tr>'
662 summarytable +=
'<th colspan="1"></th>'
665 summarytable +=
'<th colspan="2">Missing Sign-Off</th>'
667 summarytable +=
'<th colspan="%i">Defects in ATLAS Ready<br>'%len(global_systems)
668 summarytable +=
'<div style="font-size: xx-small; cursor: pointer;" onclick="toggle_dq(this)">[show tolerable]</div></th>'
669 summarytable +=
'</tr>'
672 summarytable +=
'<tr>'
675 summarytable +=
'<th>%s</th>'%item
676 summarytable +=
'</tr>'
678 totalNumberOfReadyLB = 0
683 for r,run
in enumerate(dic[DataKey(
'Run')]):
684 tot_nlb += dic[DataKey(
'#LB')][r][0]
686 runsWithoutReadyForPhysics = []
687 runsWithoutOfflineLumiInfo = []
690 for r,run
in enumerate(dic[DataKey(
'Run')]):
695 nlb = dic[DataKey(
'#LB')][r][0]
698 readylb =
range(1,nlb+1)
702 totalNumberOfReadyLB += len(readylb)
705 if len(readylb) == 0:
706 runsWithoutReadyForPhysics += [
str(run) ]
712 lumiperlb = dict(zip(readylb, [100./tot_nlb] * nlb))
713 livefrac = dict(zip(readylb, [1] * nlb))
718 if len(lumiperlb) == 0:
719 runsWithoutOfflineLumiInfo += [
str(run) ]
724 for lbl
in lumiperlb:
726 if lbl
not in readylb:
728 if lbl
not in livefrac:
732 lumiperlb[lbl]*=livefrac[lbl]
737 for i,item
in enumerate(columns):
742 ptag = dic[DataKey(
'Project tag')][r][0].value
743 year =
'20'+ptag.split(
'_')[0][4:]
744 DP = ARQ_COMA.get_periods_for_run(run)
747 content[
'Run Info'] =
'<b>%s</b> <font style="font-size:10px;">data period <b>%s</b></font><br>'%(
str(run),dp)
750 tobeignored = [
'ALFA',
'ZDC',
'LCD']
752 if re.match(
'data1[3-9]', ptag[0:6]):
756 target=
'target="blank"'
757 somelinks =
'<font style="font-size:10px;color:#0000FF">'
758 somelinks +=
'<a href="https://atlas-tagservices.cern.ch/tagservices/RunBrowser/runBrowserReport/runBrowserReport.php?runs=%s&pn=%s&fnt=%s" %s>COMA</a>'%(run,dp,ptag,target)
759 somelinks +=
', <a href="https://atlasdqm.web.cern.ch/atlasdqm/DQBrowser/makeMatrix.php?selection=All&run=%s&runup=&dbinstance=CONDBR2_DCS&tag=DEFAULT" %s>DCS</a>'%(run,target)
760 somelinks +=
', <a href="https://atlas.web.cern.ch/Atlas/GROUPS/DATAPREPARATION/DataSummary/%s/run.py?run=%s" %s>DS</a>'%(year,run,target)
761 somelinks +=
', <a href="https://atlasdqm.cern.ch/defectentry/?run=%s" %s>defects</a>'%(run,target)
762 somelinks +=
'<br><a href="https://atlasdqm.cern.ch/dqsignoff/%s/" %s>DQlogbook</a>'%(run,target)
763 somelinks +=
', <a href="https://atlasdqm.cern.ch/webdisplay/tier0?lowrun=%s&highrun=%s&show_all=on" %s>DQWebDisplay</a>'%(run,run,target)
764 somelinks +=
'</font><br>'
766 content[
'Run Info'] += somelinks
769 rundate = dic[DataKey(
'Start and endtime')][r].
split(
",")
770 tstart = time.strftime(
"%a, %d %b %Y %H:%M:%S",time.localtime(
float(rundate[0])))
771 tend = time.strftime(
"%a, %d %b %Y %H:%M:%S",time.localtime(
float(rundate[1])))
772 content[
'Run Info'] +=
'<font style="font-size:10px;">'
773 content[
'Run Info'] +=
'<br>Start: <font style="color:#4C7D7E;">%s</font>'%tstart
774 content[
'Run Info'] +=
'<br>End: <font style="color:#4C7D7E;">%s</font>'%tend
775 content[
'Run Info'] +=
'</font>'
783 lhctable =
'<br><font style="font-size:10px;">'
784 lhctable +=
'<table><tr><td>'
785 lhctable +=
'<b>Stable Beam</b><br>%s'%
'<br>'.
join([
'%s-%s: <font style="color:#4C7D7E">%s</font>'%(b.startlb,b.endlb,b.value)
for b
in dic[DataKey(
'lhc:stablebeams')][r]])
786 lhctable +=
'</td><td>'
787 lhctable +=
'<b>ATLAS Ready</b><br>%s'%
'<br>'.
join([
'%s-%s: <font style="color:#4C7D7E">%s</font>'%(b.startlb,b.endlb,b.value)
for b
in dic[DataKey(
'Ready for physics')][r]])
788 lhctable +=
'</td></tr></table>'
789 lhctable +=
'</font>'
790 content[
'Run Info'] += lhctable
794 dicsum[DataKey(
'TotalLumi')][0]+= lumitot
795 content[
'Run Info'] +=
'<font style="font-size:10px;">'
796 content[
'Run Info'] +=
'Ready Recorded: <b>%.2f</b> %s'%(lumitot,unit)
797 content[
'Run Info'] +=
'</font><br>'
800 defects = dic[DataKey(
'DQ')][r]
802 defects_primary = [d
for d
in defects
if d.value.primary]
807 detectors_lumiloss[d][run]= 0.
808 detectors_affectedLBs[d][run]=[]
809 for p
in performances:
810 performances_lumiloss[p][run]=0.
811 performances_affectedLBs[p][run]=[]
814 detectors_lumiloss[
'_TOTAL'][run]= 0.
815 detectors_affectedLBs[
'_TOTAL'][run]=[]
816 performances_lumiloss[
'_TOTAL'][run]= 0.
817 performances_affectedLBs[
'_TOTAL'][run]=[]
820 total_affectedLBs = []
825 ListOfIntolerableDefects = {}
826 ListOfTolerableDefects = {}
827 ListOfIgnoredDefects = {}
830 for item
in defects_primary:
831 defect = item.value.defect
832 comment = item.value.comment
833 user = item.value.user
834 startlb = item.startlb
836 tolerable = item.value.tolerable
841 if not dqsumGRL ==
'no' and not tolerable:
842 if defect
not in defects_primary_grl:
846 if 'GLOBAL_NOTREADY' in defect:
847 GlobalNotReady += [ lb
for lb
in range(startlb,endlb)]
849 if 'GLOBAL_BUSY' in defect:
850 GlobalBusy += [ lb
for lb
in range(startlb,endlb)
if lb
in readylb]
855 if 'UNCHECKED_FINAL' in defect:
859 if 'BULK_UNCHECKED' in defect:
860 short = defect.split(
'_UNCHECKED')[0]
861 content[
'BLK']+=
'<font style="font-size:8px;font-weight:bold;">%s</font><br>'%short
864 if 'UNCHECKED' in defect:
865 short = defect.split(
'_UNCHECKED')[0]
866 content[
'ES1']+=
'<font style="font-size:8px;font-weight:bold;">%s</font><br>'%short
871 print (
'run %s: this defect is fishy %s '%(run,defect))
875 word = defect.split(
'_')
879 if cpdet
not in detectors
and cpdet
not in performances:
880 print (
'This system is not included: %s (%s)'%(cpdet,defect))
885 RangeDefect = [ lb
for lb
in range(startlb,endlb)
if lb
in readylb ]
886 if len(RangeDefect)>0:
888 if defect
not in ListOfIntolerableDefects:
889 ListOfIntolerableDefects[defect]= [[],
'']
890 ListOfIntolerableDefects[defect][0] = RangeDefect
891 ListOfIntolerableDefects[defect][1] = user+
':'+comment
893 ListOfIntolerableDefects[defect][0]+= RangeDefect
894 if comment
not in ListOfIntolerableDefects[defect][1]:
895 ListOfIntolerableDefects[defect][1]+=
' '+user+
':'+comment
898 if cpdet
in tobeignored:
902 if cpdet
in detectors:
903 detectors_affectedLBs[cpdet][run]+=RangeDefect
904 if cpdet
in performances:
905 performances_affectedLBs[cpdet][run]+=RangeDefect
906 total_affectedLBs += RangeDefect
909 if defect
not in ListOfIgnoredDefects:
910 ListOfIgnoredDefects[defect]= [[],
'']
911 ListOfIgnoredDefects[defect][0]= RangeDefect
912 ListOfIgnoredDefects[defect][1]=
'[%s]:%s '%(user,comment)
914 ListOfIgnoredDefects[defect][0]+= RangeDefect
915 if comment
not in ListOfIgnoredDefects[defect][1]:
916 ListOfIgnoredDefects[defect][1] +=
'[%s]:%s '%(user,comment)
919 RangeDefect = [ lb
for lb
in range(startlb,endlb)
if lb
in readylb]
920 if len(RangeDefect)>0:
921 if defect
not in ListOfTolerableDefects:
922 ListOfTolerableDefects[defect]= [[],
'']
923 ListOfTolerableDefects[defect][0]= RangeDefect
924 ListOfTolerableDefects[defect][1]=
'[%s]:%s '%(user,comment)
926 ListOfTolerableDefects[defect][0]+= RangeDefect
927 if comment
not in ListOfTolerableDefects[defect][1]:
928 ListOfTolerableDefects[defect][1] +=
'[%s]:%s '%(user,comment)
932 for item
in global_systems:
933 content[item]=
'<table class="dqdefects">'
935 for item
in ListOfIntolerableDefects:
937 if system
not in global_systems:
939 lbs =
listify(ListOfIntolerableDefects[item][0])
940 tip = ListOfIntolerableDefects[item][1]
941 tipkey =
"dqcomment_int_%s_%s"%(item,run)
942 Run.addGlobalToolTip(tipkey,tip)
943 content[system]+=
'<tr class="showTip %s" >'%(tipkey)
944 content[system]+=
'<td class="intolerable">%s</td>'%(item)
945 content[system]+=
'<td class="lb">%s</td>'%(lbs)
946 content[system]+=
'</tr>'
948 for item
in ListOfIgnoredDefects:
950 if system
not in global_systems:
952 lbs =
listify(ListOfIgnoredDefects[item][0])
953 tip = ListOfIgnoredDefects[item][1]
954 tipkey =
"dqcomment_ign_%s_%s"%(item,run)
955 Run.addGlobalToolTip(tipkey,tip)
956 content[system]+=
'<tr class="showTip %s" >'%(tipkey)
957 content[system]+=
'<td class="ignored">%s</td>'%(item)
958 content[system]+=
'<td class="lb">%s</td>'%(lbs)
959 content[system]+=
'</tr>'
961 for item
in ListOfTolerableDefects:
963 if system
not in global_systems:
965 lbs =
listify(ListOfTolerableDefects[item][0])
966 tip = ListOfTolerableDefects[item][1]
967 tipkey =
"dqcomment_tol_%s_%s"%(item,run)
968 Run.addGlobalToolTip(tipkey,tip)
969 content[system]+=
'<tr class="showTip %s tolerable" style="visibility:collapse;">'%(tipkey)
970 content[system]+=
'<td>%s</td>'%(item)
971 content[system]+=
'<td class="lb">%s</td>'%(lbs)
972 content[system]+=
'</tr>'
975 for item
in global_systems:
976 content[item]+=
'</table><br>'
981 for gsys
in global_systems:
985 for defect
in ListOfTolerableDefects:
987 word = defect.split(
'_')
991 if cpdet
in tobeignored:
994 tol[defect]=ListOfTolerableDefects[defect][0]
995 for defect
in ListOfIntolerableDefects:
997 word = defect.split(
'_')
1000 cpdet +=
"_"+word[1]
1001 if cpdet
in tobeignored:
1004 int[defect]=ListOfIntolerableDefects[defect][0]
1005 for defect
in ListOfIgnoredDefects:
1007 word = defect.split(
'_')
1010 cpdet +=
"_"+word[1]
1011 if cpdet
in tobeignored:
1014 ign[defect]=ListOfIgnoredDefects[defect][0]
1020 title =
"Click to zoom"
1021 wincontent =
"<img src="%s" height=%s/>"%(h,imgsize)
1022 openwin =
"javascript:openLargeWindow('Print1','"
1023 openwin +=
"<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">"
1024 openwin +=
"<html xmlns:"my"><head><title>Defects for run %s</title></head>"%run
1025 openwin +=
"<body style="background-color:#ffffff">%s</body></html>"%wincontent
1027 content[gsys]+=
'<a title="%s" href="%s" ><img src="%s" height=%s/></a>'%(title,openwin,h,thumbsize)
1031 if not GlobalNotReady:
1032 for lb
in GlobalNotReady:
1033 _lblumi += lumiperlb[lb]
1034 dicsum[DataKey(
'TotalNotReady')] += _lblumi
1035 content[
'Run Info'] +=
'<font style="font-size:10px;">'
1036 content[
'Run Info'] +=
'Global Not Ready: <b>%.2f</b> %s'%(_lblumi,unit)
1037 content[
'Run Info'] +=
'</font><br>'
1041 for lb
in GlobalBusy:
1042 _lblumi += lumiperlb[lb]
1043 dicsum[DataKey(
'TotalBusy')]+= _lblumi
1044 content[
'Run Info'] +=
'<font style="font-size:10px;">'
1045 content[
'Run Info'] +=
'Global Busy: <b>%.2f</b> %s'%(_lblumi,unit)
1046 content[
'Run Info'] +=
'</font><br>'
1050 if len(detectors_affectedLBs[d][run]) == 0:
1053 dll =
list(
set(detectors_affectedLBs[d][run]))
1054 detectors_affectedLBs[
'_TOTAL'][run]+= dll
1056 detectors_lumiloss[d][run]+= lumiperlb[lb]
1057 for p
in performances:
1058 if len(performances_affectedLBs[p][run]) == 0:
1061 pll =
list(
set(performances_affectedLBs[p][run]))
1062 performances_affectedLBs[
'_TOTAL'][run]+= pll
1064 performances_lumiloss[p][run]+= lumiperlb[lb]
1068 detectors_affectedLBs[
'_TOTAL'][run]=
list(
set(detectors_affectedLBs[
'_TOTAL'][run]))
1069 performances_affectedLBs[
'_TOTAL'][run]=
list(
set(performances_affectedLBs[
'_TOTAL'][run]))
1070 total_affectedLBs =
list(
set(total_affectedLBs))
1071 totallossperrun = 0.
1073 for lb
in total_affectedLBs:
1074 totallossperrun += lumiperlb[lb]
1075 dicsum[DataKey(
'TotalLumi')][1] += totallossperrun
1077 for lb
in detectors_affectedLBs[
'_TOTAL'][run]:
1078 detectors_lumiloss[
'_TOTAL'][run] += lumiperlb[lb]
1080 for lb
in performances_affectedLBs[
'_TOTAL'][run]:
1081 performances_lumiloss[
'_TOTAL'][run] += lumiperlb[lb]
1084 content[
'Run Info'] +=
'<font style="font-size:10px;">'
1085 content[
'Run Info'] +=
'DQ Lumi Loss: <b>%.2f</b> %s'%(totallossperrun,unit)
1086 content[
'Run Info'] +=
'</font><br>'
1089 summarytable+=
'<tr class="out2">'
1090 for item
in columns:
1091 summarytable+=
'<td>%s</td>'%content[item]
1092 summarytable+=
'</tr>'
1095 summarytable +=
'</table></div><br>'
1097 if runsWithoutReadyForPhysics:
1098 warning +=
'<div style="margin-left: 13px; color: darkred; width: 90%%;">WARNING! The following runs have no lumi block with ATLAS Ready: %s</div><br>' %
", ".
join(runsWithoutReadyForPhysics)
1099 if runsWithoutOfflineLumiInfo:
1100 warning +=
'<div style="margin-left: 13px; color: darkred; width: 90%%;">WARNING! The following runs have no offline lumi info: %s</div><br>' %
", ".
join(runsWithoutOfflineLumiInfo)
1110 if totalNumberOfReadyLB > 0:
1118 title =
"Click to zoom"
1121 summaryplot +=
'<table align="center" width="90%">'
1122 summaryplot +=
'<th>'
1126 wincontent =
"<img src="%s" width=%s/>"%(p,imgsize)
1129 openwincmd =
"javascript:openWindow('Print%i','"%(run)
1130 openwincmd +=
"<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">"
1131 openwincmd +=
"<html xmlns:"my"><head><title>DQ Summary Plot - Run %s</title></head>"%(run)
1132 openwincmd +=
"<body style="background-color:#ffffff">%s</body></html>"%wincontent
1135 summaryplot+=
'<td><a title="%s" href="%s" ><img src="%s" height=%s/></a></td>'%(title,openwincmd,p,thumbsize)
1138 summaryplot +=
'</th></table>'
1149 totalLumiRecorded = dicsum[DataKey(
'TotalLumi')][0]
1150 totalLumiLoss = dicsum[DataKey(
'TotalLumi')][1]
1151 lumiLossFraction = (100. * totalLumiLoss / totalLumiRecorded)
if totalLumiRecorded > 0
else 0
1154 print (
'+++++++++++++++++++++++++ Summary +++++++++++++++++++')
1155 print (
' Total Ready Recorded Luminosity: %.2f %s-1' % ( totalLumiRecorded, unit ))
1156 print (
' Total Luminosity Loss (ATLAS Ready) : %.2f %s-1' % ( totalLumiLoss, unit ))
1157 print (
' Total Global Not Ready (Stable Beams): %.2f %s-1'%(dicsum[DataKey(
'TotalNotReady')],unit))
1158 print (
' Total Global Busy (Stable Beams): %.2f %s-1'%(dicsum[DataKey(
'TotalBusy')],unit))
1159 print (
'+++++++++++++++++++++++++++++++++++++++++++++++++++++')
1161 if totalNumberOfReadyLB>0:
1162 summaryinfo =
'<table align="center" style="font-size:80%;"><tr>'
1163 summaryinfo +=
'<td> Total Luminosity Loss: <b>%.2f %s</b> (%.2f%%)' % ( totalLumiLoss, unit, lumiLossFraction )
1164 summaryinfo +=
'<br>Excluded Systems: %s</td></tr>' % tobeignored
1166 summaryinfo +=
'<tr><td style="font-size:70%%">using %s // %s</td></tr>'%(lumifolder.split(
':')[2],livetrigger)
1167 summaryinfo +=
'</table>'
1169 summaryinfo =
'<table align="center" style="font-size:80%;"><tr>'
1170 summaryinfo +=
'<td> No ready lumi block </td>'
1171 summaryinfo +=
'</tr></table>'
1177 pagecontent = summaryplot +
'<br>'
1178 pagecontent += summaryinfo +
'<br>'
1180 pagecontent += summarytable
1181 pagecontent += warning