dq_defect_info_table.py

Go to the documentation of this file.

#! /usr/bin/env python
 
from argparse import ArgumentParser
from os import listdir, makedirs
from os.path import dirname, exists, join as pjoin
from importlib.resources import files
from shutil import copy2
from textwrap import dedent
 
from genshi import HTML
from genshi.output import HTMLSerializer, encode
from genshi.template import MarkupTemplate
 
from DQUtils import fetch_iovs
from DQUtils.general import timer
from DQUtils.logger import init_logger; init_logger(verbose=2)
from DQUtils.lumi import fetch_lumi_inputs, compute_lumi_many_channels
 
from DQDefects import DefectsDB
 
from IPython.Shell import IPShellEmbed; ip = IPShellEmbed(["-pdb"])
 
GRAPH_URL = "http://atlasdqm.web.cern.ch/atlasdqm/defect_graph/HEAD"
 

headings = [
    ("", "V"),
    ("", "T"),
    ("", "U"),
    ("", "B"),
    ("name_with_url", "Name"),
    ("description", "Description"),
    ("nruns", "# Runs"),
    ("latest_run", "Latest Run"),
    ("rec_lb_frac", "Potentially Recoverable LB %"),
    ("lumi", HTML("Luminosity (pb<sup>-1</sup>)")),
    #("nlumi", "nlumi"),
    #("nlumi_rec", "nlumi"),
]
 
heading_names, heading_titles = zip(*headings)
 
content_string = "".join("<td>{{d.{0}}}</td>".format(x)
                         for x in heading_names if x)
 
class Defect:
    """
    A defect instance. Forms a row in the table.
    """
    def __init__(self, name, description, iovs, virtual, intolerable, lumi):
        if description == name:
            # Don't bother pretending that there is a description if it is just
            # equal to the name
            description = ""
            
        (self.name, self.description, self.iovs, self.lumi, self.virtual, 
         self.intolerable) = (
         name, description, iovs, lumi / 1e6, virtual, intolerable)
        self.nlumi = sum(iov.length for iov in iovs 
                    if iov.since.run == iov.until.run)
        self.nlumi_rec = sum(iov.length for iov in iovs 
                             if iov.recoverable and iov.since.run == iov.until.run)
        if self.nlumi and self.nlumi_rec:
            self.rec_lb_frac = "{0:.0%}".format(self.nlumi_rec / self.nlumi)
        else:
            self.rec_lb_frac = "-"
                    
        self.users = set(iov.user for iov in iovs)
        self.users -= set(["sys:virtual"]) # exclude the "virtual" user.
        self.bots = set(u for u in self.users if u.startswith("sys:"))
        self.users -= self.bots
        
        runs = iovs.runs
        self.nruns = len(runs) if runs else "-"
        self.latest_run = max(runs) if runs else "-"
        self.lumi = "{0:.3f}".format(self.lumi) if runs else "-"
    
    @property
    def name_with_url(self):
        if self.virtual or self.intolerable:
            return '<a href="{0}/{1}.svg">{1}</a>'.format(GRAPH_URL, self.name)
        else:
            return "{0}".format(self.name)
    
    @property
    def content(self):
        td = '<td class="icon {class_}"><span>{0}</span></td>'
        bot_cls = user_cls = ""
        if len(self.users) == 1: user_cls = "user"
        if len(self.users) > 1: user_cls = "users"
        if len(self.bots): bot_cls = "bots"
        
        virt_class = "virtual" if self.virtual else "primary"
        tol_class = "intolerable" if self.intolerable else "tolerable"
        virt_sort = 1 if self.virtual else 0
        tol_sort = 1 if self.virtual else 0
        
        c = [
            td.format(virt_sort, class_=virt_class),
            td.format(tol_sort, class_=tol_class),
            td.format(", ".join(self.users), class_=user_cls),
            td.format(", ".join(self.bots), class_=bot_cls),
            content_string.format(d=self)
        ]
        return "".join(c)
 
def build_table(**kwargs):
    """
    Build the HTML content
    """
    path = files("DQDefects.data").joinpath("table.html")
    with open(path) as fd:
        template = MarkupTemplate(fd, path)
     
    stream = template.generate(HTML=HTML, **kwargs)
    serializer = HTMLSerializer(doctype="html5")
    content = encode(serializer(stream))
    return content
 
def build_defects(descriptions, virtuals, intolerables, lbs, lumis, all_defects):
    
    with timer("Sort defects by channel"):
        dbc = all_defects.by_channel
        
    with timer("Compute luminosities"):
        lumi_by_defect = compute_lumi_many_channels(lbs, lumis, all_defects)
    
    defects = []
    
    for name, description in sorted(descriptions.iteritems()):
        virtual, intolerable = name in virtuals, name in intolerables
        defect = Defect(name, description, dbc[name], virtual, intolerable, 
                        lumi_by_defect.get(name, 0))
        defects.append(defect)
 
    defects.sort(key=lambda d: (d.virtual, not d.intolerable))
 
    return defects
 
def copy_art(target_dir):
    art_base = dirname(resource_filename("DQDefects.data", "table.html"))
    
    for filename in listdir(art_base):
        _, _, ext = filename.rpartition(".")
        if ext.lower() not in ("png", "js", "css"):
            continue
        copy2(pjoin(art_base, filename), pjoin(target_dir, filename))
 
def main():
    d = DefectsDB()
        
    iov_range = None, None #
    iov_range = 185000 << 32, 185500 << 32
    #iov_range = 177682 << 32, 200000 << 32
    
    with timer("Fetch defect info"):
        all_defects = d.retrieve(*iov_range)
    
    with timer("fetch peripheral information"):    
        descriptions = d.all_defect_descriptions
        intolerable = d.get_intolerable_defects()
        virtual = d.virtual_defect_names
        
    with timer("Fetch lumi inputs"):
        lbs, lumis = fetch_lumi_inputs(iov_range, "ONLINE")
    
    d = build_defects(descriptions, virtual, intolerable, lbs, lumis, all_defects)
    
    target_dir = "/afs/cern.ch/user/p/pwaller/www/defects/test"
    art_dir = pjoin(target_dir, "extern")
    
    if not exists(art_dir):
        makedirs(art_dir)
    
    copy_art(art_dir)
    
    content = build_table(headings=heading_titles, defects=d)
    with open(pjoin(target_dir, "test.html"), "w") as fd:
        fd.write(content)
    
 
if __name__ == "__main__":
    main()