AtlRunQuerySelectorDQ.py

Go to the documentation of this file.

# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
 
 
from __future__ import print_function
from CoolRunQuery.selector.AtlRunQuerySelectorBase import Selector, RunLBBasedCondition, OOO
from CoolRunQuery.utils.AtlRunQueryIOV    import IOVRange
from CoolRunQuery.utils.AtlRunQueryUtils  import coolDbConn
from CoolRunQuery.utils.AtlRunQueryLookup import DQChannel
 
from collections import defaultdict, namedtuple
 
DQDefectPayload = namedtuple("DQDefectPayload", "defect comment user ignore primary tolerable recoverable")
DQDefectPayload.pickled = DQDefectPayload._asdict
 
DD = namedtuple("DD","description comment since until")
 
def vfgen(vfobjs):
    for obj in vfobjs:
        yield OOO(obj.channel, (str(obj.Code),obj.Comment), IOVRange(starttime=obj.since, endtime=obj.until), True)
 
class DQSelector(Selector):
    def __init__(self, name='dataquality'):
        super(DQSelector,self).__init__(name)
        self.selectors = {}
 
        from CoolRunQuery.AtlRunQuerySelectorWorker import SelectorWorker
        SelectorWorker.getRetrieveSelector('readyforphysics','ReadyForPhysicsSelector')
 
 
    def __getChannelAndFolder(self, chanAndfolder):
        cf = chanAndfolder.split()
        if len(cf)==1:
            channel=''
            folder=cf[0]
        else:
            channel, folder = cf
        return (channel,folder)
 
    def __getCondition(self, folder):
        if folder in self.selectors:
            return self.selectors[folder]
        if folder.startswith('DEFECTS'):
            self.selectors[folder] = DQDefectCondition(self.name+"DEFECTS", folder)
        else:
            self.selectors[folder] = DQCondition(self.name+folder, folder, [])
        return self.selectors[folder]
 
    def addSelectionChannel(self, dq=[]):
        """
        dq can be e.g. ['EMBA yellow SHIFTOFL', 'FCALA green+ DQMFOFL', 'EMBA,EMBC,EMECA,EMECC yellow+ SHIFTOFL']
        this example should be translated into two selectors, one for each of the folders 'SHIFTOFL' and 'DQMFOFL'
        """
        folderCriteria = defaultdict(list)
        for x in dq:
            s = x.split()
            folderCriteria[s[-1]].append(s[:-1])
        for f in folderCriteria.keys():
            if f.startswith('DEFECTS'):
                self.selectors[f] = DQDefectCondition(self.name+"DEFECTS", f )
                self.selectors[f].addSelectionChannels(folderCriteria[f])
            else:
                self.selectors[f] = DQCondition(self.name+f, f, folderCriteria[f])
 
 
    def addShowChannel(self, chanAndfolder):
        (channel,folder) = self.__getChannelAndFolder(chanAndfolder)
        condition = self.__getCondition(folder)
        condition.addShowChannel(folder, channel, 'HEAD')
 
    def setShowOutput(self):
        for sel in self.selectors.values():
            sel.setShowOutput()
 
 
    def __str__(self):
        s = ""
        for sel in self.selectors.values():
            s += "\n%s" % sel
        return s
 
 
    def select(self, runlist):
        # garbage collector
        # import gc
        # gcod = gc.get_objects()
        # print ("GC objects",len(gcod))
        # print ("GC object count",gc.get_count())
        
        for sel in self.selectors.values():
            runlist = sel.select(runlist)
        return runlist
 
    def runAfterQuery(self,runlist):
        for sel in self.selectors.values():
            sel.runAfterQuery(runlist)
        
 
 
class DQCondition(RunLBBasedCondition):
 
    code  = {'unknown': 0, 'red': 1, 'yellow': 2, 'green': 3, 'black': -1}
    color = {'n.a.': 'n.a.', '0': 'U', '1': 'R', '2': 'Y', '3': 'G', '-1': 'B'}
    invert = {'n.a.': -2, 'B': -1, 'U': 0, 'R': 1, 'Y': 2, 'G': 3 }
    
    def __init__(self, name, foldername, dq):
 
        self.foldername = foldername
        self.channelNames = []
        self.channelNamesFlat = []
        self.channels = []
        self.channelsflat = []
        self.flags = []
        self.useVirtualFlags = False
 
        dbname = 'COOLOFL_GLOBAL'
        if 'ONL' in self.foldername:
            dbname = 'COOLONL_GLOBAL'
 
        # set up virtual flag logic folder and expand wild cards for VF in dq
        self.GetVirtualFlagsExpanded(dq, Selector.compDB(), dbname)
 
        for x in dq:
            ch = x[0].split(',')
            self.channelNames     += [["%s:%s" % (self.foldername,chn) for chn in ch]]
            self.channels         += [[self.DQChannel(chn) for chn in ch]]
            self.flags            += [(x+[''])[1]]
 
        for chN in self.channelNames:
            self.channelNamesFlat += chN
        for ch in self.channels:
            self.channelsflat += ch
 
        self.channelCommentsFlat = [n+"_m" for n in self.channelNamesFlat]
 
        # choose DB based on foldername
 
        # out of 'SHIFTOFL', 'DQCALCOFL', 'DQMFOFL', 'DQMFOFLH', 'DCSOFL', 'TISUMM', 'LBSUMM', 'MUONCALIB', 'DQMFONL', 'SHIFTONL'
        # these ones have a comment field
 
        self.folderHasComment = ( (self.foldername == "SHIFTOFL") or (self.foldername.startswith("SHIFTOFL#")) or
                                  (self.foldername == "LBSUMM")   or (self.foldername.startswith("LBSUMM#")) or
                                  (self.foldername == "DQCALCOFL")   or (self.foldername.startswith("DQCALCOFL#")) or
                                  (self.foldername.startswith("MUONCALIB#")) )
 
        # channel keys:
        # [Code (Int32) : 1], [deadFrac (Float) : 0], [Thrust (Float) : 0], [Comment (String255) : "warm start"]
        if self.folderHasComment:
            channelKeys  = zip(self.channelsflat, self.channelNamesFlat, len(self.channelsflat)*[('Code','Comment')])
        else:
            channelKeys  = zip(self.channelsflat, self.channelNamesFlat, len(self.channelsflat)*['Code'])
                                         
        super(DQCondition,self).__init__(name=name,
                                         dbfolderkey='%s::/GLOBAL/DETSTATUS/%s' % (dbname, self.foldername),
                                         channelKeys = channelKeys)
 
        self.flagInterpreter = {}
        for flag in self.flags:
            self.flagInterpreter[flag] = {}
            d = self.flagInterpreter[flag]
            d['refVal']      = None
            d['passFnc']     = None
            d['passFncName'] = ''
 
            if flag=='n.a.':
                d['passFnc']     = lambda x: x==-2
                d['passFncName'] = "x=='n.a.'"
            elif flag[-1] in '+-':
                cd = self.code[flag[:-1].lower()]
                d['refVal'] = cd
                if flag[-1]=='+':
                    if cd == -1:
                        d['passFnc'] = lambda x: int(x)>=-1
                    elif cd == 0:
                        d['passFnc'] = lambda x: int(x)>=0
                    elif cd == 1:
                        d['passFnc'] = lambda x: int(x)>=1
                    elif cd == 2:
                        d['passFnc'] = lambda x: int(x)>=2
                    elif cd == 3:
                        d['passFnc'] = lambda x: int(x)>=3
                    d['passFncName'] = "x>='%i'" % d['refVal']
                else:
                    if cd == -1:
                        d['passFnc'] = lambda x: int(x)<=-1
                    elif cd == 0:
                        d['passFnc'] = lambda x: int(x)<=0
                    elif cd == 1:
                        d['passFnc'] = lambda x: int(x)<=1
                    elif cd == 2:
                        d['passFnc'] = lambda x: int(x)<=2
                    elif cd == 3:
                        d['passFnc'] = lambda x: int(x)<=3
                    d['passFncName'] = "x<='%i'" % d['refVal']
            else:
                cd = self.code[flag.lower()]
                d['refVal'] = cd
                if cd == -1:
                    d['passFnc'] = lambda x: int(x)==-1
                elif cd == 0:
                    d['passFnc'] = lambda x: int(x)==0
                elif cd == 1:
                    d['passFnc'] = lambda x: int(x)==1
                elif cd == 2:
                    d['passFnc'] = lambda x: int(x)==2
                elif cd == 3:
                    d['passFnc'] = lambda x: int(x)==3
                d['passFncName'] = "x=='%i'" % d['refVal']
                
        self.passSpecs = {}
        for chgrp, flag in zip(self.channelNames,self.flags):
            for ch in chgrp:
                self.passSpecs[ch] = self.flagInterpreter[flag]
 
    def _getFolder(self):
        from CoolRunQuery.AtlRunQueryRun import RunData
        RunData.DQLogic = self.channelNames
        RunData.DQKeys = self.channelNamesFlat
 
        f = coolDbConn.GetDBConn(schema = self.schema,db=Selector.condDB()).getFolder(self.folder)
        if self.useVirtualFlags:
            f = self.VirtualFlagFolder(f)
        else:
            if f.versioningMode()==0:
                self.tagname=""
            if self.tagname not in ["HEAD", ""]:
                self.tagname = f.resolveTag(self.tagname)
        return f
 
    def _retrieve(self, iovmin, iovmax, f, sortedRanges):
        if self.useVirtualFlags:
            channels = [x.lower().split(':')[-1] for x in self.channelNamesFlat]
            objs = vfgen(f.browseObjects( iovmin, iovmax, channels, self.tagname ))
        else:
            objs = super(DQCondition,self)._retrieve(iovmin, iovmax, f, sortedRanges)
        return objs
 
    def DQChannel(self,dqfullname):
        dqname = dqfullname.split(':')[-1].split('#')[0]
        if self.useVirtualFlags and dqname in self.vfl.get_logic_list().keys():
            try:
                return self.vfl.get_logic_list()[dqname].record.channel
            except AttributeError:
                return self.vfl.get_logic_list()[dqname].channel
        return DQChannel(dqname)
 
    def GetVirtualFlagsExpanded(self, dqlist, db, schema):
        for i in range(len(dqlist)):
            dqs = dqlist[i][0].split(',')
            newdqs = []
            for c in dqs:
                if '_' in c:
                    newdqs += self.ExpandVFlag(c,db,schema)
                else:
                    newdqs.append(c)
            dqlist[i][0] = ','.join(newdqs)
 
    def ExpandVFlag(self, cpflag, db, schema):
        vfl = self.GetVirtualFlagLogic(db, schema)
        vflags = vfl.get_logic_list().keys()
        useprimaries = cpflag[-1]=='+'
        cpflag=cpflag.rstrip('+')
        expflags = []
 
        if cpflag in vflags:
            self.AddVFHeaderData(cpflag)
            expflags += [cpflag]
            if useprimaries:
                expflags += self.getVFDef(cpflag)
        else:
            for vf in vflags:
                if vf.startswith(cpflag):
                    self.AddVFHeaderData(vf)
                    expflags += [vf]
                    if useprimaries:
                        expflags += self.getVFDef(vf)
                    
        if len(expflags)==0:
            raise RuntimeError("Virtual Flag pattern %s does not match any virtual flag: %r" % (cpflag, vfl.get_logic_list().keys()))
 
        return expflags
 
 
    def GetVirtualFlagLogic(self, db, schema):
        if self.useVirtualFlags:
            return self.vfl
        try: # import
            from VirtualFlags import VirtualFlagLogicFolder, VirtualFlagFolder
        except ImportError:
            print ("Can't import virtual flags")
            import traceback
            traceback.print_exc()
 
        self.VirtualFlagFolder = VirtualFlagFolder
        self.vfl               = VirtualFlagLogicFolder(coolDbConn.GetDBConn(schema=schema, db=db))
        self.logics            = self.vfl.get_logic_list()
        self.useVirtualFlags   = True
        return self.vfl
 
    def AddVFHeaderData(self,cpflag):
        from CoolRunQuery.AtlRunQueryRun import Run
        if cpflag in Run.Fieldinfo:
            return
        Run.Fieldinfo[cpflag] = '<strong><b>%s</b></strong><br><table width="300"><tr><td>%s</td></tr></div>' % \
                                (self.vfl.get_logic_list()[cpflag].comment,
                                 ", ".join(self.getVFDef(cpflag)) )
 
    def getVFDef(self,cpflag):
        return self.vfl.resolve_primary_flags(self.vfl.resolve_dependancies([cpflag]))
    
    
        
    def ApplySelection(self,key):
        if key in self.passSpecs:
            return True
        return False
 
    def addShowChannel(self, folder, channelname, tag):
        tmplist = [[channelname,'']]
        self.GetVirtualFlagsExpanded(tmplist, Selector.compDB(), self.schema)
 
        for shch in tmplist[0][0].split(','):
            self.addSingleShowChannel(folder+':'+shch)
        
    def addSingleShowChannel(self, channelname):
        ch = self.DQChannel(channelname)
        ssr = self.DoSelectShowRetrieve()
        if ch in self.channelsflat: # channel exists already, just need to set showoutput to true
            idx = self.channelsflat.index(ch) # location of the channel in the flat list
            self.channelNamesFlat[idx] = channelname
            ssr[idx] += 2
        else:
            self.channelNamesFlat += [channelname]
            self.channelsflat += [self.DQChannel(channelname)]
            ssr += [2]
        # re-assign
        self.channels = self.channelsflat
        
        if self.folderHasComment:
            channelKeys = zip(self.channelsflat, self.channelNamesFlat, len(self.channelsflat)*[('Code','Comment')])
        else:
            channelKeys = zip(self.channelsflat, self.channelNamesFlat, len(self.channelsflat)*['Code'])
 
        self.setChannelKeys(channelKeys,ssr)
 
    def __str__(self):
        if self.applySelection:
            s = "SELOUT Checking in the DQ folder %s" % self.foldername
            flagCh = defaultdict(list)
            for flag,ch in zip(self.flags,self.channelNames):
                flagCh[flag].append('(' + ' or '.join(ch) + ')')
            for flag in flagCh:
                chlist = ' and '.join(flagCh[flag])
                s += "\nSELOUT %s is %r" % (chlist, flag)
            return s
        else:
            return "Retrieving DQ for channels %r from folder %s" % (self.channelNamesFlat, self.foldername)
 
    def passes(self,values, key):
        passfnc = self.passSpecs[key]['passFnc']
        if isinstance(values,tuple):
            v = values[0]
        else:
            v = values
        if v=='n.a.':
            v=-2
        try:
            if passfnc(v):  # passed this DQ flag in the OR group?
                return True
        except ValueError:
            # happens when comparing 'n.a.' with 0,1,2,3 (for 'U','R','Y','G') ... and the black flag? (AH)
            # should never set pass=True
            pass
        
        return False
 
    def rejectRun(self,run):
        for k in self.ResultKey():
            passfnc = self.passSpecs[k]['passFnc']
            for entry in run.data[k]:
                v = entry.value
                dqres = v[0] if type(v)==tuple else v
                dqcode = self.invert[dqres]
                entry.rejected = not passfnc( dqcode )
        return super(DQCondition,self).rejectRun(run)
 
 
 
    def prettyValue(self, value, key):
        if type(value)==tuple:
            return (self.color[value[0]], value[1])
        else:
            return self.color[value]
                
    def runAfterQuery(self,runlist):
        dqs = ['G','Y','R','B','U','n.a.']
        for run in runlist:
            for k in self.ResultKey():
                n={}
                run.stats[k] = {}
                for dq in dqs:
                    n[dq] = 0
                blocks = []
                for entry in run.data[k]:
                    if self.folderHasComment:
                        if entry.value == 'n.a.':
                            dq, dqcomment = ('n.a.','')
                        else:
                            dq, dqcomment = entry.value
                    else:
                        dq, dqcomment = (entry.value,None)
                    
                    n[dq] += len(entry)
                    if entry.startlb == 0:
                        n[dq] -= 1 # CAUTION: problem with LB-1 = 0 ==> needs to be corrected
                    if len(blocks) > 0 and blocks[-1][0]==(dq,dqcomment) and blocks[-1][2]==entry.startlb:
                        blocks[-1][2] = entry.endlb
                    else:
                        blocks += [ [(dq, dqcomment), entry.startlb, entry.endlb] ]
 
                maxn = max(n.values())
                # find maximum DQ flag
                dqmax = dqs[0]
                for dq in dqs:
                    if n[dq]==maxn:
                        dqmax=dq
                run.result[k]=dqmax
                run.stats[k] = { "counts" : n, "max": dqmax, "blocks": blocks }
 
 
class DQDefectCondition(RunLBBasedCondition):
 
    def __init__(self, name, folder):
        self.foldername   = folder # DEFECTS#<TAG>
        self.primaries    = {} # self.primaries[runnr] is the list of primary defects that are set in each run
        self.channelNames = []
        self.selectionChannelNames = []
        self.global_ignore = set() # set of all primary defects that should be ignored in all defects
        super(DQDefectCondition,self).__init__(name=name,
                                               dbfolderkey = 'COOLOFL_GLOBAL::%s' % self.foldername, # folder name handled internally by DQDefects
                                               channelKeys = [('DQDEFECT', 'DQ',('Code','Comment'))])
 
        self.data_keys[0]._second_internal_key = self.tagname
 
    def _getFolder(self):
        from CoolRunQuery.AtlRunQueryRun import RunData
        RunData.DefectSelector = self._pass
        self.__db = coolDbConn.GetDBConn('DEFECTS',self.tagname)
        return self.__db # return the DB, not the folder
 
    def _pass(self,data):
        these_defects = []
        if data is not None:
            for x in data:
                if x.value.ignore is None: # no ignore
                    these_defects += [x.value.defect]
                else:
                    these_defects += [x.value.defect+'\\'+x.value.ignore]
        for orGroup in self.selectionChannelNames:
            any_passes = False
            for k in orGroup:
                if k[0]=='\\':
                    continue # "\Defect" are modifiers not requirements so we ignore it in the check
                if k[0]=='!':  # !DEFECT
                    kk = k[1:]
                    passes = kk not in these_defects
                else:
                    passes = k in these_defects
                if passes:
                    any_passes = True
                    break
            if not any_passes: # none pass in the OR-group
                return False
        return True
 
    def _fix_channel_names(self, db):
        newl = []
        for litem in self.selectionChannelNames:
            newlitem = []
            for k in litem:
                if k[0]=='\\':
                    newlitem.append(k)
                elif k[0] == '!':
                    newlitem.append('!' + db.normalize_defect_names(k[1:]))
                else:
                    newlitem.append(db.normalize_defect_names(k))
            newl.append(newlitem)
        self.selectionChannelNames = newl
        self.global_ignore = set(db.normalize_defect_names(self.global_ignore))
 
 
    def __getChannels(self, db, channel_name_patterns):
        """ based on the patterns in the list channel_name_patterns create a
        list of pairs of patterns and matching defects ( pattern, set(matching defects) )
        and attach it to the global Run.FieldInfo dictionary under 'DQ'
        """
        from re import compile
        from CoolRunQuery.AtlRunQueryRun import Run
        if 'DQ' not in Run.Fieldinfo:
            Run.Fieldinfo['DQ'] = {}
        if 'DefMatch' not in Run.Fieldinfo['DQ']:
            Run.Fieldinfo['DQ']['DefMatch'] = []
        Run.Fieldinfo['DQ']['IntolerableDefects'] = db.get_intolerable_defects(old_primary_only=False)
        matches = set()
        for pattern in channel_name_patterns:
            if pattern.upper() != 'DET':
                cpattern = compile(pattern)
                channelnames = set(filter(cpattern.match,db.defect_names))
                channelnames.update(filter(cpattern.match,db.virtual_defect_names))
            else:
                channelnames = set([d for d in db.defect_names if '_' not in d])
                channelnames.update([d for d in db.virtual_defect_names if '_' not in d])
            matches.update(channelnames)
            Run.Fieldinfo['DQ']['DefMatch'] += [(pattern,channelnames)]
        return matches
 
 
    def _retrieve(self, iovmin, iovmax, db, sortedRanges):
        runmin = iovmin>>32
        runmax = iovmax>>32
        since  = (runmin,iovmin&0xFFFFFFFF)
        until  = (runmax,iovmax&0xFFFFFFFF)
 
        # add the channels for showing (those can be wildcarded, so call __getChannels to get exact matches)
        channels = self.__getChannels(db,self.channelNames)
 
        # Rewrite defect names to get cases to be correct
        self._fix_channel_names(db)
 
        # add the channels for selection (those have to match exactly, they are simply added)  
        channels_with_ignore = {} # unique
        for selChans in self.selectionChannelNames:
            channels.update([x.lstrip('!') for x in selChans if '\\' not in x])
            for x in selChans: # for the defects with ignore condition
                if '\\' not in x:
                    continue
                channel, ignore_str = self.sort_uniq_ignores(x).split('\\',1)
                if ignore_str not in channels_with_ignore:
                    channels_with_ignore[ignore_str] = []
                channels_with_ignore[ignore_str] += [channel.lstrip('!')]
 
        #print ("CHANNELS",channels)
        if len(channels) + len(channels_with_ignore)==0:
            return []
 
        # we need to remove the special case ANY from the set
        if 'ANY' in channels:
            channels.remove('ANY')
 
        res = [] if len(channels)==0 else [ (db.retrieve(since=since, until=until, channels=channels, ignore=self.global_ignore, with_primary_dependencies=True).trim_iovs, None) ]
 
        # and also treat the defects with 'ignore' condition
        for _ignore_str, _channels in channels_with_ignore.items():
            ignore = set(_ignore_str.split('\\'))
            ignore.update(self.global_ignore)
            
            res += [ (db.retrieve(since=since, until=until, channels=_channels, ignore=ignore, with_primary_dependencies=True).trim_iovs, _ignore_str) ]
 
        return self.defgen(db, res, channels, channels_with_ignore)
 
    
 
    def defgen(self, db, defects_with_ignore, channels, channels_with_ignore):
        """
        defects: list of defects returned by query
        channels: explicit list of channels that matched the pattern
        """
        intolerableDefects = db.get_intolerable_defects(old_primary_only=False)
        for defects, ignore in defects_with_ignore:
            chanlist = channels if ignore is None else channels_with_ignore[ignore]
            for d in defects:
                if not d.present:
                    continue
                isVirtual = (d.user == 'sys:virtual') # db.defect_is_virtual(d.channel)
                if not isVirtual:
                    run = d.since.run
                    # fill list of primaries defects for this run (this is needed for the comments, but not for the selection)
                    if run not in self.primaries:
                        self.primaries[run] = []
                    self.primaries[run] += [d]
                if d.channel not in chanlist:
                    continue
 
                defPayload = DQDefectPayload(defect = d.channel, comment = d.comment,
                                             user = d.user, primary = not isVirtual,
                                             ignore = ignore,
                                             tolerable = (d.channel not in intolerableDefects),
                                             recoverable = d.recoverable)
                # note that the comment is either the user's comment,
                # a comment that the defect is auto-generated, or the
                # list of source defects in case of virtual defects
                
                #o = O("DQDEFECT", (d.channel, d.comment, ignore), IOVRange(starttime=d.since.real, endtime=d.until.real), True)
                o = OOO("DQDEFECT", defPayload, IOVRange(starttime=d.since.real, endtime=d.until.real), True)
                yield o
 
 
        
    def ApplySelection(self,key):
        return key=='DQ' and len(self.selectionChannelNames)!=0
 
 
    def sort_uniq_ignores(self,requirement):
        if '\\' not in requirement:
            return requirement
        x = requirement.split('\\')
        new_req = x[0] + '\\' + '\\'.join(sorted(list(set(x[1:]))))
        return new_req
    
 
    def addSelectionChannels(self, dq):
        for channels,flag in dq:
            a = []
            prefix = '!' if flag == 'green' else ''
            for x in channels.split(','):
                if x[0]=='\\':
                    self.global_ignore.update( x.lstrip('\\').split('\\') ) # "\Defect" are modifiers not a selection requirement
                else:
                    a += [ prefix+self.sort_uniq_ignores(x) ]
            if len(a)>0:
                self.selectionChannelNames += [ a ]
 
        
    def addShowChannel(self, folder, channelname, tag):
        self.channelNames += [channelname]
        from CoolRunQuery.AtlRunQueryRun import Run
        if 'DQ' not in Run.Fieldinfo:
            Run.Fieldinfo['DQ'] = {}
        if 'DefChannels' not in Run.Fieldinfo['DQ']:
            Run.Fieldinfo['DQ']['DefChannels'] = []
        Run.Fieldinfo['DQ']['DefChannels'] += [channelname]
 
    
    def __str__(self):
        if self.applySelection:
            return "SELOUT Checking the DQ defects %s" % ', '.join([', '.join(cn) for cn in self.selectionChannelNames])
        else:
            return "Retrieving DQ defects %s" % ', '.join([', '.join(cn) for cn in self.selectionChannelNames])
 
    def passes(self,values, key):
        return True
 
    def rejectRun(self,run):
        return super(DQDefectCondition,self).rejectRun(run)
 
    def mergeRanges(self,it):
        """ two adjacent defect entries are merged into one"""
        doMerge = False
        if doMerge:
            mergedByDefect = []
            x = it.next()
            (openedat,openuntil) = (x.startlb,x.endlb)
            for x in it:
                if x.startlb == openuntil:
                    openuntil = x.endlb
                else:
                    mergedByDefect += [(openedat, openuntil)]
                    openedat,openuntil = x.startlb,x.endlb
            mergedByDefect += [(openedat, openuntil)]
            return mergedByDefect
        else:
            return [(x.startlb,x.endlb) for x in it]
 
 
 
    def runAfterQuery(self,runlist):
        """
        collects, sorts, and groups defects with LBs and comments
        """
        from itertools import groupby
        from operator import attrgetter
 
        for run in runlist:
 
            for k in self.ResultKey():
                # format of run.data[k] is
                # [ <DataEntry with value = (defect_name, defect_comment or defect_composition)>, ...]
 
                #for x in run.data[k]:
                #    print ("               %r" % (x,))
                if k not in run.data.keys():
                    run.result[k] = {}
                run.stats[k] = {}
 
                all_defects = sorted(list(set([x.value[0] for x in run.data[k]])))
                
                grouped_defects = groupby(sorted(run.data[k],key=lambda x: x.value[0]),key = lambda x: x.value[0]) # first sort by defect name, then group by defect name
 
                desired_defects = [ (defect_name, self.mergeRanges(lblist)) for defect_name, lblist in grouped_defects ]
 
                run.stats[k] = { "defects": desired_defects,
                                 "primaries": {} }
 
                primaries = {}  # defect : list of primaries with lb-range
                if run.runNr in self.primaries:
                    primgroups = groupby(sorted([p for p in self.primaries[run.runNr]],key=attrgetter('channel')),key=attrgetter('channel'))
                    primaries = dict([(p,list(l)) for p,l in primgroups])
                    
                for defect in all_defects:
                    if self.__db.defect_is_virtual(defect):
                        reps = []
                        self.find_primaries(DD, self.__db.virtual_defect_logics[defect], primaries, defect, reps)
                        run.stats[k]["primaries"][defect] = reps
                    elif defect in primaries:
                        run.stats[k]["primaries"][defect] = [ DD("->%s" % def_obj.channel, def_obj.comment, def_obj.since.lumi, def_obj.until.lumi) for def_obj in primaries[defect] ]
 
 
                #import pickle
                #f = open("dev.pickle","w")
                #pickle.dump(run.stats[k]["primaries"],f)
                #f.close()
 
                #for x,l in run.stats[k]["primaries"].items():
                #    print (x)
                #    for y in sorted(l):
                #        print ("    ",y)
 
 
    def find_primaries(self, DD, defect_logic, primaries, curpath, reps):
        for pdef in defect_logic.primary_defects:
            if pdef not in primaries:
                continue
            pdefects = primaries[pdef]
            for pdeflb in pdefects:
                reps += [  DD("%s->%s" % (curpath, pdeflb.channel), pdeflb.comment, pdeflb.since.lumi, pdeflb.until.lumi) ]
        for dep in defect_logic.dependencies:
            self.find_primaries(DD, dep, primaries, curpath+"->"+dep.name, reps)