python.RunLister.coolRunLister Class Reference

Collaboration diagram for python.RunLister.coolRunLister:

Public Member Functions
def	__init__ (self, cooltdaqdbconn, cooltrigdbconn, coolstatusdbconn, oracle=False, loglevel=1)
def	close (self)
def	setSelection (self, onlyRec=False, mask=0, runType="", filenameTag="", detStatus="", detStatusTag="HEAD")
def	listFromTime (self, time1=cool.ValidityKeyMin, time2=cool.ValidityKeyMax)
def	listFromRuns (self, run1=0, run2=(1<< 31) -1)
def	runsFromTime (self, time1=cool.ValidityKeyMin, time2=cool.ValidityKeyMax)
def	runsFromSEOR (self, run1=0, run2=(1<< 31) -1)
def	correlateLB (self)
def	checkSelection (self, rec, mask, runtype, tag)
def	correlateTrigger (self)
def	listErrors (self)
def	listRuns (self, format="", lastfirst=False)
def	makeWebSummary (self, filename, header, lastfirst=False)

Public Attributes
	cooldb
	cooltrigdb
	usetrig
	coolstatusdb
	loglevel
	coolpath
	cooltlbpath
	nowtime
	onlyRec
	mask
	runType
	filenameTag
	lbrunmap
	runmap
	triglbmap
	detstatus
	detstatustag
	minrun
	maxrun
	mintime
	maxtime

Detailed Description

Extract information on ATLAS runs from online COOL information

Definition at line 164 of file RunLister.py.

Constructor & Destructor Documentation

__init__()

def python.RunLister.coolRunLister.__init__	(		self,
			cooltdaqdbconn,
			cooltrigdbconn,
			coolstatusdbconn,
			oracle = False,
			loglevel = 1
	)

Definition at line 166 of file RunLister.py.

    def __init__(self,cooltdaqdbconn,cooltrigdbconn,coolstatusdbconn,
                 oracle=False,loglevel=1):
        debug=(loglevel>1)
        try:
            self.cooldb=indirectOpen(cooltdaqdbconn,True,debug)
            if (loglevel>=1):print ("Connected to",cooltdaqdbconn,"for RunControl data")
        except Exception as e:
            print (e)
            sys.exit(-1)
        if (len(cooltrigdbconn)>0):
            try:
                self.cooltrigdb=indirectOpen(cooltrigdbconn,True,debug)
                if (loglevel>=1): print ("Connected to",cooltrigdbconn,"for CTP data")
                self.usetrig=True
            except Exception as e:
                print (e)
                sys.exit(-1)
        else:
            self.usetrig=False
        if (len(coolstatusdbconn)>0):
            try:
                self.coolstatusdb=indirectOpen(coolstatusdbconn,True,debug)
                if (loglevel>=1): print ("Connected to",coolstatusdbconn,"for detector status data")
            except Exception as e:
                print (e)
                sys.exit(-1)
        
        # store other parameters
        self.loglevel=loglevel
        self.coolpath='/TDAQ/RunCtrl'
        self.cooltlbpath='/TRIGGER/LUMI'
        self.nowtime=time.time()*1000000000
        # no restriction on initial selection
        self.onlyRec=False
        self.mask=0
        self.runType=""
        self.filenameTag=""
 
        # intial values
        self.lbrunmap={}
        self.runmap={}
        self.triglbmap={}
 

Member Function Documentation

checkSelection()

def python.RunLister.coolRunLister.checkSelection	(		self,
			rec,
			mask,
			runtype,
			tag
	)

Definition at line 403 of file RunLister.py.

    def checkSelection(self,rec,mask,runtype,tag):
        "Check whether run passes selection cuts"
        if (self.onlyRec and not rec): return False
        # mask - all specified bits must be set
        if ((self.mask & mask) != self.mask): return False
        # run type and filenametag: specified string must be included in value
        if (self.runType!="" and runtype.upper().find(self.runType.upper())==-1): return False
        if (self.filenameTag!="" and tag.find(self.filenameTag)==-1): return False
        return True
 

close()

def python.RunLister.coolRunLister.close

(

self

)

Definition at line 209 of file RunLister.py.

    def close(self):
        "Close the database connections"
        self.cooldb.closeDatabase()
        self.cooltrigdb.closeDatabase()
        self.cooldb=None
        self.cooltrigdb=None
 

correlateLB()

def python.RunLister.coolRunLister.correlateLB

(

self

)

Go through the runmap and lookup LB info, also flag missing EOR
in the error structures

Definition at line 389 of file RunLister.py.

    def correlateLB(self):
        """Go through the runmap and lookup LB info, also flag missing EOR
        in the error structures"""
        for runp in self.runmap.values():
            run=runp.run
            if run in self.lbrunmap.keys():
                lbinfo=self.lbrunmap[run]
                if (runp.maxlb is None):
                    runp.maxlb=lbinfo.lb
            else:
                runp.setErr(2)
            if runp.cleanstop is None:
                runp.setErr(0)
                

correlateTrigger()

def python.RunLister.coolRunLister.correlateTrigger

(

self

)

Definition at line 413 of file RunLister.py.

    def correlateTrigger(self):
        "Retrieve run/LB information from trigger and correlate with RunCtrl"
        srun=-1
        slbmax = None
        sstart = None
        send = None
        self.triglbmap={}
        folderTrigLBLB=self.cooltrigdb.getFolder(self.cooltlbpath+'/LBLB')
        itr=folderTrigLBLB.browseObjects((self.minrun << 32),((self.maxrun+1) << 32),cool.ChannelSelection.all())
        while itr.goToNext():
            obj=itr.currentRef()
            since=obj.since()
            run=since >> 32
            lb=since & 0xFFFFFFFF
            payload=obj.payload()
            start=payload['StartTime']
            end=payload['EndTime']
            if (run!=srun):
                # seeing a new run - store old one if needed
                if (srun>-1):
                    self.triglbmap[srun]=TrigLBParams(srun,slbmax,sstart,send)
                srun=run
                slbmax=lb
                sstart=start
                send=end
            else:
                if (lb>slbmax): slbmax=lb
                if (start<sstart): sstart=start
                if (end>send): send=end
        itr.close()
        # store last run
        if (srun>-1):
            self.triglbmap[srun]=TrigLBParams(srun,slbmax,sstart,send)
        if (self.loglevel>0):
            print ("Trigger LB map has data for %i runs" % len(self.triglbmap))
        
        # now loop through primary run list and add trigger information
        nbad=0
        npatch=0
        for runp in self.runmap.values():
            run=runp.run
            if run in self.triglbmap.keys():
                triglb=self.triglbmap[run]
                if (runp.stop is None):
                    if (self.loglevel>1):
                        print ("Trigger end time will substitute %i %i" % (run,triglb.stop))
                    runp.patchEndTime(triglb.stop)
                    npatch+=1
            else:
                nbad+=1
        if (self.loglevel>0):
            print ("Missing trigger information for %i runs, patched EOR for %i" % (nbad,npatch))
 

listErrors()

def python.RunLister.coolRunLister.listErrors

(

self

)

Definition at line 466 of file RunLister.py.

    def listErrors(self):
        "List runs which have errors to text output"
        nerr=0
        for runp in self.runmap.values():
            if (runp.errcode>0):
                print (runp.decodeErr())
                if (len(runp.errlist)>0):
                    for i in runp.errlist:
                        print ("Run %i %s" % (runp.run,i))
                nerr+=1
        print ("Total of %i runs with errors" % nerr)
 

listFromRuns()

def python.RunLister.coolRunLister.listFromRuns	(		self,
			run1 = 0,
			run2 = (1 << 31)-1
	)

Definition at line 247 of file RunLister.py.

    def listFromRuns(self,run1=0,run2=(1 << 31)-1):
        "Main entry point - setup and check data given inclusive range of runs"
        self.runmap=self.runsFromSEOR(run1,run2)
        # set the limits on run and time from the list of runs returned
        self.minrun=(1 << 31)-1
        self.maxrun=0
        self.mintime=cool.ValidityKeyMax
        self.maxtime=cool.ValidityKeyMin
        for runp in self.runmap.values():
            run=runp.run
            if (run<self.minrun): self.minrun=run
            if (run>self.maxrun): self.maxrun=run
            if (runp.start<self.mintime): self.mintime=runp.start
            if (runp.start>self.maxtime): self.maxtime=runp.start
            if (runp.stop is not None and runp.stop>self.maxtime): self.maxtime=runp.stop
        # get the LBParams info based on the derived time range
        self.lbrunmap=self.runsFromTime(self.mintime,self.maxtime)
        # add in the time/LB information and check consistency
        self.correlateLB()
        if (self.usetrig):
            self.correlateTrigger()
        
 

listFromTime()

def python.RunLister.coolRunLister.listFromTime	(		self,
			time1 = cool.ValidityKeyMin,
			time2 = cool.ValidityKeyMax
	)

Definition at line 224 of file RunLister.py.

    def listFromTime(self,time1=cool.ValidityKeyMin,time2=cool.ValidityKeyMax):
        "Main entry point - setup and check data given a range of times"
        self.lbrunmap=self.runsFromTime(time1,time2)
        # set the limits on run and time from the list of runs returned
        self.minrun=(1 << 31)-1
        self.maxrun=0
        self.mintime=cool.ValidityKeyMax
        self.maxtime=cool.ValidityKeyMin
        for (run,value) in self.lbrunmap.items():
            if (self.loglevel>1): print ("listFromTime: Process run",run)
            if (run<self.minrun): self.minrun=run
            if (run>self.maxrun): self.maxrun=run
            if (value.start<self.mintime): self.mintime=value.start
            if (value.stop>self.maxtime and value.stop<cool.ValidityKeyMax): self.maxtime=value.stop
        if (self.loglevel>0):
            print ("Run range [%i,%i] times (%s,%s)" % (self.minrun,self.maxrun,timeRep(self.mintime),timeRep(self.maxtime)))
        # now get the runmap
        self.runmap=self.runsFromSEOR(self.minrun,self.maxrun)
        # add in the time/LB information and check consistency
        self.correlateLB()
        if (self.usetrig):
            self.correlateTrigger()
            

listRuns()

def python.RunLister.coolRunLister.listRuns	(		self,
			format = "",
			lastfirst = False
	)

Definition at line 478 of file RunLister.py.

    def listRuns(self,format="",lastfirst=False):
        "List run details to text output"
        if (self.loglevel>=1):
            title="     Run   Events  LumiB"
            if ('t' in format):
                title+='            StartTime             StopTime'
            if ('a' in format):
                title+=' L1Events L2Events EFEvents'
            if ('e' in format):
                title+=' ErrC'
            if ('r' in format):
                title+=' Rec Cln'
            if ('c' in format):
                title+=' RunType                  DetectorMask'
            if ('d' in format):
                title+=' DAQConfiguration     PartitionName    FilenameTag         '
            print (title)
        runkeys=self.runmap.keys()
        runkeys.sort(reverse=lastfirst)
        for irun in runkeys:
            runp=self.runmap[irun]
            line="%8i %8i %6i" % (runp.run,noneZero(runp.storedevents),noneZero(runp.maxlb))
            if ('t' in format):
                line+=' %20s %20s' % (timeRep(runp.start),timeRep(runp.stop))
            if ('a' in format):
                line+=' %8i %8i %8i' % (noneZero(runp.l1events),noneZero(runp.l2events),noneZero(runp.efevents))
            if ('e' in format):
                line+=' %4i' % runp.errcode
            if ('r' in format):
                line+=' %3s %3s' % (boolRep(runp.rec),boolRep(runp.cleanstop))
            if ('c' in format):
                line+=' %-20s %16x' % (runp.runtype,runp.detmask)
            if ('d' in format):
                line+=' %-20s %-16s %-20s' % (runp.daqconfig,noneStr(runp.partname),runp.filetag)
            print (line)
 
        
 

makeWebSummary()

def python.RunLister.coolRunLister.makeWebSummary	(		self,
			filename,
			header,
			lastfirst = False
	)

Definition at line 516 of file RunLister.py.

    def makeWebSummary(self,filename,header,lastfirst=False):
        htfile=open(filename,'w')
        htfile.write("""<head>
<title>Run Summary for %s</title></head>
<body>
<h2>COOL/RunControl Run Summary for %s</h2>
 
Data extracted from /TDAQ/RunCtrl SOR_Params, EOR_Params and LB_Params folders
for runs %i to %i (times %s to %s) on <b>%s</b>. All time are in <b>UTC</b>,
two hours behind Geneva local time. <font color=\"008000\">Green</font> runs
were recorded with clean stop, <font color=\"FF0000\">red</font> runs were
recorded without clean stop or RunCtrl EOR record.
Black runs were not recorded.
<p>""" % (header,header,self.minrun,self.maxrun,timeRep(self.mintime),timeRep(self.maxtime),timeRep(self.nowtime)))
        htfile.write("<table border=\"0\">\n<tr align=\"left\"><th>Run</th><th>Events</th><th>L1</th><th>L2</th><th>EF</th><th>MaxLB</th><th>Start time</th><th>Stop time</th><th>Duration</th><th>Rec</th><th>Clean</th><th>RunType</th><th>DetMask</th><th>DAQConfig</th><th>Partition</th><th>FilenameTag</th></tr>\n")
        # loop over the runs
        detmasklist=[]
        runkeys=self.runmap.keys()
        runkeys.sort(reverse=lastfirst)
        #for runp in self.runmap.values():
        for irun in runkeys:
            runp=self.runmap[irun]
            if (runp.storedevents is not None):
                storeevt=str(runp.storedevents)
            else:
                storeevt="---"
            if (runp.maxlb is not None):
                maxlb=str(runp.maxlb)
            else:
                maxlb='---'
            start=timeRep(runp.start)
            stop=timeRep(runp.stop)
            if (runp.totaltime is not None):
                tottime=str(runp.totaltime)
            else:
                tottime='---'
            rec=boolRep(runp.rec)
            clean=boolRep(runp.cleanstop)
            # set colour based on status
            col='000000' # black for non-recorded runs
            if (runp.rec):
                if (runp.cleanstop is True):
                    col='008000' # green - for runs with EOR and cleanstop
                else:
                    col='FF0000' # red - for runs without EOR or cleanstop
            detmask=runp.detmask
            if (detmask==0):
                ldetmask=""
            else:
                ldetmask="<a href=#maskdecode%x>" % detmask
                if (detmask not in detmasklist): detmasklist+=[detmask]
            htfile.write("<tr style=\"color:#%s\"><td>%i</td><td>%s</td><td>%i</td><td>%i</td><td>%i</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s%x</td><td>%s</td><td>%s</td><td>%s</td></tr>\n" % (col,runp.run,storeevt,noneZero(runp.l1events),noneZero(runp.l2events),noneZero(runp.efevents),maxlb,start,stop,tottime,rec,clean,runp.runtype,ldetmask,detmask,runp.daqconfig,noneStr(runp.partname),runp.filetag))
        htfile.write("</table>\n")
        htfile.write("<p>\nTotal of %i runs analysed\n<p>" % len(self.runmap))
        # add information on decoding the detmask
        detmasklist.sort()
        htfile.write("Detector mask decoding for these runs:\n<ul>\n")
        for mask in detmasklist:
            htfile.write("<li><a name=maskdecode%x><b>%x</b>: %s</li>\n" % (mask,mask,maskDecode(mask)))
        htfile.write("</ul>\n")
        htfile.write("\n</body>\n")
        htfile.close()
        

runsFromSEOR()

def python.RunLister.coolRunLister.runsFromSEOR	(		self,
			run1 = 0,
			run2 = (1 << 31)-1
	)

Query /TDAQ/RunCtrl/LB_Params to get details of runs in runrange
Use both SOR_Params and EOR_Params to catch runs which ended badly.
Return a map of runs to RunParams objects

Definition at line 307 of file RunLister.py.

    def runsFromSEOR(self,run1=0,run2=(1 << 31)-1):
        """Query /TDAQ/RunCtrl/LB_Params to get details of runs in runrange
        Use both SOR_Params and EOR_Params to catch runs which ended badly.
        Return a map of runs to RunParams objects"""
        # get detector status information if needed
        iov1=run1 << 32
        iov2=(run2+1) << 32
        if (iov2>cool.ValidityKeyMax): iov2=cool.ValidityKeyMax
        if (self.detstatus!=""):
            if (self.loglevel>0):
                print ("Applying detector status cuts: %s" % self.detstatus)
            gooddetstatus=statusCutsToRange(self.coolstatusdb,'/GLOBAL/DETSTATUS/LBSUMM',iov1,iov2,self.detstatustag,self.detstatus)
        else:
            gooddetstatus=RangeList(iov1,iov2)
            
        runlist={}
        folderSOR_Params=self.cooldb.getFolder(self.coolpath+'/SOR_Params')
        itr=folderSOR_Params.browseObjects((run1 << 32),(run2 << 32),cool.ChannelSelection.all())
        while itr.goToNext():
            obj=itr.currentRef()
            payload=obj.payload()
            run=payload['RunNumber']
            # correction for inverted RecEnabled logic here
            if (self.checkSelection(not payload['RecordingEnabled'],payload['DetectorMask'],payload['RunType'],payload['FilenameTag']) and len(gooddetstatus.getAllowedRanges(run << 32, (run+1) << 32))>0):
                runlist[run]=RunParams(run,payload['SORTime'],payload['RunType'],payload['DAQConfiguration'],payload['DetectorMask'],payload['FilenameTag'],payload['RecordingEnabled'],payload['DataSource'])
        itr.close()
        if (self.loglevel>0):
            print ("SOR_Params has data for %i runs" % len(runlist))
 
        # now query EOR_Params and fill in missing info
        neor=0
        folderEOR_Params=self.cooldb.getFolder(self.coolpath+'/EOR_Params')
        itr=folderEOR_Params.browseObjects((run1 << 32),(run2 << 32),cool.ChannelSelection.all())
        while itr.goToNext():
            obj=itr.currentRef()
            payload=obj.payload()
            run=payload['RunNumber']
            #maxlb=(obj.until() & 0xFFFFFFF)-1
            #if (maxlb==0 or maxlb==0xFFFFFFFF): maxlb=None
            maxlb=None
            # note that run may not be found if filtered from SOR list by
            # selection
            if (run in runlist.keys()):
                runlist[run].addEORInfo(payload['EORTime'],payload['TotalTime'],payload['CleanStop'],maxlb)
                runlist[run].updateEORInfo(payload['DAQConfiguration'])
                neor+=1
        if (self.loglevel>0):
            print ("EOR_Params has data for %i runs" % neor)
        itr.close()
        
        # now query FinalSFOStats and fill in event counts
        nsfo=0
        folderSFOStat=self.cooldb.getFolder(self.coolpath+'/FinalSFOStat')
        itr=folderSFOStat.browseObjects((run1 << 32),(run2 << 32),cool.ChannelSelection.all())
        while itr.goToNext():
            obj=itr.currentRef()
            payload=obj.payload()
            run=obj.since() >> 32
            if (run in runlist.keys()):
                runlist[run].addSFOStat(payload['PartitionName'],payload['ReceivedEvents'],payload['StoredEvents'])
                nsfo+=1
        itr.close()
        if (self.loglevel>0):
            print ("FinalSFOStat has data for %i runs" % nsfo)
 
        # now query EventCounters and fill in trigger counts/partition name
        nevc=0
        folderEvtCount=self.cooldb.getFolder(self.coolpath+'/EventCounters')
        itr=folderEvtCount.browseObjects((run1 << 32),(run2 << 32),cool.ChannelSelection.all())
        while itr.goToNext():
            obj=itr.currentRef()
            payload=obj.payload()
            run=obj.since() >> 32
            if (run in runlist.keys()):
                runlist[run].addEventCounters(payload['PartitionName'],payload['L1Events'],payload['L2Events'],payload['EFEvents'],payload['RecordedEvents'])
                nevc+=1
        itr.close()
        if (self.loglevel>0):
            print ("EventCounters has data for %i runs" % nevc)
 
        return runlist
 

runsFromTime()

def python.RunLister.coolRunLister.runsFromTime	(		self,
			time1 = cool.ValidityKeyMin,
			time2 = cool.ValidityKeyMax
	)

Query /TDAQ/RunCtrl/LB_Params to get list of runs/LBs in time range,
returning a map of run numbers to LB_Params objects

Definition at line 270 of file RunLister.py.

    def runsFromTime(self,time1=cool.ValidityKeyMin,time2=cool.ValidityKeyMax):
        """Query /TDAQ/RunCtrl/LB_Params to get list of runs/LBs in time range,
        returning a map of run numbers to LB_Params objects"""
        folderLB_Params=self.cooldb.getFolder(self.coolpath+'/LB_Params')
        itr=folderLB_Params.browseObjects(time1,time2,cool.ChannelSelection.all())
        srun=-1
        slbmax = None
        sstart = None
        send = None
        runlist={}
        while itr.goToNext():
            obj=itr.currentRef()
            # ignore openended IOVs which start before the time interval
            # these will be unclosed LB records from old runs
            if (obj.until()<cool.ValidityKeyMax or obj.since()>=time1):
                payload=obj.payload()
                run=payload['RunNumber']
                lb=payload['LuminosityBlock']
                if (run!=srun):
                    # seeing a new run - store old one if needed
                    if (srun>-1):
                        runlist[srun]=LBParams(srun,slbmax,sstart,send)
                    srun=run
                    slbmax=lb
                    sstart=obj.since()
                    send=obj.until()
                else:
                    if (lb>slbmax): slbmax=lb
                    if (obj.until()>send): send=obj.until()
                    if (obj.since()<sstart): sstart=obj.since()
        itr.close()
        # store last run
        if (srun>-1):
            runlist[srun]=LBParams(srun,slbmax,sstart,send)
        if (self.loglevel>0): print ("Run list from LB_Params has %i entries" % len(runlist))
        return runlist
 

setSelection()

def python.RunLister.coolRunLister.setSelection	(		self,
			onlyRec = False,
			mask = 0,
			runType = "",
			filenameTag = "",
			detStatus = "",
			detStatusTag = "HEAD"
	)

Definition at line 216 of file RunLister.py.

    def setSelection(self,onlyRec=False,mask=0,runType="",filenameTag="",detStatus="",detStatusTag="HEAD"):
        self.onlyRec=onlyRec
        self.mask=mask
        self.runType=runType
        self.filenameTag=filenameTag
        self.detstatus=detStatus
        self.detstatustag=detStatusTag
 

Member Data Documentation

cooldb

python.RunLister.coolRunLister.cooldb

Definition at line 169 of file RunLister.py.

coolpath

python.RunLister.coolRunLister.coolpath

Definition at line 194 of file RunLister.py.

coolstatusdb

python.RunLister.coolRunLister.coolstatusdb

Definition at line 186 of file RunLister.py.

cooltlbpath

python.RunLister.coolRunLister.cooltlbpath

Definition at line 195 of file RunLister.py.

cooltrigdb

python.RunLister.coolRunLister.cooltrigdb

Definition at line 176 of file RunLister.py.

detstatus

python.RunLister.coolRunLister.detstatus

Definition at line 221 of file RunLister.py.

detstatustag

python.RunLister.coolRunLister.detstatustag

Definition at line 222 of file RunLister.py.

filenameTag

python.RunLister.coolRunLister.filenameTag

Definition at line 201 of file RunLister.py.

lbrunmap

python.RunLister.coolRunLister.lbrunmap

Definition at line 204 of file RunLister.py.

loglevel

python.RunLister.coolRunLister.loglevel

Definition at line 193 of file RunLister.py.

mask

python.RunLister.coolRunLister.mask

Definition at line 199 of file RunLister.py.

maxrun

python.RunLister.coolRunLister.maxrun

Definition at line 229 of file RunLister.py.

maxtime

python.RunLister.coolRunLister.maxtime

Definition at line 231 of file RunLister.py.

minrun

python.RunLister.coolRunLister.minrun

Definition at line 228 of file RunLister.py.

mintime

python.RunLister.coolRunLister.mintime

Definition at line 230 of file RunLister.py.

nowtime

python.RunLister.coolRunLister.nowtime

Definition at line 196 of file RunLister.py.

onlyRec

python.RunLister.coolRunLister.onlyRec

Definition at line 198 of file RunLister.py.

runmap

python.RunLister.coolRunLister.runmap

Definition at line 205 of file RunLister.py.

runType

python.RunLister.coolRunLister.runType

Definition at line 200 of file RunLister.py.

triglbmap

python.RunLister.coolRunLister.triglbmap

Definition at line 206 of file RunLister.py.

usetrig

python.RunLister.coolRunLister.usetrig

Definition at line 178 of file RunLister.py.

---

The documentation for this class was generated from the following file:

• RunLister.py