python.AtlRunQueryLib.AtlRunQuery Class Reference

Collaboration diagram for python.AtlRunQueryLib.AtlRunQuery:

Public Member Functions
def	__init__ (self, options, readoracle=False, loglevel=1, html="AUTO", origQuery="", datapath='data', parsedstring="")
def	config (self, options, readoracle, loglevel, html, origQuery, datapath, parsedstring)
def	run (self)
def	evaluate (self, runlist)
def	output (self, runlist, dic, dicsum, xmlhtmlstr, roothtmlstr)
def	finalize (self)

Public Attributes
	cmdlineOptions
	origQuery
	parsedstring
	maxNumOfRuns
	makeSummary
	makeDQSummary
	makeDQPlots
	makeDQeff
	xmlFileName
	xmlFileLabel
	querystart
	selectionOutput
	html
	prodgrl
	dictroot
	dqsumgrl
	dbbtag

Detailed Description

Definition at line 35 of file AtlRunQueryLib.py.

Constructor & Destructor Documentation

__init__()

def python.AtlRunQueryLib.AtlRunQuery.__init__	(		self,
			options,
			readoracle = False,
			loglevel = 1,
			html = "AUTO",
			origQuery = "",
			datapath = 'data',
			parsedstring = ""
	)

Definition at line 37 of file AtlRunQueryLib.py.

    def __init__(self, options, readoracle=False, loglevel=1, html="AUTO", origQuery="", datapath='data', parsedstring=""):
 
        with timer('total'):
 
            with timer('config'):
                self.config(options, readoracle, loglevel, html, origQuery, datapath, parsedstring)
 
            with timer('run all'):
                runlist = self.run()
 
            with timer('evaluate all'):
                (dic, dicsum, xmlhtmlstr, roothtmlstr) = self.evaluate(runlist)
 
            with timer('output all'):
                self.output(runlist, dic, dicsum, xmlhtmlstr, roothtmlstr)
 
            with timer('finalize'):
                self.finalize()
 
        from CoolRunQuery.utils.AtlRunQueryTimer import TimerStats as TS
        TS.printTimeSummary()
        TS.printTimeFlat()
 
 
 

Member Function Documentation

config()

def python.AtlRunQueryLib.AtlRunQuery.config	(		self,
			options,
			readoracle,
			loglevel,
			html,
			origQuery,
			datapath,
			parsedstring
	)

Definition at line 62 of file AtlRunQueryLib.py.

    def config(self, options, readoracle, loglevel, html, origQuery, datapath, parsedstring):
 
        self.cmdlineOptions = options
        self.origQuery = origQuery
        QC.datapath = datapath
        self.parsedstring = parsedstring
        self.maxNumOfRuns = -1    # in case a maximum number of to be selected runs is given
        self.makeSummary = 'summary' in options.show  # in case a summary shall be printed
        self.makeDQSummary = 'dqsummary' in options.show  # in case a DQ efficiency summary shall be printed
        self.makeDQPlots = 'dqplots' in options.show  # in case only DQ plots shall be printed 
        self.makeDQeff = 'dqeff' in options.show  # in case a DQ efficiency summary shall be printed
        self.xmlFileName  = 'MyLBCollection.xml'
        self.xmlFileLabel = 'MyLBCollection'
        self.querystart = time()
        self.selectionOutput = []
 
        # ensure the output directory exists
        if QC.datapath=='':
            QC.datapath='.'
        else:
            try:
                os.makedirs(QC.datapath, exist_ok = True)
                print("Using output directory: %s" % QC.datapath)
            except (PermissionError, FileExistsError) as err:
                print("Could not create data output directory: %s" % QC.datapath)
                print("Reason: %s" % err)
                print("Will write to local directory")
                QC.datapath = "."
 
        # if no show argument is specified, we use the minimal version
        if not self.cmdlineOptions.show:
            self.cmdlineOptions.show = ['run','time']
 
        if self.cmdlineOptions.xmlfile is not None:
            # format 'file.xml:runlist' or 'runlist:file.xml'
            str1,str2 = self.cmdlineOptions.xmlfile.strip().split(':',1)
            if   '.xml' in str1:
                self.xmlFileName = str1
                if str2 != '':
                    self.xmlFileLabel = str2
            elif '.xml' in str2:
                self.xmlFileName = str2
                if str1 != '':
                    self.xmlFileLabel = str1            
                
        if self.cmdlineOptions.html:
            self.html = self.cmdlineOptions.html
        else:
            if html=="AUTO":
                self.html = runsOnServer()
            elif html=="YES":
                self.html = True
            elif html=="NO":
                self.html = False
            else:
                raise RuntimeError ("Unknown argument for option 'html': %s" % html)
        Run.writehtml = self.html
 
        if self.cmdlineOptions.prodgrl is not None : # happens only if --nogrl is specified
            self.prodgrl = self.cmdlineOptions.prodgrl
        elif self.cmdlineOptions.xmlfile is not None :
            self.prodgrl = True # if an xml file is specified it will be produced
        else:
            self.prodgrl = not runsOnServer()
 
        if self.cmdlineOptions.dictroot is not None : # happens if --root or --noroot is specified
            self.dictroot = self.cmdlineOptions.dictroot
        else:
            self.dictroot = not runsOnServer()
            
 
        if self.cmdlineOptions.database and self.cmdlineOptions.database.upper()=='MC':
            Selector.setCondDBMC()
 
        if self.cmdlineOptions.condtag:
            Selector.condtag = self.cmdlineOptions.condtag
 
        QC.localtime = (self.cmdlineOptions.utc is None)
        QC.settimezone()
 
        # check which runs currently participate in prompt calibration
        # we need this for the web display only
        if self.html:
            nemo_dir = '/afs/cern.ch/user/a/atlcond/scratch0/nemo/prod/web/'
            try:
                f = open( nemo_dir + 'calibruns.txt' )
                for line in f:
                    try:
                        if line:
                            Run.PromptCalibRuns.append( int(line.strip()) )
                    except ValueError:
                        pass
                # read list of NEMO tasks
                # do we need this for each query, or just for the web??
                fname = os.listdir( nemo_dir + 'tasks' )
                i = 0
                for taskname in fname:
                    m = re.match(r'.*(task_(\d+)_\d+.txt)',taskname)
                    if m:
                        i+=1
                        fname, runnr = m.groups()
                        Run.NemoTasks[int(runnr)] = fname
            except IOError as err:
                print (err)
                print ("Because of this I can't mark runs that are currently in the calibration loop")
        
        #DQ summary relative to GRL
        if 'cosmics' in options.show:
            self.dqsumgrl = "PHYS_Cosmics_AllGood"
        elif 'heavyions' in options.show:
            self.dqsumgrl = "PHYS_HeavyIonP_All_Good"
        elif self.cmdlineOptions.dqsumgrl:
            self.dqsumgrl = self.cmdlineOptions.dqsumgrl
        else:
            self.dqsumgrl = 'PHYS_StandardGRL_All_Good_25ns'
            
        #defect and logic tag of defect database
        if self.cmdlineOptions.defecttag and self.cmdlineOptions.logictag:
            self.dbbtag = (self.cmdlineOptions.defecttag, self.cmdlineOptions.logictag)
        else:
            self.dbbtag = ("HEAD", "HEAD") 
 
 

evaluate()

def python.AtlRunQueryLib.AtlRunQuery.evaluate	(		self,
			runlist
	)

Definition at line 287 of file AtlRunQueryLib.py.

    def evaluate(self, runlist):
 
        # provide pickled dictionary of query result
        with timer('CreateResultDict'):
            from CoolRunQuery.output.AtlRunQuerySave import CreateResultDict
            dic, dicsum = CreateResultDict( runlist )
 
 
        # create XML file (and return pretty html output for print)
        if self.prodgrl:
            with timer('CreateXMLFile'):
                print ("Producing XML file")
                from CoolRunQuery.output.AtlRunQueryXML import CreateXMLFile
                from CoolRunQuery.AtlRunQueryVersion import SvnVersion
                xmlhtmlstr = CreateXMLFile( runlist, self.cmdlineOptions, self.origQuery, QC.datapath,
                                            self.xmlFileName, self.xmlFileLabel, SvnVersion )
        else:
            xmlhtmlstr = None
            print ("Creation of GRL disabled")
 
        if self.dictroot and len(runlist) > 0:
            with timer('CreateRootFile'):
                # create root file from dictionary - and make plots
                from CoolRunQuery.output.AtlRunQueryRoot import CreateRootFile
                rootfilename, roothtmlstr = CreateRootFile( dic )
        else:
            roothtmlstr = None
            print ("Creation of root file disabled")
 
        return (dic, dicsum, xmlhtmlstr, roothtmlstr)
        
 

finalize()

def python.AtlRunQueryLib.AtlRunQuery.finalize

(

self

)

Definition at line 374 of file AtlRunQueryLib.py.

    def finalize(self):
        coolDbConn.CloseAll()
            

output()

def python.AtlRunQueryLib.AtlRunQuery.output	(		self,
			runlist,
			dic,
			dicsum,
			xmlhtmlstr,
			roothtmlstr
	)

Definition at line 319 of file AtlRunQueryLib.py.

    def output(self, runlist, dic, dicsum, xmlhtmlstr, roothtmlstr):
 
        if self.html:
 
            with timer('GetOKS link info'):
                from CoolRunQuery.AtlRunQuerySFO import SetOKSLinks
                SetOKSLinks(runlist)
 
            # create web page
            from CoolRunQuery.html.AtlRunQueryHTML import ResultPageMaker
            #try:
            pageinfo = { 'datapath'      : QC.datapath,
                         'origQuery'     : self.origQuery,
                         'fullQuery'     : self.parsedstring,
                         'runlist'       : runlist,
                         'dic'           : dic,
                         'dicsum'        : dicsum,
                         'makeSummary'   : self.makeSummary,
                         'makeDQeff'     : self.makeDQeff,
                         'makeDQSummary' : self.makeDQSummary,
                         'makeDQPlots'   : self.makeDQPlots,
                         'dqsumgrl'      : self.dqsumgrl,
                         'dbbtag'        : self.dbbtag,
                         'roothtmlstr'   : roothtmlstr,
                         'xmlfilename'   : self.xmlFileName,
                         'xmlhtmlstr'    : xmlhtmlstr,
                         'querytime'     : time() - self.querystart,
                         'selout'        : self.selectionOutput,
                         }
            with timer("run ResultPageMaker makePage"):
                ResultPageMaker.makePage(pageinfo)
            #except ImportError, ie:
            #    print ("Can't import pagemaker, no web page made",ie)
 
        else:
            print ('---------------------------------------------------------------------')
            print (Run.header())
            runlist.reverse()
            for r in runlist:
                print (r)
 
            from CoolRunQuery.utils.AtlRunQueryUtils import addKommaToNumber, filesize
            print ('Summary:')
            for data_key, summary in dicsum.items():
                if data_key.Type==DataKey.STREAM:
                    key = data_key.ResultKey.replace('STR:','')
                    val = '%s (%s)' % ( addKommaToNumber(summary[0]), filesize(summary[1]) )
                else:
                    key = data_key.ResultKey
                    val = addKommaToNumber(summary)
                print ('%20s : %s' % (key,val))
            duration = time() - self.querystart
            print ("%20s : %g sec" % ('Total execution time',duration))
            print ('---------------------------------------------------------------------')
        

run()

def python.AtlRunQueryLib.AtlRunQuery.run

(

self

)

Definition at line 185 of file AtlRunQueryLib.py.

    def run(self):
        # is last run still open ?
        from CoolRunQuery.AtlRunQuerySFO import GetSFO_lastNruns
        sfoconnection = coolDbConn.GetSFODBConnection()
        sfocursor     = sfoconnection.cursor()
        retlist       = GetSFO_lastNruns( sfocursor, 1 )
        if 'OPENED' in retlist[0][1]:
            Run.runnropen = retlist[0][0]
 
        if self.cmdlineOptions.runlist is not None:
            runlist = self.cmdlineOptions.runlist
            if any('last' in x for x in self.cmdlineOptions.runlist):
                for idx in range(len(runlist)):
                    rrange = runlist[idx]
                    if 'last' not in rrange:
                        continue
                    rrange = rrange.replace('last','')
                    self.maxNumOfRuns = int(rrange)
                    # list of last n runs run numbers, where at least one stream is not calibration
                    retlist    = GetSFO_lastNruns( sfocursor, 1.1*self.maxNumOfRuns )
                    runlist[idx] = '%i+' % (retlist[-1][0])
 
        elif self.cmdlineOptions.timelist is not None:
            # translate time into run-range
            if 'last' in self.cmdlineOptions.timelist[0]:
                # contains 'last'
                from CoolRunQuery.utils.AtlRunQueryUtils import get_runs_last_dt
                rlist = get_runs_last_dt(self.cmdlineOptions.timelist[0])
 
                runlist = ['%i+' % rlist[-1]] if (len(rlist)>0) else ['99999999+']
 
            else:
                # translate time into run-range
                from CoolRunQuery.utils.AtlRunQueryUtils import timeStringToSecondsUTC,secondsToTimeStringUTC,get_run_range2,GetTimeRanges
                timelist = ','.join(self.cmdlineOptions.timelist)
                timeranges,timerangesHR = GetTimeRanges(timelist, intRepFnc=timeStringToSecondsUTC, maxval=time())
                timerangesAsString = [ list(map(secondsToTimeStringUTC, tr)) for tr in timeranges ]
                runranges = [ ( "%s-%s" % get_run_range2(tr[0],tr[1]) ) for tr in timerangesAsString]
                runlist = [','.join(runranges)]
 
 
        # the run selector
        rtSel = RunTimeSelector(name = 'runtime', runlist = runlist)
 
        # find the begin and end of each interesting run
        runlist = rtSel.select()
 
        # if Selector.condDB()=="CONDBR2":
        #     self.cmdlineOptions.show = [x for x in self.cmdlineOptions.show if x not in ["lhc","olclumi","olcfillparams","olclbdata"]]
        #     self.cmdlineOptions.partition=None
        #     self.cmdlineOptions.projecttag=None
        #     print ("Pre-run2 selection removed partition and projecttag from querying")
        #     print ("Pre-run2 selection removed lhc, olc from showing. Modified show list: %r" % self.cmdlineOptions.show)
            
        self.selectionOutput += ["%s" % rtSel]
 
 
        from CoolRunQuery.AtlRunQuerySelectorWorker import SelectorWorker
 
        # create all selectors that are actively select
        SelectorWorker.parseSelectorOptions(self.cmdlineOptions)
 
        # parse show option:
        SelectorWorker.parseShowOption(self.cmdlineOptions)
 
        # call setApply for all selectors that actively select
        SelectorWorker.setApplySelection()
 
        # call setShowOutput for all selectors that show
        SelectorWorker.setShowOutput()
 
        SelectorWorker.getRetrieveSelector('trigkey','TrigKeySelector')
 
        # Selectors can implement an initialize function which runs after any constructor and setShow 
        for s in SelectorWorker.getOrderedSelectorList():
            s = s.selector
            if hasattr(s, 'initialize'):
                with timer("initializing selector '%s'" % s.name):
                    s.verbose = True
                    s.initialize()
 
        # apply the selectors to initial run list
        for s in SelectorWorker.selectors():
            with timer("run selector '%s'" % s.name):
                s.verbose = True
                runlist = s.select(runlist)
                self.selectionOutput += ["%s" % s.__str__()]
            with timer("run AfterQuery for selector '%s'" % s.name):
                s.runAfterQuery(runlist)
 
        # reverse list for presentation: newest run on top
        runlist.reverse()
 
        # if "last" option used, the runlist has to be truncated
        if self.maxNumOfRuns > 0:
            print ('SELOUT Selecting the %i most recent runs' % self.maxNumOfRuns)
            runlist = runlist[0:self.maxNumOfRuns]
            self.selectionOutput += ['SELOUT Selecting the %i most recent runs' % self.maxNumOfRuns]
            
        return runlist
 
 

Member Data Documentation

cmdlineOptions

python.AtlRunQueryLib.AtlRunQuery.cmdlineOptions

Definition at line 64 of file AtlRunQueryLib.py.

dbbtag

python.AtlRunQueryLib.AtlRunQuery.dbbtag

Definition at line 180 of file AtlRunQueryLib.py.

dictroot

python.AtlRunQueryLib.AtlRunQuery.dictroot

Definition at line 128 of file AtlRunQueryLib.py.

dqsumgrl

python.AtlRunQueryLib.AtlRunQuery.dqsumgrl

Definition at line 170 of file AtlRunQueryLib.py.

html

python.AtlRunQueryLib.AtlRunQuery.html

Definition at line 108 of file AtlRunQueryLib.py.

makeDQeff

python.AtlRunQueryLib.AtlRunQuery.makeDQeff

Definition at line 72 of file AtlRunQueryLib.py.

makeDQPlots

python.AtlRunQueryLib.AtlRunQuery.makeDQPlots

Definition at line 71 of file AtlRunQueryLib.py.

makeDQSummary

python.AtlRunQueryLib.AtlRunQuery.makeDQSummary

Definition at line 70 of file AtlRunQueryLib.py.

makeSummary

python.AtlRunQueryLib.AtlRunQuery.makeSummary

Definition at line 69 of file AtlRunQueryLib.py.

maxNumOfRuns

python.AtlRunQueryLib.AtlRunQuery.maxNumOfRuns

Definition at line 68 of file AtlRunQueryLib.py.

origQuery

python.AtlRunQueryLib.AtlRunQuery.origQuery

Definition at line 65 of file AtlRunQueryLib.py.

parsedstring

python.AtlRunQueryLib.AtlRunQuery.parsedstring

Definition at line 67 of file AtlRunQueryLib.py.

prodgrl

python.AtlRunQueryLib.AtlRunQuery.prodgrl

Definition at line 121 of file AtlRunQueryLib.py.

querystart

python.AtlRunQueryLib.AtlRunQuery.querystart

Definition at line 75 of file AtlRunQueryLib.py.

selectionOutput

python.AtlRunQueryLib.AtlRunQuery.selectionOutput

Definition at line 76 of file AtlRunQueryLib.py.

xmlFileLabel

python.AtlRunQueryLib.AtlRunQuery.xmlFileLabel

Definition at line 74 of file AtlRunQueryLib.py.

xmlFileName

python.AtlRunQueryLib.AtlRunQuery.xmlFileName

Definition at line 73 of file AtlRunQueryLib.py.

---

The documentation for this class was generated from the following file:

• AtlRunQueryLib.py