python.AtlRunQueryLib.AtlRunQuery Class Reference

Collaboration diagram for python.AtlRunQueryLib.AtlRunQuery:

Public Member Functions
	__init__ (self, options, readoracle=False, loglevel=1, html="AUTO", origQuery="", datapath='data', parsedstring="")
	config (self, options, readoracle, loglevel, html, origQuery, datapath, parsedstring)
	run (self)
	evaluate (self, runlist)
	output (self, runlist, dic, dicsum, xmlhtmlstr, roothtmlstr)
	finalize (self)

Public Attributes
	cmdlineOptions = options
	origQuery = origQuery
	parsedstring = parsedstring
int	maxNumOfRuns = -1
str	makeSummary = 'summary' in options.show
str	makeDQSummary = 'dqsummary' in options.show
str	makeDQPlots = 'dqplots' in options.show
str	makeDQeff = 'dqeff' in options.show
str	xmlFileName = 'MyLBCollection.xml'
str	xmlFileLabel = 'MyLBCollection'
	querystart = time()
list	selectionOutput = []
bool	html = self.cmdlineOptions.html
bool	prodgrl = self.cmdlineOptions.prodgrl
	dictroot = self.cmdlineOptions.dictroot
str	dqsumgrl = "PHYS_Cosmics_AllGood"
tuple	dbbtag = (self.cmdlineOptions.defecttag, self.cmdlineOptions.logictag)

Detailed Description

Definition at line 34 of file AtlRunQueryLib.py.

Constructor & Destructor Documentation

__init__()

python.AtlRunQueryLib.AtlRunQuery.__init__	(		self,
			options,
			readoracle = False,
			loglevel = 1,
			html = "AUTO",
			origQuery = "",
			datapath = 'data',
			parsedstring = "" )

Definition at line 36 of file AtlRunQueryLib.py.

    def __init__(self, options, readoracle=False, loglevel=1, html="AUTO", origQuery="", datapath='data', parsedstring=""):
 
        with timer('total'):
 
            with timer('config'):
                self.config(options, readoracle, loglevel, html, origQuery, datapath, parsedstring)
 
            with timer('run all'):
                runlist = self.run()
 
            with timer('evaluate all'):
                (dic, dicsum, xmlhtmlstr, roothtmlstr) = self.evaluate(runlist)
 
            with timer('output all'):
                self.output(runlist, dic, dicsum, xmlhtmlstr, roothtmlstr)
 
            with timer('finalize'):
                self.finalize()
 
        from CoolRunQuery.utils.AtlRunQueryTimer import TimerStats as TS
        TS.printTimeSummary()
        TS.printTimeFlat()
 
 
 

Member Function Documentation

config()

python.AtlRunQueryLib.AtlRunQuery.config	(		self,
			options,
			readoracle,
			loglevel,
			html,
			origQuery,
			datapath,
			parsedstring )

Definition at line 61 of file AtlRunQueryLib.py.

    def config(self, options, readoracle, loglevel, html, origQuery, datapath, parsedstring):
 
        self.cmdlineOptions = options
        self.origQuery = origQuery
        QC.datapath = datapath
        self.parsedstring = parsedstring
        self.maxNumOfRuns = -1    # in case a maximum number of to be selected runs is given
        self.makeSummary = 'summary' in options.show  # in case a summary shall be printed
        self.makeDQSummary = 'dqsummary' in options.show  # in case a DQ efficiency summary shall be printed
        self.makeDQPlots = 'dqplots' in options.show  # in case only DQ plots shall be printed 
        self.makeDQeff = 'dqeff' in options.show  # in case a DQ efficiency summary shall be printed
        self.xmlFileName  = 'MyLBCollection.xml'
        self.xmlFileLabel = 'MyLBCollection'
        self.querystart = time()
        self.selectionOutput = []
 
        # ensure the output directory exists
        if QC.datapath=='':
            QC.datapath='.'
        else:
            try:
                os.makedirs(QC.datapath, exist_ok = True)
                print("Using output directory: %s" % QC.datapath)
            except (PermissionError, FileExistsError) as err:
                print("Could not create data output directory: %s" % QC.datapath)
                print("Reason: %s" % err)
                print("Will write to local directory")
                QC.datapath = "."
 
        # if no show argument is specified, we use the minimal version
        if not self.cmdlineOptions.show:
            self.cmdlineOptions.show = ['run','time']
 
        if self.cmdlineOptions.xmlfile is not None:
            # format 'file.xml:runlist' or 'runlist:file.xml'
            str1,str2 = self.cmdlineOptions.xmlfile.strip().split(':',1)
            if   '.xml' in str1:
                self.xmlFileName = str1
                if str2 != '':
                    self.xmlFileLabel = str2
            elif '.xml' in str2:
                self.xmlFileName = str2
                if str1 != '':
                    self.xmlFileLabel = str1            
                
        if self.cmdlineOptions.html:
            self.html = self.cmdlineOptions.html
        else:
            if html=="AUTO":
                self.html = runsOnServer()
            elif html=="YES":
                self.html = True
            elif html=="NO":
                self.html = False
            else:
                raise RuntimeError ("Unknown argument for option 'html': %s" % html)
        Run.writehtml = self.html
 
        if self.cmdlineOptions.prodgrl is not None : # happens only if --nogrl is specified
            self.prodgrl = self.cmdlineOptions.prodgrl
        elif self.cmdlineOptions.xmlfile is not None :
            self.prodgrl = True # if an xml file is specified it will be produced
        else:
            self.prodgrl = not runsOnServer()
 
        if self.cmdlineOptions.dictroot is not None : # happens if --root or --noroot is specified
            self.dictroot = self.cmdlineOptions.dictroot
        else:
            self.dictroot = not runsOnServer()
            
 
        if self.cmdlineOptions.database and self.cmdlineOptions.database.upper()=='MC':
            Selector.setCondDBMC()
 
        if self.cmdlineOptions.condtag:
            Selector.condtag = self.cmdlineOptions.condtag
 
        QC.localtime = (self.cmdlineOptions.utc is None)
        QC.settimezone()
 
        # check which runs currently participate in prompt calibration
        # we need this for the web display only
        if self.html:
            nemo_dir = '/afs/cern.ch/user/a/atlcond/scratch0/nemo/prod/web/'
            try:
                f = open( nemo_dir + 'calibruns.txt' )
                for line in f:
                    try:
                        if line:
                            Run.PromptCalibRuns.append( int(line.strip()) )
                    except ValueError:
                        pass
                # read list of NEMO tasks
                # do we need this for each query, or just for the web??
                fname = os.listdir( nemo_dir + 'tasks' )
                i = 0
                for taskname in fname:
                    m = re.match(r'.*(task_(\d+)_\d+.txt)',taskname)
                    if m:
                        i+=1
                        fname, runnr = m.groups()
                        Run.NemoTasks[int(runnr)] = fname
            except IOError as err:
                print (err)
                print ("Because of this I can't mark runs that are currently in the calibration loop")
        
        #DQ summary relative to GRL
        if 'cosmics' in options.show:
            self.dqsumgrl = "PHYS_Cosmics_AllGood"
        elif 'heavyions' in options.show:
            self.dqsumgrl = "PHYS_HeavyIonP_All_Good"
        elif self.cmdlineOptions.dqsumgrl:
            self.dqsumgrl = self.cmdlineOptions.dqsumgrl
        else:
            self.dqsumgrl = 'PHYS_StandardGRL_All_Good_25ns'
            
        #defect and logic tag of defect database
        if self.cmdlineOptions.defecttag and self.cmdlineOptions.logictag:
            self.dbbtag = (self.cmdlineOptions.defecttag, self.cmdlineOptions.logictag)
        else:
            self.dbbtag = ("HEAD", "HEAD") 
 
 

evaluate()

python.AtlRunQueryLib.AtlRunQuery.evaluate	(		self,
			runlist )

Definition at line 286 of file AtlRunQueryLib.py.

    def evaluate(self, runlist):
 
        # provide pickled dictionary of query result
        with timer('CreateResultDict'):
            from CoolRunQuery.output.AtlRunQuerySave import CreateResultDict
            dic, dicsum = CreateResultDict( runlist )
 
 
        # create XML file (and return pretty html output for print)
        if self.prodgrl:
            with timer('CreateXMLFile'):
                print ("Producing XML file")
                from CoolRunQuery.output.AtlRunQueryXML import CreateXMLFile
                from CoolRunQuery.AtlRunQueryVersion import SvnVersion
                xmlhtmlstr = CreateXMLFile( runlist, self.cmdlineOptions, self.origQuery, QC.datapath,
                                            self.xmlFileName, self.xmlFileLabel, SvnVersion )
        else:
            xmlhtmlstr = None
            print ("Creation of GRL disabled")
 
        if self.dictroot and len(runlist) > 0:
            with timer('CreateRootFile'):
                # create root file from dictionary - and make plots
                from CoolRunQuery.output.AtlRunQueryRoot import CreateRootFile
                rootfilename, roothtmlstr = CreateRootFile( dic )
        else:
            roothtmlstr = None
            print ("Creation of root file disabled")
 
        return (dic, dicsum, xmlhtmlstr, roothtmlstr)
        
 

finalize()

python.AtlRunQueryLib.AtlRunQuery.finalize

(

self

)

Definition at line 373 of file AtlRunQueryLib.py.

    def finalize(self):
        coolDbConn.CloseAll()
            

output()

python.AtlRunQueryLib.AtlRunQuery.output	(		self,
			runlist,
			dic,
			dicsum,
			xmlhtmlstr,
			roothtmlstr )

Definition at line 318 of file AtlRunQueryLib.py.

    def output(self, runlist, dic, dicsum, xmlhtmlstr, roothtmlstr):
 
        if self.html:
 
            with timer('GetOKS link info'):
                from CoolRunQuery.AtlRunQuerySFO import SetOKSLinks
                SetOKSLinks(runlist)
 
            # create web page
            from CoolRunQuery.html.AtlRunQueryHTML import ResultPageMaker
            #try:
            pageinfo = { 'datapath'      : QC.datapath,
                         'origQuery'     : self.origQuery,
                         'fullQuery'     : self.parsedstring,
                         'runlist'       : runlist,
                         'dic'           : dic,
                         'dicsum'        : dicsum,
                         'makeSummary'   : self.makeSummary,
                         'makeDQeff'     : self.makeDQeff,
                         'makeDQSummary' : self.makeDQSummary,
                         'makeDQPlots'   : self.makeDQPlots,
                         'dqsumgrl'      : self.dqsumgrl,
                         'dbbtag'        : self.dbbtag,
                         'roothtmlstr'   : roothtmlstr,
                         'xmlfilename'   : self.xmlFileName,
                         'xmlhtmlstr'    : xmlhtmlstr,
                         'querytime'     : time() - self.querystart,
                         'selout'        : self.selectionOutput,
                         }
            with timer("run ResultPageMaker makePage"):
                ResultPageMaker.makePage(pageinfo)
            #except ImportError, ie:
            #    print ("Can't import pagemaker, no web page made",ie)
 
        else:
            print ('---------------------------------------------------------------------')
            print (Run.header())
            runlist.reverse()
            for r in runlist:
                print (r)
 
            from CoolRunQuery.utils.AtlRunQueryUtils import addKommaToNumber, filesize
            print ('Summary:')
            for data_key, summary in dicsum.items():
                if data_key.Type==DataKey.STREAM:
                    key = data_key.ResultKey.replace('STR:','')
                    val = '%s (%s)' % ( addKommaToNumber(summary[0]), filesize(summary[1]) )
                else:
                    key = data_key.ResultKey
                    val = addKommaToNumber(summary)
                print ('%20s : %s' % (key,val))
            duration = time() - self.querystart
            print ("%20s : %g sec" % ('Total execution time',duration))
            print ('---------------------------------------------------------------------')
        

run()

python.AtlRunQueryLib.AtlRunQuery.run

(

self

)

Definition at line 184 of file AtlRunQueryLib.py.

    def run(self):
        # is last run still open ?
        from CoolRunQuery.AtlRunQuerySFO import GetSFO_lastNruns
        sfoconnection = coolDbConn.GetSFODBConnection()
        sfocursor     = sfoconnection.cursor()
        retlist       = GetSFO_lastNruns( sfocursor, 1 )
        if 'OPENED' in retlist[0][1]:
            Run.runnropen = retlist[0][0]
 
        if self.cmdlineOptions.runlist is not None:
            runlist = self.cmdlineOptions.runlist
            if any('last' in x for x in self.cmdlineOptions.runlist):
                for idx in range(len(runlist)):
                    rrange = runlist[idx]
                    if 'last' not in rrange:
                        continue
                    rrange = rrange.replace('last','')
                    self.maxNumOfRuns = int(rrange)
                    # list of last n runs run numbers, where at least one stream is not calibration
                    retlist    = GetSFO_lastNruns( sfocursor, 1.1*self.maxNumOfRuns )
                    runlist[idx] = '%i+' % (retlist[-1][0])
 
        elif self.cmdlineOptions.timelist is not None:
            # translate time into run-range
            if 'last' in self.cmdlineOptions.timelist[0]:
                # contains 'last'
                from CoolRunQuery.utils.AtlRunQueryUtils import get_runs_last_dt
                rlist = get_runs_last_dt(self.cmdlineOptions.timelist[0])
 
                runlist = ['%i+' % rlist[-1]] if (len(rlist)>0) else ['99999999+']
 
            else:
                # translate time into run-range
                from CoolRunQuery.utils.AtlRunQueryUtils import timeStringToSecondsUTC,secondsToTimeStringUTC,get_run_range2,GetTimeRanges
                timelist = ','.join(self.cmdlineOptions.timelist)
                timeranges,timerangesHR = GetTimeRanges(timelist, intRepFnc=timeStringToSecondsUTC, maxval=time())
                timerangesAsString = [ list(map(secondsToTimeStringUTC, tr)) for tr in timeranges ]
                runranges = [ ( "%s-%s" % get_run_range2(tr[0],tr[1]) ) for tr in timerangesAsString]
                runlist = [','.join(runranges)]
 
 
        # the run selector
        rtSel = RunTimeSelector(name = 'runtime', runlist = runlist)
 
        # find the begin and end of each interesting run
        runlist = rtSel.select()
 
        # if Selector.condDB()=="CONDBR2":
        #     self.cmdlineOptions.show = [x for x in self.cmdlineOptions.show if x not in ["lhc","olclumi","olcfillparams","olclbdata"]]
        #     self.cmdlineOptions.partition=None
        #     self.cmdlineOptions.projecttag=None
        #     print ("Pre-run2 selection removed partition and projecttag from querying")
        #     print ("Pre-run2 selection removed lhc, olc from showing. Modified show list: %r" % self.cmdlineOptions.show)
            
        self.selectionOutput += ["%s" % rtSel]
 
 
        from CoolRunQuery.AtlRunQuerySelectorWorker import SelectorWorker
 
        # create all selectors that are actively select
        SelectorWorker.parseSelectorOptions(self.cmdlineOptions)
 
        # parse show option:
        SelectorWorker.parseShowOption(self.cmdlineOptions)
 
        # call setApply for all selectors that actively select
        SelectorWorker.setApplySelection()
 
        # call setShowOutput for all selectors that show
        SelectorWorker.setShowOutput()
 
        SelectorWorker.getRetrieveSelector('trigkey','TrigKeySelector')
 
        # Selectors can implement an initialize function which runs after any constructor and setShow 
        for s in SelectorWorker.getOrderedSelectorList():
            s = s.selector
            if hasattr(s, 'initialize'):
                with timer("initializing selector '%s'" % s.name):
                    s.verbose = True
                    s.initialize()
 
        # apply the selectors to initial run list
        for s in SelectorWorker.selectors():
            with timer("run selector '%s'" % s.name):
                s.verbose = True
                runlist = s.select(runlist)
                self.selectionOutput += ["%s" % s.__str__()]
            with timer("run AfterQuery for selector '%s'" % s.name):
                s.runAfterQuery(runlist)
 
        # reverse list for presentation: newest run on top
        runlist.reverse()
 
        # if "last" option used, the runlist has to be truncated
        if self.maxNumOfRuns > 0:
            print ('SELOUT Selecting the %i most recent runs' % self.maxNumOfRuns)
            runlist = runlist[0:self.maxNumOfRuns]
            self.selectionOutput += ['SELOUT Selecting the %i most recent runs' % self.maxNumOfRuns]
            
        return runlist
 
 

Member Data Documentation

cmdlineOptions

python.AtlRunQueryLib.AtlRunQuery.cmdlineOptions = options

Definition at line 63 of file AtlRunQueryLib.py.

dbbtag

tuple python.AtlRunQueryLib.AtlRunQuery.dbbtag = (self.cmdlineOptions.defecttag, self.cmdlineOptions.logictag)

Definition at line 179 of file AtlRunQueryLib.py.

dictroot

python.AtlRunQueryLib.AtlRunQuery.dictroot = self.cmdlineOptions.dictroot

Definition at line 127 of file AtlRunQueryLib.py.

dqsumgrl

str python.AtlRunQueryLib.AtlRunQuery.dqsumgrl = "PHYS_Cosmics_AllGood"

Definition at line 169 of file AtlRunQueryLib.py.

html

bool python.AtlRunQueryLib.AtlRunQuery.html = self.cmdlineOptions.html

Definition at line 107 of file AtlRunQueryLib.py.

makeDQeff

str python.AtlRunQueryLib.AtlRunQuery.makeDQeff = 'dqeff' in options.show

Definition at line 71 of file AtlRunQueryLib.py.

makeDQPlots

str python.AtlRunQueryLib.AtlRunQuery.makeDQPlots = 'dqplots' in options.show

Definition at line 70 of file AtlRunQueryLib.py.

makeDQSummary

str python.AtlRunQueryLib.AtlRunQuery.makeDQSummary = 'dqsummary' in options.show

Definition at line 69 of file AtlRunQueryLib.py.

makeSummary

str python.AtlRunQueryLib.AtlRunQuery.makeSummary = 'summary' in options.show

Definition at line 68 of file AtlRunQueryLib.py.

maxNumOfRuns

python.AtlRunQueryLib.AtlRunQuery.maxNumOfRuns = -1

Definition at line 67 of file AtlRunQueryLib.py.

origQuery

python.AtlRunQueryLib.AtlRunQuery.origQuery = origQuery

Definition at line 64 of file AtlRunQueryLib.py.

parsedstring

python.AtlRunQueryLib.AtlRunQuery.parsedstring = parsedstring

Definition at line 66 of file AtlRunQueryLib.py.

prodgrl

bool python.AtlRunQueryLib.AtlRunQuery.prodgrl = self.cmdlineOptions.prodgrl

Definition at line 120 of file AtlRunQueryLib.py.

querystart

python.AtlRunQueryLib.AtlRunQuery.querystart = time()

Definition at line 74 of file AtlRunQueryLib.py.

selectionOutput

list python.AtlRunQueryLib.AtlRunQuery.selectionOutput = []

Definition at line 75 of file AtlRunQueryLib.py.

xmlFileLabel

str python.AtlRunQueryLib.AtlRunQuery.xmlFileLabel = 'MyLBCollection'

Definition at line 73 of file AtlRunQueryLib.py.

xmlFileName

str python.AtlRunQueryLib.AtlRunQuery.xmlFileName = 'MyLBCollection.xml'

Definition at line 72 of file AtlRunQueryLib.py.

---

The documentation for this class was generated from the following file:

• AtlRunQueryLib.py