ATLAS Offline Software
Public Member Functions | Public Attributes | List of all members
python.AtlRunQueryLib.AtlRunQuery Class Reference
Collaboration diagram for python.AtlRunQueryLib.AtlRunQuery:

Public Member Functions

def __init__ (self, options, readoracle=False, loglevel=1, html="AUTO", origQuery="", datapath='data', parsedstring="")
 
def config (self, options, readoracle, loglevel, html, origQuery, datapath, parsedstring)
 
def run (self)
 
def evaluate (self, runlist)
 
def output (self, runlist, dic, dicsum, xmlhtmlstr, roothtmlstr)
 
def finalize (self)
 

Public Attributes

 cmdlineOptions
 
 origQuery
 
 parsedstring
 
 maxNumOfRuns
 
 makeSummary
 
 makeDQSummary
 
 makeDQPlots
 
 makeDQeff
 
 xmlFileName
 
 xmlFileLabel
 
 querystart
 
 selectionOutput
 
 html
 
 prodgrl
 
 dictroot
 
 dqsumgrl
 
 dbbtag
 

Detailed Description

Definition at line 34 of file AtlRunQueryLib.py.

Constructor & Destructor Documentation

◆ __init__()

def python.AtlRunQueryLib.AtlRunQuery.__init__ (   self,
  options,
  readoracle = False,
  loglevel = 1,
  html = "AUTO",
  origQuery = "",
  datapath = 'data',
  parsedstring = "" 
)

Definition at line 36 of file AtlRunQueryLib.py.

36  def __init__(self, options, readoracle=False, loglevel=1, html="AUTO", origQuery="", datapath='data', parsedstring=""):
37 
38  with timer('total'):
39 
40  with timer('config'):
41  self.config(options, readoracle, loglevel, html, origQuery, datapath, parsedstring)
42 
43  with timer('run all'):
44  runlist = self.run()
45 
46  with timer('evaluate all'):
47  (dic, dicsum, xmlhtmlstr, roothtmlstr) = self.evaluate(runlist)
48 
49  with timer('output all'):
50  self.output(runlist, dic, dicsum, xmlhtmlstr, roothtmlstr)
51 
52  with timer('finalize'):
53  self.finalize()
54 
55  from CoolRunQuery.utils.AtlRunQueryTimer import TimerStats as TS
56  TS.printTimeSummary()
57  TS.printTimeFlat()
58 
59 
60 

Member Function Documentation

◆ config()

def python.AtlRunQueryLib.AtlRunQuery.config (   self,
  options,
  readoracle,
  loglevel,
  html,
  origQuery,
  datapath,
  parsedstring 
)

Definition at line 61 of file AtlRunQueryLib.py.

61  def config(self, options, readoracle, loglevel, html, origQuery, datapath, parsedstring):
62 
63  self.cmdlineOptions = options
64  self.origQuery = origQuery
65  QC.datapath = datapath
66  self.parsedstring = parsedstring
67  self.maxNumOfRuns = -1 # in case a maximum number of to be selected runs is given
68  self.makeSummary = 'summary' in options.show # in case a summary shall be printed
69  self.makeDQSummary = 'dqsummary' in options.show # in case a DQ efficiency summary shall be printed
70  self.makeDQPlots = 'dqplots' in options.show # in case only DQ plots shall be printed
71  self.makeDQeff = 'dqeff' in options.show # in case a DQ efficiency summary shall be printed
72  self.xmlFileName = 'MyLBCollection.xml'
73  self.xmlFileLabel = 'MyLBCollection'
74  self.querystart = time()
75  self.selectionOutput = []
76 
77  # ensure the output directory exists
78  if QC.datapath=='':
79  QC.datapath='.'
80  else:
81  try:
82  os.makedirs(QC.datapath, exist_ok = True)
83  print("Using output directory: %s" % QC.datapath)
84  except (PermissionError, FileExistsError) as err:
85  print("Could not create data output directory: %s" % QC.datapath)
86  print("Reason: %s" % err)
87  print("Will write to local directory")
88  QC.datapath = "."
89 
90  # if no show argument is specified, we use the minimal version
91  if not self.cmdlineOptions.show:
92  self.cmdlineOptions.show = ['run','time']
93 
94  if self.cmdlineOptions.xmlfile is not None:
95  # format 'file.xml:runlist' or 'runlist:file.xml'
96  str1,str2 = self.cmdlineOptions.xmlfile.strip().split(':',1)
97  if '.xml' in str1:
98  self.xmlFileName = str1
99  if str2 != '':
100  self.xmlFileLabel = str2
101  elif '.xml' in str2:
102  self.xmlFileName = str2
103  if str1 != '':
104  self.xmlFileLabel = str1
105 
106  if self.cmdlineOptions.html:
107  self.html = self.cmdlineOptions.html
108  else:
109  if html=="AUTO":
110  self.html = runsOnServer()
111  elif html=="YES":
112  self.html = True
113  elif html=="NO":
114  self.html = False
115  else:
116  raise RuntimeError ("Unknown argument for option 'html': %s" % html)
117  Run.writehtml = self.html
118 
119  if self.cmdlineOptions.prodgrl is not None : # happens only if --nogrl is specified
120  self.prodgrl = self.cmdlineOptions.prodgrl
121  elif self.cmdlineOptions.xmlfile is not None :
122  self.prodgrl = True # if an xml file is specified it will be produced
123  else:
124  self.prodgrl = not runsOnServer()
125 
126  if self.cmdlineOptions.dictroot is not None : # happens if --root or --noroot is specified
127  self.dictroot = self.cmdlineOptions.dictroot
128  else:
129  self.dictroot = not runsOnServer()
130 
131 
132  if self.cmdlineOptions.database and self.cmdlineOptions.database.upper()=='MC':
133  Selector.setCondDBMC()
134 
135  if self.cmdlineOptions.condtag:
136  Selector.condtag = self.cmdlineOptions.condtag
137 
138  QC.localtime = (self.cmdlineOptions.utc is None)
139  QC.settimezone()
140 
141  # check which runs currently participate in prompt calibration
142  # we need this for the web display only
143  if self.html:
144  nemo_dir = '/afs/cern.ch/user/a/atlcond/scratch0/nemo/prod/web/'
145  try:
146  f = open( nemo_dir + 'calibruns.txt' )
147  for line in f:
148  try:
149  if line:
150  Run.PromptCalibRuns.append( int(line.strip()) )
151  except ValueError:
152  pass
153  # read list of NEMO tasks
154  # do we need this for each query, or just for the web??
155  fname = os.listdir( nemo_dir + 'tasks' )
156  i = 0
157  for taskname in fname:
158  m = re.match(r'.*(task_(\d+)_\d+.txt)',taskname)
159  if m:
160  i+=1
161  fname, runnr = m.groups()
162  Run.NemoTasks[int(runnr)] = fname
163  except IOError as err:
164  print (err)
165  print ("Because of this I can't mark runs that are currently in the calibration loop")
166 
167  #DQ summary relative to GRL
168  if 'cosmics' in options.show:
169  self.dqsumgrl = "PHYS_Cosmics_AllGood"
170  elif 'heavyions' in options.show:
171  self.dqsumgrl = "PHYS_HeavyIonP_All_Good"
172  elif self.cmdlineOptions.dqsumgrl:
173  self.dqsumgrl = self.cmdlineOptions.dqsumgrl
174  else:
175  self.dqsumgrl = 'PHYS_StandardGRL_All_Good_25ns'
176 
177  #defect and logic tag of defect database
178  if self.cmdlineOptions.defecttag and self.cmdlineOptions.logictag:
179  self.dbbtag = (self.cmdlineOptions.defecttag, self.cmdlineOptions.logictag)
180  else:
181  self.dbbtag = ("HEAD", "HEAD")
182 
183 

◆ evaluate()

def python.AtlRunQueryLib.AtlRunQuery.evaluate (   self,
  runlist 
)

Definition at line 286 of file AtlRunQueryLib.py.

286  def evaluate(self, runlist):
287 
288  # provide pickled dictionary of query result
289  with timer('CreateResultDict'):
290  from CoolRunQuery.output.AtlRunQuerySave import CreateResultDict
291  dic, dicsum = CreateResultDict( runlist )
292 
293 
294  # create XML file (and return pretty html output for print)
295  if self.prodgrl:
296  with timer('CreateXMLFile'):
297  print ("Producing XML file")
298  from CoolRunQuery.output.AtlRunQueryXML import CreateXMLFile
299  from CoolRunQuery.AtlRunQueryVersion import SvnVersion
300  xmlhtmlstr = CreateXMLFile( runlist, self.cmdlineOptions, self.origQuery, QC.datapath,
301  self.xmlFileName, self.xmlFileLabel, SvnVersion )
302  else:
303  xmlhtmlstr = None
304  print ("Creation of GRL disabled")
305 
306  if self.dictroot and len(runlist) > 0:
307  with timer('CreateRootFile'):
308  # create root file from dictionary - and make plots
309  from CoolRunQuery.output.AtlRunQueryRoot import CreateRootFile
310  rootfilename, roothtmlstr = CreateRootFile( dic )
311  else:
312  roothtmlstr = None
313  print ("Creation of root file disabled")
314 
315  return (dic, dicsum, xmlhtmlstr, roothtmlstr)
316 
317 

◆ finalize()

def python.AtlRunQueryLib.AtlRunQuery.finalize (   self)

Definition at line 373 of file AtlRunQueryLib.py.

373  def finalize(self):
374  coolDbConn.CloseAll()
375 

◆ output()

def python.AtlRunQueryLib.AtlRunQuery.output (   self,
  runlist,
  dic,
  dicsum,
  xmlhtmlstr,
  roothtmlstr 
)

Definition at line 318 of file AtlRunQueryLib.py.

318  def output(self, runlist, dic, dicsum, xmlhtmlstr, roothtmlstr):
319 
320  if self.html:
321 
322  with timer('GetOKS link info'):
323  from CoolRunQuery.AtlRunQuerySFO import SetOKSLinks
324  SetOKSLinks(runlist)
325 
326  # create web page
327  from CoolRunQuery.html.AtlRunQueryHTML import ResultPageMaker
328  #try:
329  pageinfo = { 'datapath' : QC.datapath,
330  'origQuery' : self.origQuery,
331  'fullQuery' : self.parsedstring,
332  'runlist' : runlist,
333  'dic' : dic,
334  'dicsum' : dicsum,
335  'makeSummary' : self.makeSummary,
336  'makeDQeff' : self.makeDQeff,
337  'makeDQSummary' : self.makeDQSummary,
338  'makeDQPlots' : self.makeDQPlots,
339  'dqsumgrl' : self.dqsumgrl,
340  'dbbtag' : self.dbbtag,
341  'roothtmlstr' : roothtmlstr,
342  'xmlfilename' : self.xmlFileName,
343  'xmlhtmlstr' : xmlhtmlstr,
344  'querytime' : time() - self.querystart,
345  'selout' : self.selectionOutput,
346  }
347  with timer("run ResultPageMaker makePage"):
348  ResultPageMaker.makePage(pageinfo)
349  #except ImportError, ie:
350  # print ("Can't import pagemaker, no web page made",ie)
351 
352  else:
353  print ('---------------------------------------------------------------------')
354  print (Run.header())
355  runlist.reverse()
356  for r in runlist:
357  print (r)
358 
359  from CoolRunQuery.utils.AtlRunQueryUtils import addKommaToNumber, filesize
360  print ('Summary:')
361  for data_key, summary in dicsum.items():
362  if data_key.Type==DataKey.STREAM:
363  key = data_key.ResultKey.replace('STR:','')
364  val = '%s (%s)' % ( addKommaToNumber(summary[0]), filesize(summary[1]) )
365  else:
366  key = data_key.ResultKey
367  val = addKommaToNumber(summary)
368  print ('%20s : %s' % (key,val))
369  duration = time() - self.querystart
370  print ("%20s : %g sec" % ('Total execution time',duration))
371  print ('---------------------------------------------------------------------')
372 

◆ run()

def python.AtlRunQueryLib.AtlRunQuery.run (   self)

Definition at line 184 of file AtlRunQueryLib.py.

184  def run(self):
185  # is last run still open ?
186  from CoolRunQuery.AtlRunQuerySFO import GetSFO_lastNruns
187  sfoconnection = coolDbConn.GetSFODBConnection()
188  sfocursor = sfoconnection.cursor()
189  retlist = GetSFO_lastNruns( sfocursor, 1 )
190  if 'OPENED' in retlist[0][1]:
191  Run.runnropen = retlist[0][0]
192 
193  if self.cmdlineOptions.runlist is not None:
194  runlist = self.cmdlineOptions.runlist
195  if any('last' in x for x in self.cmdlineOptions.runlist):
196  for idx in range(len(runlist)):
197  rrange = runlist[idx]
198  if 'last' not in rrange:
199  continue
200  rrange = rrange.replace('last','')
201  self.maxNumOfRuns = int(rrange)
202  # list of last n runs run numbers, where at least one stream is not calibration
203  retlist = GetSFO_lastNruns( sfocursor, 1.1*self.maxNumOfRuns )
204  runlist[idx] = '%i+' % (retlist[-1][0])
205 
206  elif self.cmdlineOptions.timelist is not None:
207  # translate time into run-range
208  if 'last' in self.cmdlineOptions.timelist[0]:
209  # contains 'last'
210  from CoolRunQuery.utils.AtlRunQueryUtils import get_runs_last_dt
211  rlist = get_runs_last_dt(self.cmdlineOptions.timelist[0])
212 
213  runlist = ['%i+' % rlist[-1]] if (len(rlist)>0) else ['99999999+']
214 
215  else:
216  # translate time into run-range
217  from CoolRunQuery.utils.AtlRunQueryUtils import timeStringToSecondsUTC,secondsToTimeStringUTC,get_run_range2,GetTimeRanges
218  timelist = ','.join(self.cmdlineOptions.timelist)
219  timeranges,timerangesHR = GetTimeRanges(timelist, intRepFnc=timeStringToSecondsUTC, maxval=time())
220  timerangesAsString = [ list(map(secondsToTimeStringUTC, tr)) for tr in timeranges ]
221  runranges = [ ( "%s-%s" % get_run_range2(tr[0],tr[1]) ) for tr in timerangesAsString]
222  runlist = [','.join(runranges)]
223 
224 
225  # the run selector
226  rtSel = RunTimeSelector(name = 'runtime', runlist = runlist)
227 
228  # find the begin and end of each interesting run
229  runlist = rtSel.select()
230 
231  # if Selector.condDB()=="CONDBR2":
232  # self.cmdlineOptions.show = [x for x in self.cmdlineOptions.show if x not in ["lhc","olclumi","olcfillparams","olclbdata"]]
233  # self.cmdlineOptions.partition=None
234  # self.cmdlineOptions.projecttag=None
235  # print ("Pre-run2 selection removed partition and projecttag from querying")
236  # print ("Pre-run2 selection removed lhc, olc from showing. Modified show list: %r" % self.cmdlineOptions.show)
237 
238  self.selectionOutput += ["%s" % rtSel]
239 
240 
241  from CoolRunQuery.AtlRunQuerySelectorWorker import SelectorWorker
242 
243  # create all selectors that are actively select
244  SelectorWorker.parseSelectorOptions(self.cmdlineOptions)
245 
246  # parse show option:
247  SelectorWorker.parseShowOption(self.cmdlineOptions)
248 
249  # call setApply for all selectors that actively select
250  SelectorWorker.setApplySelection()
251 
252  # call setShowOutput for all selectors that show
253  SelectorWorker.setShowOutput()
254 
255  SelectorWorker.getRetrieveSelector('trigkey','TrigKeySelector')
256 
257  # Selectors can implement an initialize function which runs after any constructor and setShow
258  for s in SelectorWorker.getOrderedSelectorList():
259  s = s.selector
260  if hasattr(s, 'initialize'):
261  with timer("initializing selector '%s'" % s.name):
262  s.verbose = True
263  s.initialize()
264 
265  # apply the selectors to initial run list
266  for s in SelectorWorker.selectors():
267  with timer("run selector '%s'" % s.name):
268  s.verbose = True
269  runlist = s.select(runlist)
270  self.selectionOutput += ["%s" % s.__str__()]
271  with timer("run AfterQuery for selector '%s'" % s.name):
272  s.runAfterQuery(runlist)
273 
274  # reverse list for presentation: newest run on top
275  runlist.reverse()
276 
277  # if "last" option used, the runlist has to be truncated
278  if self.maxNumOfRuns > 0:
279  print ('SELOUT Selecting the %i most recent runs' % self.maxNumOfRuns)
280  runlist = runlist[0:self.maxNumOfRuns]
281  self.selectionOutput += ['SELOUT Selecting the %i most recent runs' % self.maxNumOfRuns]
282 
283  return runlist
284 
285 

Member Data Documentation

◆ cmdlineOptions

python.AtlRunQueryLib.AtlRunQuery.cmdlineOptions

Definition at line 63 of file AtlRunQueryLib.py.

◆ dbbtag

python.AtlRunQueryLib.AtlRunQuery.dbbtag

Definition at line 179 of file AtlRunQueryLib.py.

◆ dictroot

python.AtlRunQueryLib.AtlRunQuery.dictroot

Definition at line 127 of file AtlRunQueryLib.py.

◆ dqsumgrl

python.AtlRunQueryLib.AtlRunQuery.dqsumgrl

Definition at line 169 of file AtlRunQueryLib.py.

◆ html

python.AtlRunQueryLib.AtlRunQuery.html

Definition at line 107 of file AtlRunQueryLib.py.

◆ makeDQeff

python.AtlRunQueryLib.AtlRunQuery.makeDQeff

Definition at line 71 of file AtlRunQueryLib.py.

◆ makeDQPlots

python.AtlRunQueryLib.AtlRunQuery.makeDQPlots

Definition at line 70 of file AtlRunQueryLib.py.

◆ makeDQSummary

python.AtlRunQueryLib.AtlRunQuery.makeDQSummary

Definition at line 69 of file AtlRunQueryLib.py.

◆ makeSummary

python.AtlRunQueryLib.AtlRunQuery.makeSummary

Definition at line 68 of file AtlRunQueryLib.py.

◆ maxNumOfRuns

python.AtlRunQueryLib.AtlRunQuery.maxNumOfRuns

Definition at line 67 of file AtlRunQueryLib.py.

◆ origQuery

python.AtlRunQueryLib.AtlRunQuery.origQuery

Definition at line 64 of file AtlRunQueryLib.py.

◆ parsedstring

python.AtlRunQueryLib.AtlRunQuery.parsedstring

Definition at line 66 of file AtlRunQueryLib.py.

◆ prodgrl

python.AtlRunQueryLib.AtlRunQuery.prodgrl

Definition at line 120 of file AtlRunQueryLib.py.

◆ querystart

python.AtlRunQueryLib.AtlRunQuery.querystart

Definition at line 74 of file AtlRunQueryLib.py.

◆ selectionOutput

python.AtlRunQueryLib.AtlRunQuery.selectionOutput

Definition at line 75 of file AtlRunQueryLib.py.

◆ xmlFileLabel

python.AtlRunQueryLib.AtlRunQuery.xmlFileLabel

Definition at line 73 of file AtlRunQueryLib.py.

◆ xmlFileName

python.AtlRunQueryLib.AtlRunQuery.xmlFileName

Definition at line 72 of file AtlRunQueryLib.py.


The documentation for this class was generated from the following file:
python.ZdcOnlineRecMonitorConfig.evaluate
evaluate
Definition: ZdcOnlineRecMonitorConfig.py:344
python.tests.PyTestsLib.finalize
def finalize(self)
_info( "content of StoreGate..." ) self.sg.dump()
Definition: PyTestsLib.py:50
python.utils.AtlRunQueryUtils.get_runs_last_dt
def get_runs_last_dt(last)
Definition: AtlRunQueryUtils.py:589
python.utils.AtlRunQueryUtils.filesize
def filesize(no)
Definition: AtlRunQueryUtils.py:197
python.utils.AtlRunQueryUtils.addKommaToNumber
def addKommaToNumber(no)
Definition: AtlRunQueryUtils.py:189
python.base_data.config
config
Definition: base_data.py:20
python.output.AtlRunQuerySave.CreateResultDict
def CreateResultDict(runlist)
Definition: AtlRunQuerySave.py:22
run
int run(int argc, char *argv[])
Definition: ttree2hdf5.cxx:28
python.utils.AtlRunQueryUtils.GetTimeRanges
def GetTimeRanges(timeranges, intRepFnc=timeStringToSecondsUTC, maxval=1<< 30)
Definition: AtlRunQueryUtils.py:368
python.utils.AtlRunQueryUtils.runsOnServer
def runsOnServer()
Definition: AtlRunQueryUtils.py:47
python.utils.AtlRunQueryTimer.timer
def timer(name, disabled=False)
Definition: AtlRunQueryTimer.py:85
python.utils.AtlRunQueryUtils.get_run_range2
def get_run_range2(start, end=None)
Definition: AtlRunQueryUtils.py:622
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:194
python.AtlRunQuerySFO.SetOKSLinks
def SetOKSLinks(runlist)
The following function is used to get OKS info.
Definition: AtlRunQuerySFO.py:279
python.output.AtlRunQueryXML.CreateXMLFile
def CreateXMLFile(runlist, options, origQuery, datapath, xmlfname, xmllabel, svnversion='Unknown')
Definition: AtlRunQueryXML.py:73
histSizes.list
def list(name, path='/')
Definition: histSizes.py:38
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:25
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
python.AtlRunQuerySFO.GetSFO_lastNruns
def GetSFO_lastNruns(cursor, nruns)
Definition: AtlRunQuerySFO.py:266
python.processes.powheg.ZZ.ZZ.__init__
def __init__(self, base_directory, **kwargs)
Constructor: all process options are set here.
Definition: ZZ.py:18
Trk::open
@ open
Definition: BinningType.h:40
python.CaloAddPedShiftConfig.int
int
Definition: CaloAddPedShiftConfig.py:45
python.output.AtlRunQueryRoot.CreateRootFile
def CreateRootFile(dic)
Definition: AtlRunQueryRoot.py:784
Trk::split
@ split
Definition: LayerMaterialProperties.h:38