ATLAS Offline Software
Classes | Functions | Variables
logParser Namespace Reference

Classes

class  bcolors
 

Functions

def main ()
 
def getJOsList (JOsDict)
 
def checkBlackList (branch, cache, MCJobOptions, outnamedir, JOsList)
 
def loginfo (out1, out2)
 
def loggood (out1, out2)
 
def logerr (out1, out2)
 
def logwarn (out1, out2)
 
def checkLine (line, lineIdentifier, dict, splitby)
 

Variables

 parser = optparse.OptionParser(usage=__doc__)
 
 default
 
 dest
 
 metavar
 
 help
 
 action
 
 False
 
 opts
 
 fileargs
 

Function Documentation

◆ checkBlackList()

def logParser.checkBlackList (   branch,
  cache,
  MCJobOptions,
  outnamedir,
  JOsList 
)

Definition at line 398 of file logParser.py.

398 def checkBlackList(branch,cache,MCJobOptions,outnamedir,JOsList) :
399 
400  import getpass
401  user = getpass.getuser()
402 
403  aJOs=[]
404  for l in JOsList:
405  if "MC15." in l:
406  aJOs.append(l)
407  print("JOSH",aJOs)
408 
409  blacklist = 'svn export svn+ssh://svn.cern.ch/reps/atlasoff/Generators/'+MCJobOptions+'/trunk/common/BlackList_caches.txt'
410  tmpblackfile = "%s/BlackList_caches.txt" % (outnamedir)
411 
412  isError = False
413  if 'MC15' in MCJobOptions :
414  svnexcomm='%s %s' % (blacklist,tmpblackfile)
415  retcode = subprocess.Popen(svnexcomm, shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE).communicate()
416  if retcode[1].find("exist") != -1 or retcode[1].find("cannot") != -1 :
417  logerr("","export failed= BlackList_caches.txt" )
418  isError = True
419 
420  bfile = open(tmpblackfile)
421  for line in bfile:
422  if not line.strip():
423  continue
424 
425  bad = "".join(line.split()).split(",")
426 
427  #Match Generator and release type e.g. AtlasProduction, MCProd
428  if any( bad[2] in s for s in aJOs ) and branch in bad[0]:
429  #Match cache
430  cacheMatch=True
431  for i,c in enumerate(cache.split('.')):
432  if not c == bad[1].split('.')[i]:
433  cacheMatch=False
434  break
435 
436  if cacheMatch:
437  #logerr("", "Combination %s_%s for %s it is NOT allowed"%(bad[0],bad[1],bad[2]))
438  isError = True
439  return isError
440 
441 
442  return isError
443  else:
444  return False
445 
446 

◆ checkLine()

def logParser.checkLine (   line,
  lineIdentifier,
  dict,
  splitby 
)

Definition at line 481 of file logParser.py.

481 def checkLine(line, lineIdentifier, dict, splitby ):
482  if lineIdentifier in line:
483  for param in dict:
484  if param in line:
485  if len(line.split(splitby))==0:
486  raise RuntimeError("Found bad entry %s"%line)
487  else:
488  thing="".join(line.split(lineIdentifier)[1].split(splitby)[1:]).strip()
489  dict[param].append(thing)
490  break
491 
492 

◆ getJOsList()

def logParser.getJOsList (   JOsDict)

Definition at line 387 of file logParser.py.

387 def getJOsList(JOsDict):
388  liststr=''
389  if len(JOsDict["including file \"MC15JobOptions/"]):
390  liststr+="|".join(JOsDict["including file \"MC15JobOptions/"]).replace("nonStandard/","")
391  if len(JOsDict["including file \"MC15."]):
392  liststr+="|".join(JOsDict["including file \"MC15."].replace("nonStandard/",""))
393  liststr=liststr.replace('MC15JobOptions/','').replace('"','').replace('including file','').replace(' ','')
394  tmplist=liststr.split('|')
395  return tmplist
396 
397 

◆ logerr()

def logParser.logerr (   out1,
  out2 
)

Definition at line 475 of file logParser.py.

475 def logerr(out1,out2):
476  print(str(out1),bcolors.FAIL + str(out2) + bcolors.ENDC)

◆ loggood()

def logParser.loggood (   out1,
  out2 
)

Definition at line 473 of file logParser.py.

473 def loggood(out1,out2):
474  print(str(out1),bcolors.OKGREEN + str(out2) + bcolors.ENDC)

◆ loginfo()

def logParser.loginfo (   out1,
  out2 
)

Definition at line 471 of file logParser.py.

471 def loginfo(out1,out2):
472  print(str(out1),bcolors.OKBLUE + str(out2) + bcolors.ENDC)

◆ logwarn()

def logParser.logwarn (   out1,
  out2 
)

Definition at line 477 of file logParser.py.

477 def logwarn(out1,out2):
478  print(str(out1),bcolors.WARNING + str(out2) + bcolors.ENDC)
479 
480 # find identifiers and variables in log file lines

◆ main()

def logParser.main ( )

Definition at line 17 of file logParser.py.

17 def main():
18 
19 
20  if opts.INPUT_FILE=="-":
21  parser.print_help()
22  return
23 
24  # define dictionaries with keys as variables to be searched for and values to store the results
25 
26  JOsDict={
27  'using release':[],
28  "including file \"MC15JobOptions/":[],
29  "including file \"MC15.":[]
30  }
31 
32  testHepMCDict={
33  'Events passed':[],
34  'Efficiency':[]
35  }
36 
37  countHepMCDict={
38  'Events passing all checks and written':[]
39  }
40 
41  evgenFilterSeqDict={
42  'Weighted Filter Efficiency':[],
43  'Filter Efficiency':[]
44  }
45 
46  simTimeEstimateDict={
47  'RUN INFORMATION':[]
48  }
49 
50  metaDataDict={
51  'physicsComment':[],
52  'generatorName':[],
53  'generatorTune':[],
54  'keywords':[],
55  'specialConfig':[],
56  'contactPhysicist':[],
57  'randomSeed':[],
58  'genFilterNames':[],
59  'cross-section (nb)':[],
60  'generator':[],
61  'weights':[],
62  'GenFiltEff':[]
63  }
64 
65  generateDict={
66  'minevents':[0]
67  }
68 
69  perfMonDict={
70  'snapshot_post_fin':[],
71  'last -evt vmem':[]
72  }
73 
74 
75  # open and read log file
76  file=open(opts.INPUT_FILE,"r")
77  lines=file.readlines()
78 
79  # check each line
80  for line in lines:
81 
82  checkLine(line,'Py:Athena',JOsDict,'INFO')
83 
84  checkLine(line,'MetaData',metaDataDict,'=')
85  checkLine(line,'Py:Generate',generateDict,'=')
86 
87  checkLine(line,'Py:PerfMonSvc',perfMonDict,':')
88  checkLine(line,'PMonSD',perfMonDict,'---')
89 
90  checkLine(line,'TestHepMC',testHepMCDict,'=')
91  checkLine(line,'Py:EvgenFilterSeq',evgenFilterSeqDict,'=')
92  checkLine(line,'CountHepMC',countHepMCDict,'=')
93  checkLine(line,'SimTimeEstimate',simTimeEstimateDict,'|')
94 
95 
96 
97  # print results
98  JOsErrors=[]
99  print("")
100  print("---------------------")
101  print("jobOptions and release:")
102  print("---------------------")
103 
104  #Checking jobOptions
105  JOsList=getJOsList(JOsDict)
106  if not len(JOsList):
107  JOsErrors.append("including file \"MC15JobOptions/")
108  JOsErrors.append("including file \"MC15.")
109  else:
110 
111 
112  if not len(JOsDict["including file \"MC15JobOptions/"]):
113  JOsErrors.append("including file \"MC15JobOptions/")
114  if not len(JOsDict["including file \"MC15."]):
115  JOsErrors.append("including file \"MC15.")
116 
117  DSIDxxx=''
118  topJO=''
119  nTopJO=0
120  loginfo( '- jobOptions =',"")
121  for jo in JOsList:
122  pieces=jo.replace('.py','').split('.')
123  if len(pieces) and pieces[0]=='MC15':
124 
125  nTopJO=nTopJO+1
126  topJO=jo
127  DSID=pieces[1]
128  DSIDxxx="DSID"+DSID[:3]+"xxx"
129 
130  if nTopJO!=1:
131  logerr( "","ERROR: More than one \"top\" JO file found!")
132  raise RuntimeError("More than one \"top\" JO file found")
133  else:
134  #Check if JOs are in SVN!
135  if not opts.NO_SVN:
136  svnexcomm="svn ls --depth infinity svn+ssh://mcfayden@svn.cern.ch/reps/atlasoff/Generators/MC15JobOptions/trunk/"
137  retcode = subprocess.Popen(svnexcomm, shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE).communicate()
138  svnJOs=retcode[0].split()
139  if len(retcode[1]):
140  print(retcode[1])
141  logerr("","ERROR: Problem reading from SVN")
142  raise RuntimeError("Problem reading from SVN")
143 
144  for jo in JOsList:
145  if jo == topJO:
146  if "share/%s/%s"%(DSIDxxx,jo) in retcode[0]:
147  loggood("",jo)
148  else:
149  logerr("",jo+" ERROR <-- jobOptions not found in SVN!")
150  else:
151  indices = [i for i,s in enumerate(svnJOs) if "/"+jo in s]
152  if len(indices)==1:
153  loggood("",jo)
154  else:
155  if not len(indices):
156  logerr("",jo+" - jobOptions not found in SVN!")
157  else:
158  logwarn("",jo+" - multiple instances of jobOptions not found in SVN!")
159  for ix in indices:
160  logwarn("",jo+" --> %s"%svnJOs[ix])
161 
162  else:
163  for jo in JOsList:
164  loginfo("",jo)
165 
166 
167  #Checking release
168  if not len(JOsDict['using release']):
169  JOsErrors.append(JOsDict['using release'])
170  else:
171  name='using release'
172  tmp=JOsDict[name][0].replace('using release','').strip().split()[0]
173  val=tmp.replace('[','').replace(']','')
174  #checkForBlacklist
175  if not opts.NO_SVN:
176  if checkBlackList(val.split('-')[0],val.split('-')[1],'MC15JobOptions',".",JOsList) :
177  logerr( '- '+name+' = ',"".join(val)+" <-- ERROR: Cache is blacklisted for this generator")
178  else:
179  loggood( '- '+name+' = ',"".join(val))
180  else:
181  loginfo( '- '+name+' = ',"".join(val))
182 
183 
184 
185  if len(JOsErrors):
186  print("---------------------")
187  print("MISSING JOs:")
188  for i in JOsErrors:
189  if i == "including file \"MC15.":
190  #do nothing
191  logwarn("","INFO: local version of JOs used?")
192  else:
193  logerr("","ERROR: %s is missing!"%i)
194 
195 
196 
197  generateErrors=[]
198  print("")
199  print("---------------------")
200  print("Generate params:")
201  print("---------------------")
202  for key in generateDict:
203  name=key
204  val=generateDict[key]
205 
206  if not len(val):
207  generate.append(name)
208  else:
209  if key == 'minevents':
210  tmp=str(val[2]).split('#')[0].strip()
211  generateDict['minevents']=tmp
212  val=tmp
213 
214  loginfo( '- '+name+' = ',"".join(val))
215 
216  if len(generateErrors):
217  print("---------------------")
218  print("MISSING Generate params:")
219  for i in generateErrors:
220  logerr("","ERROR: %s is missing!"%i)
221 
222 
223 
224  metaDataErrors=[]
225  print("")
226  print("---------------------")
227  print("Metadata:")
228  print("---------------------")
229  for key in metaDataDict:
230  name=key
231  val=metaDataDict[key]
232  if not len(val):
233  metaDataErrors.append(name)
234  else:
235  if name=="contactPhysicist":
236  if not '@' in val:
237  logerr( '- '+name+' = ',"".join(val)+" <-- ERROR: No email found")
238  continue
239 
240  loginfo( '- '+name+' = ',"".join(val))
241 
242  if len(metaDataErrors):
243  print("---------------------")
244  print("MISSING Metadata:")
245  for i in metaDataErrors:
246  if i=="weights":
247  loginfo("INFO:","%s is missing"%i)
248  else:
249  logerr("","ERROR: %s is missing!"%i)
250 
251 
252 
253  perfMonErrors=[]
254  print("")
255  print("---------------------")
256  print("Performance metrics:")
257  print("---------------------")
258  for key in perfMonDict:
259  name=key
260  val=perfMonDict[key]
261  if not len(val):
262  perfMonErrors.append(name)
263  else:
264 
265  if key == 'snapshot_post_fin':
266  name = 'CPU'
267  tmp = 0.
268  tmp=float(val[0].split()[3])
269  tmp=tmp/(1000.*60.*60.)
270  val = "%.2f hrs"%tmp
271  if tmp > 18.:
272  logerr( '- '+name+' = ',"".join(val))
273  else:
274  loggood( '- '+name+' = ',"".join(val))
275 
276  if key == 'last -evt vmem':
277  name = 'Virtual memory'
278  tmp=float(val[0].split()[0])
279  if tmp > 2048.:
280  logerr( '- '+name+' = ',"".join(val))
281  else:
282  loggood( '- '+name+' = ',"".join(val))
283 
284 
285  if len(perfMonErrors):
286  print("---------------------")
287  print("MISSING Performance metric:")
288  for i in perfMonErrors:
289  print("ERROR: %s is missing!"%i)
290 
291 
292 
293 
294  testDict = {
295  'TestHepMC':testHepMCDict,
296  'EvgenFilterSeq':evgenFilterSeqDict,
297  'CountHepMC':countHepMCDict,
298  'SimTimeEstimate':simTimeEstimateDict
299  }
300 
301  testErrors=[]
302  print("")
303  print("---------------------")
304  print("Event tests:")
305  print("---------------------")
306  for dictkey in testDict:
307  for key in testDict[dictkey]:
308  name=key
309  val=testDict[dictkey][key]
310  if not len(val):
311  testErrors.append("%s %s"%(dictkey,name))
312  else:
313  #Check final Nevents processed
314  if dictkey=="CountHepMC":
315  allowed_minevents_lt1000 = [1, 2, 5, 10, 20, 25, 50, 100, 200, 500]
316  if int(val[0])%1000!=0 and not int(val[0]) in allowed_minevents_lt1000:
317  logerr( '- '+dictkey+" "+name+' = ',"".join(val)+" <-- ERROR: Not an acceptable number of events for production")
318  elif int(val[0]) != int(generateDict['minevents']):
319  logerr( '- '+dictkey+" "+name+' = ',"".join(val)+" <-- ERROR: This is not equal to minevents")
320  else:
321  loggood( '- '+dictkey+" "+name+' = ',"".join(val))
322  continue
323 
324  #Check filter efficiencies aren not too low
325  if dictkey=="EvgenFilterSeq":
326  if float(val[0].split()[0])<0.01:
327  logerr( '- '+dictkey+" "+name+' = ',"".join(val))
328  else:
329  loggood( '- '+dictkey+" "+name+' = ',"".join(val))
330  continue
331 
332  if dictkey=="TestHepMC" and name=="Efficiency":
333  if float(val[0].replace('%',''))<100.:
334  logerr( '- '+dictkey+" "+name+' = ',"".join(val))
335  else:
336  loggood( '- '+dictkey+" "+name+' = ',"".join(val))
337  continue
338 
339 
340  loginfo( '- '+dictkey+" "+name+' = ',"".join(val))
341 
342  if len(testErrors):
343  print("---------------------")
344  print("Failed tests:")
345  for i in testErrors:
346  if i =="SimTimeEstimate RUN INFORMATION":
347  logwarn("","WARNING: %s is missing!"%i)
348  else:
349  logerr("","ERROR: %s is missing!"%i)
350 
351 
352 
353  if opts.TOTAL_EVENTS:
354  print("")
355  print("")
356  print("---------------------")
357  print(" Others:")
358 
359  if opts.TOTAL_EVENTS:
360  xs_nb=float(metaDataDict['cross-section (nb)'][0])
361  eff_lumi_fb=float(opts.TOTAL_EVENTS)/(1.E+06*xs_nb)
362  if eff_lumi_fb > 1000.:
363  logwarn("- Effective lumi (fb-1):",str(eff_lumi_fb))
364  elif eff_lumi_fb < 40.:
365  logwarn("- Effective lumi (fb-1):",str(eff_lumi_fb))
366  else:
367  loggood("- Effective lumi (fb-1):",str(eff_lumi_fb))
368  minevents=int(generateDict['minevents'])
369  #int(countHepMCDict['Events passing all checks and written'][0])
370  loginfo("- Number of jobs:",int(opts.TOTAL_EVENTS)/minevents)
371 
372  if not opts.TOTAL_EVENTS or opts.NO_SVN:
373  print("")
374  print("")
375  print("---------------------")
376  print("Incomplete:")
377  if not opts.TOTAL_EVENTS:
378  logwarn("","WARNING: --Ntotal (-N) flag is not used - total number of events not given - impossible to calculated effective lumi.")
379  if opts.NO_SVN:
380  logwarn("","WARNING: --nosvn (-x) flag is used - could not check that SVN JOs are registered or whether release is blacklisted.")
381 
382  print("")
383  return
384 
385 
386 

Variable Documentation

◆ action

logParser.action

Definition at line 12 of file logParser.py.

◆ default

logParser.default

Definition at line 10 of file logParser.py.

◆ dest

logParser.dest

Definition at line 10 of file logParser.py.

◆ False

logParser.False

Definition at line 12 of file logParser.py.

◆ fileargs

logParser.fileargs

Definition at line 15 of file logParser.py.

◆ help

logParser.help

Definition at line 10 of file logParser.py.

◆ metavar

logParser.metavar

Definition at line 10 of file logParser.py.

◆ opts

logParser.opts

Definition at line 15 of file logParser.py.

◆ parser

logParser.parser = optparse.OptionParser(usage=__doc__)

Definition at line 9 of file logParser.py.

replace
std::string replace(std::string s, const std::string &s2, const std::string &s3)
Definition: hcg.cxx:307
find
std::string find(const std::string &s)
return a remapped string
Definition: hcg.cxx:135
logParser.logwarn
def logwarn(out1, out2)
Definition: logParser.py:477
CaloCellPos2Ntuple.int
int
Definition: CaloCellPos2Ntuple.py:24
logParser.checkLine
def checkLine(line, lineIdentifier, dict, splitby)
Definition: logParser.py:481
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
logParser.getJOsList
def getJOsList(JOsDict)
Definition: logParser.py:387
logParser.main
def main()
Definition: logParser.py:17
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
logParser.loginfo
def loginfo(out1, out2)
Definition: logParser.py:471
Trk::open
@ open
Definition: BinningType.h:40
Muon::print
std::string print(const MuPatSegment &)
Definition: MuonTrackSteering.cxx:28
logParser.checkBlackList
def checkBlackList(branch, cache, MCJobOptions, outnamedir, JOsList)
Definition: logParser.py:398
str
Definition: BTagTrackIpAccessor.cxx:11
logParser.logerr
def logerr(out1, out2)
Definition: logParser.py:475
logParser.loggood
def loggood(out1, out2)
Definition: logParser.py:473
readCCLHist.float
float
Definition: readCCLHist.py:83
Trk::split
@ split
Definition: LayerMaterialProperties.h:38
Generated on Thu Jul 4 2024 22:18:57 for ATLAS Offline Software by   doxygen 1.8.18