ATLAS Offline Software
logParser.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 
3 # Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
4 
5 
6 import optparse, sys, math, subprocess
7 from collections import OrderedDict
8 
9 parser = optparse.OptionParser(usage=__doc__)
10 parser.add_option("-i", "--input", default="-", dest="INPUT_FILE", metavar="PATH", help="input logfile")
11 parser.add_option("-N", "--Ntotal", default=0, dest="TOTAL_EVENTS", metavar="", help="Total number of events")
12 parser.add_option("-s", "--nosvn", action="store_true", dest="NO_SVN", default=False, help="Turn off any checks that require SVN access")
13 parser.add_option("-c", "--nocolour", action="store_true", dest="NO_COLOUR", default=False, help="Turn off colour for copying to file")
14 
15 opts, fileargs = parser.parse_args()
16 
17 def main():
18 
19 
20  if opts.INPUT_FILE=="-":
21  parser.print_help()
22  return
23 
24  # define dictionaries with keys as variables to be searched for and values to store the results
25 
26  JOsDict={
27  'using release':[],
28  "including file \"MC15JobOptions/":[],
29  "including file \"MC15.":[]
30  }
31 
32  testHepMCDict={
33  'Events passed':[],
34  'Efficiency':[]
35  }
36 
37  countHepMCDict={
38  'Events passing all checks and written':[]
39  }
40 
41  evgenFilterSeqDict={
42  'Weighted Filter Efficiency':[],
43  'Filter Efficiency':[]
44  }
45 
46  simTimeEstimateDict={
47  'RUN INFORMATION':[]
48  }
49 
50  metaDataDict={
51  'physicsComment':[],
52  'generatorName':[],
53  'generatorTune':[],
54  'keywords':[],
55  'specialConfig':[],
56  'contactPhysicist':[],
57  'randomSeed':[],
58  'genFilterNames':[],
59  'cross-section (nb)':[],
60  'generator':[],
61  'weights':[],
62  'GenFiltEff':[]
63  }
64 
65  generateDict={
66  'minevents':[0]
67  }
68 
69  perfMonDict={
70  'snapshot_post_fin':[],
71  'last -evt vmem':[]
72  }
73 
74 
75  # open and read log file
76  file=open(opts.INPUT_FILE,"r")
77  lines=file.readlines()
78 
79  # check each line
80  for line in lines:
81 
82  checkLine(line,'Py:Athena',JOsDict,'INFO')
83 
84  checkLine(line,'MetaData',metaDataDict,'=')
85  checkLine(line,'Py:Generate',generateDict,'=')
86 
87  checkLine(line,'Py:PerfMonSvc',perfMonDict,':')
88  checkLine(line,'PMonSD',perfMonDict,'---')
89 
90  checkLine(line,'TestHepMC',testHepMCDict,'=')
91  checkLine(line,'Py:EvgenFilterSeq',evgenFilterSeqDict,'=')
92  checkLine(line,'CountHepMC',countHepMCDict,'=')
93  checkLine(line,'SimTimeEstimate',simTimeEstimateDict,'|')
94 
95 
96 
97  # print results
98  JOsErrors=[]
99  print("")
100  print("---------------------")
101  print("jobOptions and release:")
102  print("---------------------")
103 
104  #Checking jobOptions
105  JOsList=getJOsList(JOsDict)
106  if not len(JOsList):
107  JOsErrors.append("including file \"MC15JobOptions/")
108  JOsErrors.append("including file \"MC15.")
109  else:
110 
111 
112  if not len(JOsDict["including file \"MC15JobOptions/"]):
113  JOsErrors.append("including file \"MC15JobOptions/")
114  if not len(JOsDict["including file \"MC15."]):
115  JOsErrors.append("including file \"MC15.")
116 
117  DSIDxxx=''
118  topJO=''
119  nTopJO=0
120  loginfo( '- jobOptions =',"")
121  for jo in JOsList:
122  pieces=jo.replace('.py','').split('.')
123  if len(pieces) and pieces[0]=='MC15':
124 
125  nTopJO=nTopJO+1
126  topJO=jo
127  DSID=pieces[1]
128  DSIDxxx="DSID"+DSID[:3]+"xxx"
129 
130  if nTopJO!=1:
131  logerr( "","ERROR: More than one \"top\" JO file found!")
132  raise RuntimeError("More than one \"top\" JO file found")
133  else:
134  #Check if JOs are in SVN!
135  if not opts.NO_SVN:
136  svnexcomm="svn ls --depth infinity svn+ssh://mcfayden@svn.cern.ch/reps/atlasoff/Generators/MC15JobOptions/trunk/"
137  retcode = subprocess.Popen(svnexcomm, shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE).communicate()
138  svnJOs=retcode[0].split()
139  if len(retcode[1]):
140  print(retcode[1])
141  logerr("","ERROR: Problem reading from SVN")
142  raise RuntimeError("Problem reading from SVN")
143 
144  for jo in JOsList:
145  if jo == topJO:
146  if "share/%s/%s"%(DSIDxxx,jo) in retcode[0]:
147  loggood("",jo)
148  else:
149  logerr("",jo+" ERROR <-- jobOptions not found in SVN!")
150  else:
151  indices = [i for i,s in enumerate(svnJOs) if "/"+jo in s]
152  if len(indices)==1:
153  loggood("",jo)
154  else:
155  if not len(indices):
156  logerr("",jo+" - jobOptions not found in SVN!")
157  else:
158  logwarn("",jo+" - multiple instances of jobOptions not found in SVN!")
159  for ix in indices:
160  logwarn("",jo+" --> %s"%svnJOs[ix])
161 
162  else:
163  for jo in JOsList:
164  loginfo("",jo)
165 
166 
167  #Checking release
168  if not len(JOsDict['using release']):
169  JOsErrors.append(JOsDict['using release'])
170  else:
171  name='using release'
172  tmp=JOsDict[name][0].replace('using release','').strip().split()[0]
173  val=tmp.replace('[','').replace(']','')
174  #checkForBlacklist
175  if not opts.NO_SVN:
176  if checkBlackList(val.split('-')[0],val.split('-')[1],'MC15JobOptions',".",JOsList) :
177  logerr( '- '+name+' = ',"".join(val)+" <-- ERROR: Cache is blacklisted for this generator")
178  else:
179  loggood( '- '+name+' = ',"".join(val))
180  else:
181  loginfo( '- '+name+' = ',"".join(val))
182 
183 
184 
185  if len(JOsErrors):
186  print("---------------------")
187  print("MISSING JOs:")
188  for i in JOsErrors:
189  if i == "including file \"MC15.":
190  #do nothing
191  logwarn("","INFO: local version of JOs used?")
192  else:
193  logerr("","ERROR: %s is missing!"%i)
194 
195 
196 
197  generateErrors=[]
198  print("")
199  print("---------------------")
200  print("Generate params:")
201  print("---------------------")
202  for key in generateDict:
203  name=key
204  val=generateDict[key]
205 
206  if not len(val):
207  generate.append(name)
208  else:
209  if key == 'minevents':
210  tmp=str(val[2]).split('#')[0].strip()
211  generateDict['minevents']=tmp
212  val=tmp
213 
214  loginfo( '- '+name+' = ',"".join(val))
215 
216  if len(generateErrors):
217  print("---------------------")
218  print("MISSING Generate params:")
219  for i in generateErrors:
220  logerr("","ERROR: %s is missing!"%i)
221 
222 
223 
224  metaDataErrors=[]
225  print("")
226  print("---------------------")
227  print("Metadata:")
228  print("---------------------")
229  for key in metaDataDict:
230  name=key
231  val=metaDataDict[key]
232  if not len(val):
233  metaDataErrors.append(name)
234  else:
235  if name=="contactPhysicist":
236  if not '@' in val:
237  logerr( '- '+name+' = ',"".join(val)+" <-- ERROR: No email found")
238  continue
239 
240  loginfo( '- '+name+' = ',"".join(val))
241 
242  if len(metaDataErrors):
243  print("---------------------")
244  print("MISSING Metadata:")
245  for i in metaDataErrors:
246  if i=="weights":
247  loginfo("INFO:","%s is missing"%i)
248  else:
249  logerr("","ERROR: %s is missing!"%i)
250 
251 
252 
253  perfMonErrors=[]
254  print("")
255  print("---------------------")
256  print("Performance metrics:")
257  print("---------------------")
258  for key in perfMonDict:
259  name=key
260  val=perfMonDict[key]
261  if not len(val):
262  perfMonErrors.append(name)
263  else:
264 
265  if key == 'snapshot_post_fin':
266  name = 'CPU'
267  tmp = 0.
268  tmp=float(val[0].split()[3])
269  tmp=tmp/(1000.*60.*60.)
270  val = "%.2f hrs"%tmp
271  if tmp > 18.:
272  logerr( '- '+name+' = ',"".join(val))
273  else:
274  loggood( '- '+name+' = ',"".join(val))
275 
276  if key == 'last -evt vmem':
277  name = 'Virtual memory'
278  tmp=float(val[0].split()[0])
279  if tmp > 2048.:
280  logerr( '- '+name+' = ',"".join(val))
281  else:
282  loggood( '- '+name+' = ',"".join(val))
283 
284 
285  if len(perfMonErrors):
286  print("---------------------")
287  print("MISSING Performance metric:")
288  for i in perfMonErrors:
289  print("ERROR: %s is missing!"%i)
290 
291 
292 
293 
294  testDict = {
295  'TestHepMC':testHepMCDict,
296  'EvgenFilterSeq':evgenFilterSeqDict,
297  'CountHepMC':countHepMCDict,
298  'SimTimeEstimate':simTimeEstimateDict
299  }
300 
301  testErrors=[]
302  print("")
303  print("---------------------")
304  print("Event tests:")
305  print("---------------------")
306  for dictkey in testDict:
307  for key in testDict[dictkey]:
308  name=key
309  val=testDict[dictkey][key]
310  if not len(val):
311  testErrors.append("%s %s"%(dictkey,name))
312  else:
313  #Check final Nevents processed
314  if dictkey=="CountHepMC":
315  allowed_minevents_lt1000 = [1, 2, 5, 10, 20, 25, 50, 100, 200, 500]
316  if int(val[0])%1000!=0 and not int(val[0]) in allowed_minevents_lt1000:
317  logerr( '- '+dictkey+" "+name+' = ',"".join(val)+" <-- ERROR: Not an acceptable number of events for production")
318  elif int(val[0]) != int(generateDict['minevents']):
319  logerr( '- '+dictkey+" "+name+' = ',"".join(val)+" <-- ERROR: This is not equal to minevents")
320  else:
321  loggood( '- '+dictkey+" "+name+' = ',"".join(val))
322  continue
323 
324  #Check filter efficiencies aren not too low
325  if dictkey=="EvgenFilterSeq":
326  if float(val[0].split()[0])<0.01:
327  logerr( '- '+dictkey+" "+name+' = ',"".join(val))
328  else:
329  loggood( '- '+dictkey+" "+name+' = ',"".join(val))
330  continue
331 
332  if dictkey=="TestHepMC" and name=="Efficiency":
333  if float(val[0].replace('%',''))<100.:
334  logerr( '- '+dictkey+" "+name+' = ',"".join(val))
335  else:
336  loggood( '- '+dictkey+" "+name+' = ',"".join(val))
337  continue
338 
339 
340  loginfo( '- '+dictkey+" "+name+' = ',"".join(val))
341 
342  if len(testErrors):
343  print("---------------------")
344  print("Failed tests:")
345  for i in testErrors:
346  if i =="SimTimeEstimate RUN INFORMATION":
347  logwarn("","WARNING: %s is missing!"%i)
348  else:
349  logerr("","ERROR: %s is missing!"%i)
350 
351 
352 
353  if opts.TOTAL_EVENTS:
354  print("")
355  print("")
356  print("---------------------")
357  print(" Others:")
358 
359  if opts.TOTAL_EVENTS:
360  xs_nb=float(metaDataDict['cross-section (nb)'][0])
361  eff_lumi_fb=float(opts.TOTAL_EVENTS)/(1.E+06*xs_nb)
362  if eff_lumi_fb > 1000.:
363  logwarn("- Effective lumi (fb-1):",str(eff_lumi_fb))
364  elif eff_lumi_fb < 40.:
365  logwarn("- Effective lumi (fb-1):",str(eff_lumi_fb))
366  else:
367  loggood("- Effective lumi (fb-1):",str(eff_lumi_fb))
368  minevents=int(generateDict['minevents'])
369  #int(countHepMCDict['Events passing all checks and written'][0])
370  loginfo("- Number of jobs:",int(opts.TOTAL_EVENTS)/minevents)
371 
372  if not opts.TOTAL_EVENTS or opts.NO_SVN:
373  print("")
374  print("")
375  print("---------------------")
376  print("Incomplete:")
377  if not opts.TOTAL_EVENTS:
378  logwarn("","WARNING: --Ntotal (-N) flag is not used - total number of events not given - impossible to calculated effective lumi.")
379  if opts.NO_SVN:
380  logwarn("","WARNING: --nosvn (-x) flag is used - could not check that SVN JOs are registered or whether release is blacklisted.")
381 
382  print("")
383  return
384 
385 
386 
387 def getJOsList(JOsDict):
388  liststr=''
389  if len(JOsDict["including file \"MC15JobOptions/"]):
390  liststr+="|".join(JOsDict["including file \"MC15JobOptions/"]).replace("nonStandard/","")
391  if len(JOsDict["including file \"MC15."]):
392  liststr+="|".join(JOsDict["including file \"MC15."].replace("nonStandard/",""))
393  liststr=liststr.replace('MC15JobOptions/','').replace('"','').replace('including file','').replace(' ','')
394  tmplist=liststr.split('|')
395  return tmplist
396 
397 
398 def checkBlackList(branch,cache,MCJobOptions,outnamedir,JOsList) :
399 
400  import getpass
401  user = getpass.getuser()
402 
403  aJOs=[]
404  for l in JOsList:
405  if "MC15." in l:
406  aJOs.append(l)
407  print("JOSH",aJOs)
408 
409  blacklist = 'svn export svn+ssh://svn.cern.ch/reps/atlasoff/Generators/'+MCJobOptions+'/trunk/common/BlackList_caches.txt'
410  tmpblackfile = "%s/BlackList_caches.txt" % (outnamedir)
411 
412  isError = False
413  if 'MC15' in MCJobOptions :
414  svnexcomm='%s %s' % (blacklist,tmpblackfile)
415  retcode = subprocess.Popen(svnexcomm, shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE).communicate()
416  if retcode[1].find("exist") != -1 or retcode[1].find("cannot") != -1 :
417  logerr("","export failed= BlackList_caches.txt" )
418  isError = True
419 
420  bfile = open(tmpblackfile)
421  for line in bfile:
422  if not line.strip():
423  continue
424 
425  bad = "".join(line.split()).split(",")
426 
427  #Match Generator and release type e.g. AtlasProduction, MCProd
428  if any( bad[2] in s for s in aJOs ) and branch in bad[0]:
429  #Match cache
430  cacheMatch=True
431  for i,c in enumerate(cache.split('.')):
432  if not c == bad[1].split('.')[i]:
433  cacheMatch=False
434  break
435 
436  if cacheMatch:
437  #logerr("", "Combination %s_%s for %s it is NOT allowed"%(bad[0],bad[1],bad[2]))
438  isError = True
439  return isError
440 
441 
442  return isError
443  else:
444  return False
445 
446 
447 class bcolors:
448  if not opts.NO_COLOUR:
449  HEADER = '\033[95m'
450  OKBLUE = '\033[94m'
451  OKGREEN = '\033[92m'
452  WARNING = '\033[93m'
453  FAIL = '\033[91m'
454  ENDC = '\033[0m'
455  else:
456  HEADER = ''
457  OKBLUE = ''
458  OKGREEN = ''
459  WARNING = ''
460  FAIL = ''
461  ENDC = ''
462 
463  def disable(self):
464  self.HEADER = ''
465  self.OKBLUE = ''
466  self.OKGREEN = ''
467  self.WARNING = ''
468  self.FAIL = ''
469  self.ENDC = ''
470 
471 def loginfo(out1,out2):
472  print(str(out1),bcolors.OKBLUE + str(out2) + bcolors.ENDC)
473 def loggood(out1,out2):
474  print(str(out1),bcolors.OKGREEN + str(out2) + bcolors.ENDC)
475 def logerr(out1,out2):
476  print(str(out1),bcolors.FAIL + str(out2) + bcolors.ENDC)
477 def logwarn(out1,out2):
478  print(str(out1),bcolors.WARNING + str(out2) + bcolors.ENDC)
479 
480 # find identifiers and variables in log file lines
481 def checkLine(line, lineIdentifier, dict, splitby ):
482  if lineIdentifier in line:
483  for param in dict:
484  if param in line:
485  if len(line.split(splitby))==0:
486  raise RuntimeError("Found bad entry %s"%line)
487  else:
488  thing="".join(line.split(lineIdentifier)[1].split(splitby)[1:]).strip()
489  dict[param].append(thing)
490  break
491 
492 
493 if __name__ == "__main__":
494  main()
replace
std::string replace(std::string s, const std::string &s2, const std::string &s3)
Definition: hcg.cxx:307
find
std::string find(const std::string &s)
return a remapped string
Definition: hcg.cxx:135
logParser.logwarn
def logwarn(out1, out2)
Definition: logParser.py:477
CaloCellPos2Ntuple.int
int
Definition: CaloCellPos2Ntuple.py:24
logParser.checkLine
def checkLine(line, lineIdentifier, dict, splitby)
Definition: logParser.py:481
logParser.bcolors.HEADER
string HEADER
Definition: logParser.py:449
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
logParser.getJOsList
def getJOsList(JOsDict)
Definition: logParser.py:387
logParser.bcolors.OKGREEN
string OKGREEN
Definition: logParser.py:451
logParser.bcolors.disable
def disable(self)
Definition: logParser.py:463
logParser.main
def main()
Definition: logParser.py:17
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
logParser.bcolors.WARNING
string WARNING
Definition: logParser.py:452
logParser.loginfo
def loginfo(out1, out2)
Definition: logParser.py:471
logParser.bcolors
Definition: logParser.py:447
Trk::open
@ open
Definition: BinningType.h:40
logParser.checkBlackList
def checkBlackList(branch, cache, MCJobOptions, outnamedir, JOsList)
Definition: logParser.py:398
str
Definition: BTagTrackIpAccessor.cxx:11
dbg::print
void print(std::FILE *stream, std::format_string< Args... > fmt, Args &&... args)
Definition: SGImplSvc.cxx:70
logParser.bcolors.OKBLUE
string OKBLUE
Definition: logParser.py:450
logParser.logerr
def logerr(out1, out2)
Definition: logParser.py:475
logParser.bcolors.ENDC
string ENDC
Definition: logParser.py:454
logParser.loggood
def loggood(out1, out2)
Definition: logParser.py:473
readCCLHist.float
float
Definition: readCCLHist.py:83
Trk::split
@ split
Definition: LayerMaterialProperties.h:38
logParser.bcolors.FAIL
string FAIL
Definition: logParser.py:453
Generated on Thu Nov 7 2024 21:20:14 for ATLAS Offline Software by   doxygen 1.8.18