ATLAS Offline Software
Loading...
Searching...
No Matches
getProblemFolderFromLogs.py
Go to the documentation of this file.
1# Copyright (C) 2002-2026 CERN for the benefit of the ATLAS collaboration
2
3
14
15import sys
16import os.path
17import argparse
18
19from os import path
20
21
22if __name__ == "__main__":
23 # Parse arguments
24 parser = argparse.ArgumentParser(description='Coolr browser.', add_help=False)
25 parser.add_argument('--input', required=True,help='Input file in text folrat. It is output of athena job. It require has lines with info about load data: INFO Retrieved object: folder ')
26 parser.add_argument('--gtag', required=True,
27 help='Global tag for CREST')
28 parser.add_argument('--gcooltag', required=True,
29 help='Global tag for COOL')
30 parser.add_argument('--host', default='https://crest.cern.ch',
31 help='Host of the CREST service (default: https://crest.cern.ch)')
32 parser.add_argument('--port', default='443',
33 help='Port of the CREST service (default: 443)')
34 parser.add_argument('--out', default='log.out',
35 help='Out file for logs (default: log.out)')
36 parser.add_argument('--path', default='cool_crest_compare',
37 help='Path to compare (default: cool_crest_compare)')
38 parser.add_argument('--debug', default=False, action=argparse.BooleanOptionalAction, help='Set debug output')
39
40 args = parser.parse_args()
41 server = "{0}:{1}".format(args.host, args.port)
42 fOut="log.out"
43 appPath=args.path
44 if args.out:
45 fOut=args.out
46 if not path.exists(args.input) or not path.isfile(args.input):
47 print("Error: input file is not exists")
48 sys.exit(2)
49 data = {}
50 st=""
51 noUsedData=[]
52 # Parce athena log and found all folders and other parameters for compare data
53 with open(args.input) as inp_file:
54 for line in inp_file:
55 if line.rstrip().find('/TagInfo<metaOnly/>')>-1:
56 lst=line.rstrip().split(',')
57 for e in lst:
58 if e.find('<db>')<0:
59 continue
60 el ={}
61 if e.find('<tag>')>0:
62 sTag=e[e.find('<tag>')+5:]
63 if sTag.find('</tag>')>0:
64 sTag=sTag[:sTag.find('</tag>')]
65 el['tag']=sTag
66 db=e[e.find('<db>')+4:]
67 db=db[:db.find('</db>')]
68 if e.find('<db>')<4:
69 e=e[e.find('</db>')+5:]
70 folder = e[e.find('/'):]
71 if folder.find('<')>-1:
72 folder = folder[:folder.find('<')]
73 elif folder.find(' ')>-1:
74 folder = folder[:folder.find(' ')]
75 else:
76 folder = folder[:folder.find('\'')]
77 el['db']=db.strip()
78 data[folder.strip()]=el
79 if line.rstrip().find('Using tag ')>0:
80 st = line.rstrip()[line.rstrip().find('Using tag ')+10:]
81 sTag = st[:st.find(' ')]
82 if st.rstrip().find('for folder ')>0:
83 st = st.rstrip()[st.rstrip().find('for folder '):]
84 st = st[st.find('/'):]
85 sFolder = st.strip()
86 el = data.setdefault(sFolder.strip(), {})
87 el['tag']=sTag
88 data[sFolder.strip()]=el
89 if line.rstrip().find(' is requested but no data retrieved')>0:
90 st = line.rstrip()[:line.rstrip().find(' is requested but no data retrieved')]
91 st = st[st.find('WARNING Folder')+14:]
92 noUsedData.append(st.strip())
93 if line.rstrip().find('Retrieved object: folder ')>0:
94 st = line.rstrip()[line.rstrip().find('Retrieved object: folder '):]
95 st = st[st.find('/'):]
96 sFolder = st[:st.find(' ')]
97 st= st[st.find('IOV ')+4:]
98 st=st[:st.find(' ')]
99 el = data.setdefault(sFolder.strip(), {})
100 # Keep every key the folder was retrieved at, each one once
101 if st not in el.setdefault('timestamps', []):
102 el['timestamps'].append(st)
103 if args.debug:
104 print(data)
105 print("List of commands for compare:")
106 # The out file keeps the output of earlier runs, only the part written from here on is parsed
107 with open(fOut, 'a') as f:
108 logStart=f.tell()
109 noDbFolder=[]
110 # run compare
111 for elem in data.keys():
112 js=data[elem]
113 if 'timestamps' not in js:
114 continue
115 if 'db' not in js:
116 noDbFolder.append(elem)
117 continue
118 # cool_crest_compare takes all keys of a folder in one -t and prints one result line per key
119 command = appPath + ' -g '+ args.gtag+' -G '+args.gcooltag+ ' -f '+elem + ' -c "'+ js['db']+'"'
120 command +=' -t ' + ' '.join(js['timestamps'])+' -C '+server
121 if 'tag' in js:
122 if js['tag'].find('HEAD')>=0:
123 command +=' --head'
124 else:
125 command +=' -T '+js['tag']
126 js['command']=command
127 command +=' >>'+fOut+' 2>&1'
128 print(command)
129 with open(fOut, 'a') as f:
130 print('Compare folder name:', elem, file=f)
131 print(command,file=f)
132 os.system(command)
133 #parse output of compare folders (CREST and COOL)
134 print("Problem folders:")
135 curFolder=""
136 coolProblem={}
137 crestProblem={}
138 coolCrash={}
139 crestCrash={}
140 difFolder={}
141 noCrestData={}
142 correctFolder={}
143 correctData={}
144 wrongTag={}
145 allFolder=[]
146 #status: 0 - the same, 1 - the same data, 2 - different data, 3 - no CREST data, 4 - CREST problem, 5 - COOL problem, 6 - CREST crash, 7 - COOL crash
147 results={}
148 with open(fOut) as log_file:
149 log_file.seek(logStart)
150 for line in log_file:
151
152 if line.find('Compare folder name:')>-1:
153 curFolder=line.rstrip()[line.rstrip().find('Compare folder name:')+20:]
154 curFolder=curFolder.lstrip()
155 if curFolder not in allFolder:
156 allFolder.append(curFolder)
157 results.setdefault(curFolder, {})
158 elif curFolder == "":
159 continue
160 if line.find('Start CREST dump:')>-1:
161 results[curFolder]["-1"]=6
162 continue
163 elif line.find('Start COOL dump:')>-1:
164 results[curFolder]["-1"]=7
165 continue
166 elif line.find('ERROR in Crest. Global tag for folder')>-1:
167 results[curFolder]["-1"]=8
168 continue
169 if line.find('timestamp:')>-1:
170 ts = line[line.find('timestamp:')+10:].strip()
171 else:
172 continue
173 if line.rstrip().find('is the same in COOL and CREST')>-1:
174 results[curFolder][ts]=0
175 elif line.rstrip().find('has the same only data in COOL and CREST')>-1:
176 results[curFolder][ts]=1
177 elif line.rstrip().find('is different in COOL and CREST')>-1:
178 results[curFolder][ts]=2
179 elif line.find('ERROR in Crest. CREST output has no data.')>-1:
180 results[curFolder][ts]=3
181 elif line.find('CREST output file problem')>-1:
182 results[curFolder][ts]=4
183 elif line.find('COOL output file problem')>-1:
184 results[curFolder][ts]=5
185 for curFolder in allFolder:
186 js=data[curFolder]
187 if curFolder not in results:
188 continue
189 res=results[curFolder]
190 if len(res)>1:
191 res.pop("-1", None)
192 isPrint=False
193 for key,value in res.items():
194 if value==0:
195 el = correctFolder.setdefault(curFolder, [])
196 if key not in el:
197 el.append(key)
198 elif value==1:
199 el = correctData.setdefault(curFolder, [])
200 if key not in el:
201 el.append(key)
202 elif value==2:
203 isPrint=True
204 el = difFolder.setdefault(curFolder, [])
205 if key not in el:
206 el.append(key)
207 elif value==3:
208 isPrint=True
209 el = noCrestData.setdefault(curFolder, [])
210 if key not in el:
211 el.append(key)
212 elif value==4:
213 isPrint=True
214 el = crestProblem.setdefault(curFolder, [])
215 if key not in el:
216 el.append(key)
217 elif value==5:
218 isPrint=True
219 el = coolProblem.setdefault(curFolder, [])
220 if key not in el:
221 el.append(key)
222 elif value==6:
223 isPrint=True
224 el = crestCrash.setdefault(curFolder, [])
225 if key not in el:
226 el.append(key)
227 elif value==7:
228 isPrint=True
229 el = coolCrash.setdefault(curFolder, [])
230 if key not in el:
231 el.append(key)
232 elif value==8:
233 isPrint=True
234 el = wrongTag.setdefault(curFolder, [])
235 if key not in el:
236 el.append(key)
237 if isPrint:
238 print(js['command'])
239 unCompared=data.keys()-allFolder
240 compList=set(unCompared)-set(noUsedData)
241 print("Total number of folders in logs:",len(data),"; Total number of folders which compare (used):",len(allFolder), "; Total number of unread folders:", len(noUsedData),"; Correct folders:",len(correctFolder),"; Correct data:",len(correctData),"; Different folders:",len(difFolder),"; No CREST data:",len(noCrestData),"; COOL problems:",len(coolProblem),"; CREST problems:",len(crestProblem),"; COOL crash:",len(coolCrash),"; CREST crash:",len(crestCrash),"; Wrong CREST tag:",len(wrongTag))
242 if len(correctData)>0:
243 print("Only data:",correctData)
244 if len(difFolder)>0:
245 print("Different data:",difFolder)
246 if len(noCrestData)>0:
247 print("No data in CREST:",noCrestData)
248 if len(coolProblem)>0:
249 print("COOL problem folders:",coolProblem)
250 if len(crestProblem)>0:
251 print("CREST problem folders:",crestProblem)
252 if len(coolCrash)>0:
253 print("COOL crash in folders:",coolCrash)
254 if len(crestCrash)>0:
255 print("CREST crash in folders:",crestCrash)
256 if len(wrongTag)>0:
257 print("CREST globaltagmap has wrong tag:",wrongTag)
258 if len(noDbFolder)>0:
259 print("Folders without <db> in the Folders property (not compared):",noDbFolder)
260 if len(compList)>0:
261 if len(unCompared):
262 print("Incomparable folders:",unCompared)
263 if len(noUsedData)>0:
264 print("Unread (Unused) folders:",noUsedData)
265 elif len(noUsedData)>0:
266 print("Unread (Unused) folders:",noUsedData)
267 if len(correctFolder)>0 and args.debug:
268 print("Correct folders:",correctFolder)
269 if len(allFolder)>0 and args.debug:
270 print("Full folder list:",allFolder)
271 if args.debug:
272 print("Total list folders in log:",list(data.keys()))
273 if len(difFolder)+len(coolProblem)+len(crestProblem)+len(crestCrash)+len(coolCrash)+len(noDbFolder)>0:
274 sys.exit(1)
275
276
void print(char *figname, TCanvas *c1)
STL class.
std::string find(const std::string &s)
return a remapped string
Definition hcg.cxx:140
std::vector< std::string > split(const std::string &s, const std::string &t=":")
Definition hcg.cxx:179