ATLAS Offline Software
Loading...
Searching...
No Matches
python.trfValidateRootFile Namespace Reference

Functions

 checkBranch (branch)
 checkTreeBasketWise (tree)
 checkTreeEventWise (tree, printInterval=150000)
 checkNTupleEventWise (ntuple, printInterval=150000)
 checkNTupleFieldWiseCluster (clusterDescriptor, subFields, model)
 checkNTupleFieldWise (ntuple)
 checkDirectory (directory, the_type, requireTree, depth)
 checkFile (fileName, the_type, requireTree)
 checkNEvents (fileName, nEntries)
 usage ()
 main (argv)

Variables

 ROOT = RootUtils.import_root()
 msg = logging.getLogger(__name__)
 ch = logging.StreamHandler(sys.stdout)
 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 rc = main(sys.argv)

Function Documentation

◆ checkBranch()

checkBranch ( branch)

Definition at line 22 of file trfValidateRootFile.py.

22def checkBranch(branch):
23
24 msg.debug('Checking branch %s ...', branch.GetName())
25
26 nBaskets=branch.GetWriteBasket()
27
28 msg.debug('Checking %s baskets ...', nBaskets)
29
30 for iBasket in range(nBaskets):
31 basket=branch.GetBasket(iBasket)
32 if not basket:
33 msg.warning('Basket %s of branch %s is corrupted.', iBasket, branch.GetName() )
34 return 1
35
36 listOfSubBranches=branch.GetListOfBranches()
37 msg.debug('Checking %s subbranches ...', listOfSubBranches.GetEntries())
38 for subBranch in listOfSubBranches:
39 if checkBranch(subBranch)==1:
40 return 1
41
42 msg.debug('Branch %s looks ok.', branch.GetName())
43 return 0
44
45

◆ checkDirectory()

checkDirectory ( directory,
the_type,
requireTree,
depth )

Definition at line 151 of file trfValidateRootFile.py.

151def checkDirectory(directory, the_type, requireTree, depth):
152
153 from PyUtils import PoolFile
154 nentries = None
155 hasMetadata = False
156
157 msg.debug('Checking directory %s ...', directory.GetName())
158
159 listOfKeys=directory.GetListOfKeys()
160
161 msg.debug('Checking %s keys ... ', listOfKeys.GetEntries())
162
163 for key in listOfKeys:
164
165 msg.debug('Looking at key %s ...', key.GetName())
166 msg.debug('Key is of class %s.', key.GetClassName())
167
168 the_object=directory.Get(key.GetName())
169 if not the_object:
170 msg.warning("Can't get object of key %s.", key.GetName())
171 return 1
172
173 if requireTree and not isinstance(the_object, TTree):
174 msg.warning("Object of key %s is not of class TTree!", key.GetName())
175 return 1
176
177 if isinstance(the_object,TTree):
178
179 msg.debug('Checking tree %s ...', the_object.GetName())
180
181 if depth == 0:
182 if PoolFile.PoolOpts.TTreeNames.EventData == the_object.GetName():
183 nentries = the_object.GetEntries()
184 msg.debug(f' contains {nentries} events')
185 elif PoolFile.PoolOpts.TTreeNames.MetaData == the_object.GetName():
186 hasMetadata = True
187 msg.debug(' contains MetaData')
188
189 if the_type=='event':
190 if checkTreeEventWise(the_object)==1:
191 return 1
192 elif the_type=='basket':
193 if checkTreeBasketWise(the_object)==1:
194 return 1
195
196 msg.debug('Tree %s looks ok.', the_object.GetName())
197
198 if isRNTuple(the_object):
199
200 msg.debug('Checking ntuple of key %s ...', key.GetName())
201
202 try:
203 reader=RNTupleReader.Open(the_object)
204 except Exception as err:
205 msg.warning('Could not open ntuple %s: %s', the_object, err)
206 return 1
207
208 if depth == 0:
209 if PoolFile.PoolOpts.RNTupleNames.EventData == reader.GetDescriptor().GetName():
210 nentries = reader.GetNEntries()
211 msg.debug(f' contains {nentries} events')
212 elif PoolFile.PoolOpts.RNTupleNames.MetaData == reader.GetDescriptor().GetName():
213 hasMetadata = True
214 msg.debug(' contains MetaData')
215
216 if the_type=='event':
217 if checkNTupleEventWise(the_object)==1:
218 return 1
219 elif the_type=='basket':
220 if checkNTupleFieldWise(the_object)==1:
221 return 1
222
223 msg.debug('NTuple of key %s looks ok.', key.GetName())
224
225 if isinstance(the_object, TDirectory):
226 if checkDirectory(the_object, the_type, requireTree, depth + 1)==1:
227 return 1
228
229 # Only check if metadata object is available as in standard POOL files
230 if depth == 0 and hasMetadata and checkNEvents(directory.GetName(), nentries)==1:
231 return 1
232 else:
233 msg.debug('Directory %s looks ok.', directory.GetName())
234 return 0
235
236

◆ checkFile()

checkFile ( fileName,
the_type,
requireTree )

Definition at line 237 of file trfValidateRootFile.py.

237def checkFile(fileName, the_type, requireTree):
238
239 msg.info('Checking file %s ...', fileName)
240
241 enabledIMT = False
242 if not ROOT.ROOT.IsImplicitMTEnabled() and os.environ.keys() >= {'TRF_MULTITHREADED_VALIDATION', 'ATHENA_CORE_NUMBER'}:
243 if (nThreads := int(os.environ['ATHENA_CORE_NUMBER'])) >= 0:
244 msg.info(f"Setting the number of implicit ROOT threads to {nThreads}")
245 ROOT.ROOT.EnableImplicitMT(nThreads)
246 enabledIMT = True
247 else:
248 msg.warning(f"Ignored negative ATHENA_CORE_NUMBER ({nThreads})")
249
250 try:
251 file_handle=TFile.Open(fileName)
252 except OSError as err:
253 msg.error('Could not open file %s: %s', fileName, err)
254 return 1
255
256 if not file_handle:
257 msg.warning("Can't access file %s.", fileName)
258 return 1
259
260 if not file_handle.IsOpen():
261 msg.warning("Can't open file %s.", fileName)
262 return 1
263
264 if file_handle.IsZombie():
265 msg.warning("File %s is a zombie.", fileName)
266 file_handle.Close()
267 return 1
268
269 if file_handle.TestBit(TFile.kRecovered):
270 msg.warning("File %s needed to be recovered.", fileName)
271 file_handle.Close()
272 return 1
273
274 if checkDirectory(file_handle, the_type, requireTree, 0)==1:
275 msg.warning("File %s is corrupted.", fileName)
276 file_handle.Close()
277 return 1
278
279 file_handle.Close()
280 msg.info("File %s looks ok.", fileName)
281
282 if enabledIMT:
283 ROOT.ROOT.DisableImplicitMT()
284
285 return 0
286
287

◆ checkNEvents()

checkNEvents ( fileName,
nEntries )
Check consistency of number of events in file with metadata.

fileName   name of file to check consistency of
nEntries   number of events in fileName (e.g., obtained by examining event data object)
return     0 in case of consistency, 1 otherwise

Definition at line 288 of file trfValidateRootFile.py.

288def checkNEvents(fileName, nEntries):
289 """Check consistency of number of events in file with metadata.
290
291 fileName name of file to check consistency of
292 nEntries number of events in fileName (e.g., obtained by examining event data object)
293 return 0 in case of consistency, 1 otherwise
294 """
295 from PyUtils.MetaReader import read_metadata
296
297 msg.debug('Checking number of events in file %s ...', fileName)
298
299 meta = read_metadata(fileName, mode='lite')[fileName]
300 msg.debug(' according to metadata: {0}'.format(meta["nentries"]))
301 msg.debug(' according to event data: {0}'.format(nEntries))
302 if meta["nentries"] and nEntries and meta["nentries"] != nEntries \
303 or meta["nentries"] and not nEntries \
304 or not meta["nentries"] and nEntries:
305 msg.warning(f' number of events ({nEntries}) inconsistent with metadata ({meta["nentries"]}) in file {fileName!r}.')
306 return 1
307 else:
308 msg.debug(" looks ok.")
309 return 0
310

◆ checkNTupleEventWise()

checkNTupleEventWise ( ntuple,
printInterval = 150000 )

Definition at line 77 of file trfValidateRootFile.py.

77def checkNTupleEventWise(ntuple, printInterval = 150000):
78
79 try:
80 reader=RNTupleReader.Open(ntuple)
81 except Exception as err:
82 msg.warning('Could not open ntuple %s: %s', ntuple, err)
83 return 1
84
85 nEntries=reader.GetNEntries()
86
87 msg.debug('Checking %s entries ...', nEntries)
88
89 entry = reader.CreateEntry()
90 for i in range(nEntries):
91 try:
92 reader.LoadEntry(i, entry)
93 except Exception as err:
94 msg.warning('Event %s of ntuple %s is corrupted: %s', i, reader.GetDescriptor().GetName(), err)
95 return 1
96
97 # Show a sign of life for long validation jobs: ATLASJT-433
98 if (i%printInterval)==0 and i>0:
99 msg.info('Validated %s events so far ...', i)
100
101 return 0
102

◆ checkNTupleFieldWise()

checkNTupleFieldWise ( ntuple)
For each cluster, bulk read each top level field.

Definition at line 120 of file trfValidateRootFile.py.

120def checkNTupleFieldWise(ntuple):
121 """For each cluster, bulk read each top level field.
122 """
123 from ROOT import RException
124
125 try:
126 reader=RNTupleReader.Open(ntuple)
127 except Exception as err:
128 msg.warning('Could not open ntuple %r: %r', ntuple, err)
129 return 1
130
131 try:
132 descriptor = reader.GetDescriptor()
133 msg.debug(f"ntupleName={descriptor.GetName()}")
134
135 model = reader.GetModel()
136 fieldZero = model.GetConstFieldZero()
137 subFields = fieldZero.GetConstSubfields()
138 msg.debug(f"Top level fields number {subFields.size()}")
139 for cg in descriptor.GetClusterGroupIterable():
140 for cid in cg.GetClusterIds():
141 clusterDescriptor = descriptor.GetClusterDescriptor(cid)
142 checkNTupleFieldWiseCluster(clusterDescriptor, subFields, model)
143
144 except RException as err:
145 from traceback import format_exception
146 msg.error("Exception reading ntuple %r\n%s", ntuple, "".join(format_exception(err)))
147 return 1
148
149 return 0
150

◆ checkNTupleFieldWiseCluster()

checkNTupleFieldWiseCluster ( clusterDescriptor,
subFields,
model )

Definition at line 103 of file trfValidateRootFile.py.

103def checkNTupleFieldWiseCluster(clusterDescriptor, subFields, model):
104 size = int(clusterDescriptor.GetNEntries())
105 if msg.isEnabledFor(logging.DEBUG):
106 msg.debug(f" cluster #{clusterDescriptor.GetId()}"
107 f" firstEntryIndex={clusterDescriptor.GetFirstEntryIndex()}"
108 f" nEntries={size}")
109 clusterRange = ROOT.RNTupleLocalRange(clusterDescriptor.GetId(), 0, size)
110 for field in subFields:
111 if msg.isEnabledFor(logging.DEBUG):
112 msg.debug(f"fieldName={field.GetFieldName()} typeName={field.GetTypeName()}")
113 bulk = model.CreateBulk(field.GetFieldName())
114 values = bulk.ReadBulk(clusterRange)
115 if msg.isEnabledFor(logging.DEBUG):
116 msg.debug(f" values array at {values}")
117 # Be sure that this gets destroyed before field.
118 del bulk
119 return

◆ checkTreeBasketWise()

checkTreeBasketWise ( tree)

Definition at line 46 of file trfValidateRootFile.py.

46def checkTreeBasketWise(tree):
47
48 listOfBranches=tree.GetListOfBranches()
49
50 msg.debug('Checking %s branches ...', listOfBranches.GetEntries())
51
52 for branch in listOfBranches:
53 if checkBranch(branch)==1:
54 msg.warning('Tree %s is corrupted (branch %s ).', tree.GetName(), branch.GetName())
55 return 1
56
57 return 0
58
59

◆ checkTreeEventWise()

checkTreeEventWise ( tree,
printInterval = 150000 )

Definition at line 60 of file trfValidateRootFile.py.

60def checkTreeEventWise(tree, printInterval = 150000):
61
62 nEntries=tree.GetEntries()
63
64 msg.debug('Checking %s entries ...', nEntries)
65
66 for i in range(nEntries):
67 if tree.GetEntry(i)<0:
68 msg.warning('Event %s of tree %s is corrupted.', i, tree.GetName())
69 return 1
70
71 # Show a sign of life for long validation jobs: ATLASJT-433
72 if (i%printInterval)==0 and i>0:
73 msg.info('Validated %s events so far ...', i)
74
75 return 0
76

◆ main()

main ( argv)

Definition at line 320 of file trfValidateRootFile.py.

320def main(argv):
321
322 clock=TStopwatch()
323
324 argc=len(argv)
325
326 if (argc!=5):
327 return usage()
328
329 fileName=argv[1]
330 the_type=argv[2]
331 requireTree=argv[3]
332 verbosity=argv[4]
333
334
335 if the_type!="event" and the_type!="basket":
336 return usage()
337
338 if requireTree=="true":
339 requireTree=True
340 elif requireTree=="false":
341 requireTree=False
342 else:
343 return usage()
344
345 if verbosity=="on":
346 msg.setLevel(logging.DEBUG)
347 elif verbosity=="off":
348 msg.setLevel(logging.INFO)
349 else:
350 return usage()
351
352 rc=checkFile(fileName,the_type, requireTree)
353 msg.debug('Returning %s', rc)
354
355 clock.Stop()
356 clock.Print()
357
358 return rc
359
360
StatusCode usage()
int main()
Definition hello.cxx:18

◆ usage()

usage ( )

Definition at line 311 of file trfValidateRootFile.py.

311def usage():
312 print("Usage: validate filename type requireTree verbosity")
313 print("'type' must be either 'event' or 'basket'")
314 print("'requireTree' must be either 'true' or 'false'")
315 print("'verbosity' must be either 'on' or 'off'")
316
317 return 2
318
319
void print(char *figname, TCanvas *c1)

Variable Documentation

◆ ch

python.trfValidateRootFile.ch = logging.StreamHandler(sys.stdout)

Definition at line 363 of file trfValidateRootFile.py.

◆ formatter

python.trfValidateRootFile.formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

Definition at line 364 of file trfValidateRootFile.py.

◆ msg

python.trfValidateRootFile.msg = logging.getLogger(__name__)

Definition at line 20 of file trfValidateRootFile.py.

◆ rc

python.trfValidateRootFile.rc = main(sys.argv)

Definition at line 368 of file trfValidateRootFile.py.

◆ ROOT

python.trfValidateRootFile.ROOT = RootUtils.import_root()

Definition at line 15 of file trfValidateRootFile.py.