ATLAS Offline Software
trfValidateRootFile.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
4 
5 
9 
10 
11 import sys, os
12 import logging
13 
14 from PyUtils import RootUtils
15 ROOT = RootUtils.import_root()
16 from ROOT import TFile, TTree, TDirectory, TStopwatch
17 from ROOT.Experimental import RNTupleReader
18 from PyUtils.PoolFile import isRNTuple
19 
20 msg = logging.getLogger(__name__)
21 
22 def checkBranch(branch):
23 
24  msg.debug('Checking branch %s ...', branch.GetName())
25 
26  nBaskets=branch.GetWriteBasket()
27 
28  msg.debug('Checking %s baskets ...', nBaskets)
29 
30  for iBasket in range(nBaskets):
31  basket=branch.GetBasket(iBasket)
32  if not basket:
33  msg.warning('Basket %s of branch %s is corrupted.', iBasket, branch.GetName() )
34  return 1
35 
36  listOfSubBranches=branch.GetListOfBranches()
37  msg.debug('Checking %s subbranches ...', listOfSubBranches.GetEntries())
38  for subBranch in listOfSubBranches:
39  if checkBranch(subBranch)==1:
40  return 1
41 
42  msg.debug('Branch %s looks ok.', branch.GetName())
43  return 0
44 
45 
46 def checkTreeBasketWise(tree):
47 
48  listOfBranches=tree.GetListOfBranches()
49 
50  msg.debug('Checking %s branches ...', listOfBranches.GetEntries())
51 
52  for branch in listOfBranches:
53  if checkBranch(branch)==1:
54  msg.warning('Tree %s is corrupted (branch %s ).', tree.GetName(), branch.GetName())
55  return 1
56 
57  return 0
58 
59 
60 def checkTreeEventWise(tree, printInterval = 150000):
61 
62  nEntries=tree.GetEntries()
63 
64  msg.debug('Checking %s entries ...', nEntries)
65 
66  for i in range(nEntries):
67  if tree.GetEntry(i)<0:
68  msg.warning('Event %s of tree %s is corrupted.', i, tree.GetName())
69  return 1
70 
71  # Show a sign of life for long validation jobs: ATLASJT-433
72  if (i%printInterval)==0 and i>0:
73  msg.info('Validated %s events so far ...', i)
74 
75  return 0
76 
77 def checkNTupleEventWise(ntuple, printInterval = 150000):
78 
79  try:
80  reader=RNTupleReader.Open(ntuple)
81  except Exception as err:
82  msg.warning('Could not open ntuple %s: %s', ntuple, err)
83  return 1
84 
85  msg.debug('Checking %s entries ...', reader.GetNEntries())
86 
87  try:
88  entry = reader.CreateEntry()
89  except AttributeError:
90  entry = reader.GetModel().CreateEntry()
91  for i in reader:
92  try:
93  reader.LoadEntry(i, entry)
94  except Exception as err:
95  msg.warning('Event %s of ntuple %s is corrupted: %s', i, reader.GetDescriptor().GetName(), err)
96  return 1
97 
98  # Show a sign of life for long validation jobs: ATLASJT-433
99  if (i%printInterval)==0 and i>0:
100  msg.info('Validated %s events so far ...', i)
101 
102  return 0
103 
104 def checkNTupleFieldWise(ntuple):
105  """Bulk read each top level field cluster by cluster.
106  """
107  from array import array
108  try:
109  from ROOT import RException
110  except ImportError:
111  from ROOT.Experimental import RException
112 
113  try:
114  reader=RNTupleReader.Open(ntuple)
115  except Exception as err:
116  msg.warning('Could not open ntuple %r: %r', ntuple, err)
117  return 1
118 
119  try:
120  descriptor = reader.GetDescriptor()
121  msg.debug(f"ntupleName={descriptor.GetName()}")
122 
123  model = reader.GetModel()
124  try:
125  fieldZero = model.GetFieldZero()
126  except AttributeError:
127  # ROOT Version: 6.35.01
128  fieldZero = model.GetConstFieldZero()
129  subFields = fieldZero.GetSubFields()
130  msg.debug(f"Top level fields number {subFields.size()}")
131  for field in subFields:
132  msg.debug(f"fieldName={field.GetFieldName()} typeName={field.GetTypeName()}")
133  bulk = model.CreateBulk(field.GetFieldName())
134 
135  for clusterDescriptor in descriptor.GetClusterIterable():
136  try:
137  clusterIndex = ROOT.Experimental.RClusterIndex(clusterDescriptor.GetId(), 0)
138  except AttributeError:
139  # ROOT Version: 6.35.01
140  clusterIndex = ROOT.RNTupleLocalIndex(clusterDescriptor.GetId(), 0)
141  size = int(clusterDescriptor.GetNEntries())
142  maskReq = array('b', (True for i in range(size)))
143  msg.debug(f" cluster #{clusterIndex.GetClusterId()}"
144  f" firstEntryIndex={clusterDescriptor.GetFirstEntryIndex()}"
145  f" nEntries={size}")
146  values = bulk.ReadBulk(clusterIndex, maskReq, size)
147  msg.debug(f" values array at {values}")
148 
149  except RException as err:
150  from traceback import format_exception
151  msg.error("Exception reading ntuple %r\n%s", ntuple, "".join(format_exception(err)))
152  return 1
153 
154  return 0
155 
156 def checkDirectory(directory, the_type, requireTree, depth):
157 
158  from PyUtils import PoolFile
159  nentries = None
160  hasMetadata = False
161 
162  msg.debug('Checking directory %s ...', directory.GetName())
163 
164  listOfKeys=directory.GetListOfKeys()
165 
166  msg.debug('Checking %s keys ... ', listOfKeys.GetEntries())
167 
168  for key in listOfKeys:
169 
170  msg.debug('Looking at key %s ...', key.GetName())
171  msg.debug('Key is of class %s.', key.GetClassName())
172 
173  the_object=directory.Get(key.GetName())
174  if not the_object:
175  msg.warning("Can't get object of key %s.", key.GetName())
176  return 1
177 
178  if requireTree and not isinstance(the_object, TTree):
179  msg.warning("Object of key %s is not of class TTree!", key.GetName())
180  return 1
181 
182  if isinstance(the_object,TTree):
183 
184  msg.debug('Checking tree %s ...', the_object.GetName())
185 
186  if depth == 0:
187  if PoolFile.PoolOpts.TTreeNames.EventData == the_object.GetName():
188  nentries = the_object.GetEntries()
189  msg.debug(f' contains {nentries} events')
190  elif PoolFile.PoolOpts.TTreeNames.MetaData == the_object.GetName():
191  hasMetadata = True
192  msg.debug(' contains MetaData')
193 
194  if the_type=='event':
195  if checkTreeEventWise(the_object)==1:
196  return 1
197  elif the_type=='basket':
198  if checkTreeBasketWise(the_object)==1:
199  return 1
200 
201  msg.debug('Tree %s looks ok.', the_object.GetName())
202 
203  if isRNTuple(the_object):
204 
205  msg.debug('Checking ntuple of key %s ...', key.GetName())
206 
207  try:
208  reader=RNTupleReader.Open(the_object)
209  except Exception as err:
210  msg.warning('Could not open ntuple %s: %s', the_object, err)
211  return 1
212 
213  if depth == 0:
214  if PoolFile.PoolOpts.RNTupleNames.EventData == reader.GetDescriptor().GetName():
215  nentries = reader.GetNEntries()
216  msg.debug(f' contains {nentries} events')
217  elif PoolFile.PoolOpts.RNTupleNames.MetaData == reader.GetDescriptor().GetName():
218  hasMetadata = True
219  msg.debug(' contains MetaData')
220 
221  if the_type=='event':
222  if checkNTupleEventWise(the_object)==1:
223  return 1
224  elif the_type=='basket':
225  if checkNTupleFieldWise(the_object)==1:
226  return 1
227 
228  msg.debug('NTuple of key %s looks ok.', key.GetName())
229 
230  if isinstance(the_object, TDirectory):
231  if checkDirectory(the_object, the_type, requireTree, depth + 1)==1:
232  return 1
233 
234  # Only check if metadata object is available as in standard POOL files
235  if depth == 0 and hasMetadata and checkNEvents(directory.GetName(), nentries)==1:
236  return 1
237  else:
238  msg.debug('Directory %s looks ok.', directory.GetName())
239  return 0
240 
241 
242 def checkFile(fileName, the_type, requireTree):
243 
244  msg.info('Checking file %s ...', fileName)
245 
246  enabledIMT = False
247  if not ROOT.ROOT.IsImplicitMTEnabled() and 'TRF_MULTITHREADED_VALIDATION' in os.environ and 'ATHENA_CORE_NUMBER' in os.environ:
248  if (nThreads := int(os.environ['ATHENA_CORE_NUMBER'])) >= 0:
249  msg.info(f"Setting the number of implicit ROOT threads to {nThreads}")
250  ROOT.ROOT.EnableImplicitMT(nThreads)
251  enabledIMT = True
252  else:
253  msg.warning(f"Ignored negative ATHENA_CORE_NUMBER ({nThreads})")
254 
255  file_handle=TFile.Open(fileName)
256 
257  if not file_handle:
258  msg.warning("Can't access file %s.", fileName)
259  return 1
260 
261  if not file_handle.IsOpen():
262  msg.warning("Can't open file %s.", fileName)
263  return 1
264 
265  if file_handle.IsZombie():
266  msg.warning("File %s is a zombie.", fileName)
267  file_handle.Close()
268  return 1
269 
270  if file_handle.TestBit(TFile.kRecovered):
271  msg.warning("File %s needed to be recovered.", fileName)
272  file_handle.Close()
273  return 1
274 
275  if checkDirectory(file_handle, the_type, requireTree, 0)==1:
276  msg.warning("File %s is corrupted.", fileName)
277  file_handle.Close()
278  return 1
279 
280  file_handle.Close()
281  msg.info("File %s looks ok.", fileName)
282 
283  if enabledIMT:
284  ROOT.ROOT.DisableImplicitMT()
285 
286  return 0
287 
288 
289 def checkNEvents(fileName, nEntries):
290  """Check consistency of number of events in file with metadata.
291 
292  fileName name of file to check consistency of
293  nEntries number of events in fileName (e.g., obtained by examining event data object)
294  return 0 in case of consistency, 1 otherwise
295  """
296  from PyUtils.MetaReader import read_metadata
297 
298  msg.debug('Checking number of events in file %s ...', fileName)
299 
300  meta = read_metadata(fileName, mode='lite')[fileName]
301  msg.debug(' according to metadata: {0}'.format(meta["nentries"]))
302  msg.debug(' according to event data: {0}'.format(nEntries))
303  if meta["nentries"] and nEntries and meta["nentries"] != nEntries \
304  or meta["nentries"] and not nEntries \
305  or not meta["nentries"] and nEntries:
306  msg.warning(f' number of events ({nEntries}) inconsistent with metadata ({meta["nentries"]}) in file {fileName!r}.')
307  return 1
308  else:
309  msg.debug(" looks ok.")
310  return 0
311 
312 def usage():
313  print("Usage: validate filename type requireTree verbosity")
314  print("'type' must be either 'event' or 'basket'")
315  print("'requireTree' must be either 'true' or 'false'")
316  print("'verbosity' must be either 'on' or 'off'")
317 
318  return 2
319 
320 
321 def main(argv):
322 
323  clock=TStopwatch()
324 
325  argc=len(argv)
326 
327  if (argc!=5):
328  return usage()
329 
330  fileName=argv[1]
331  the_type=argv[2]
332  requireTree=argv[3]
333  verbosity=argv[4]
334 
335 
336  if the_type!="event" and the_type!="basket":
337  return usage()
338 
339  if requireTree=="true":
340  requireTree=True
341  elif requireTree=="false":
342  requireTree=False
343  else:
344  return usage()
345 
346  if verbosity=="on":
347  msg.setLevel(logging.DEBUG)
348  elif verbosity=="off":
349  msg.setLevel(logging.INFO)
350  else:
351  return usage()
352 
353  rc=checkFile(fileName,the_type, requireTree)
354  msg.debug('Returning %s', rc)
355 
356  clock.Stop()
357  clock.Print()
358 
359  return rc
360 
361 
362 if __name__ == '__main__':
363 
364  ch=logging.StreamHandler(sys.stdout)
365  formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
366  ch.setFormatter(formatter)
367  msg.addHandler(ch)
368 
369  rc=main(sys.argv)
370  sys.exit(rc)
371 
python.trfValidateRootFile.usage
def usage()
Definition: trfValidateRootFile.py:312
vtune_athena.format
format
Definition: vtune_athena.py:14
python.MetaReader.read_metadata
def read_metadata(filenames, file_type=None, mode='lite', promote=None, meta_key_filter=None, unique_tag_info_values=True, ignoreNonExistingLocalFiles=False)
Definition: MetaReader.py:53
python.trfValidateRootFile.checkTreeBasketWise
def checkTreeBasketWise(tree)
Definition: trfValidateRootFile.py:46
python.trfValidateRootFile.checkNTupleFieldWise
def checkNTupleFieldWise(ntuple)
Definition: trfValidateRootFile.py:104
python.trfValidateRootFile.checkDirectory
def checkDirectory(directory, the_type, requireTree, depth)
Definition: trfValidateRootFile.py:156
python.trfValidateRootFile.main
def main(argv)
Definition: trfValidateRootFile.py:321
python.trfValidateRootFile.checkNEvents
def checkNEvents(fileName, nEntries)
Definition: trfValidateRootFile.py:289
python.LArMinBiasAlgConfig.int
int
Definition: LArMinBiasAlgConfig.py:59
python.trfValidateRootFile.checkNTupleEventWise
def checkNTupleEventWise(ntuple, printInterval=150000)
Definition: trfValidateRootFile.py:77
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:195
checkFile
Definition: checkFile.py:1
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:25
array
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
python.PoolFile.isRNTuple
def isRNTuple(obj)
Definition: PoolFile.py:36
ROOT::Experimental
Definition: RNTCollection.h:26
python.trfValidateRootFile.checkBranch
def checkBranch(branch)
Definition: trfValidateRootFile.py:22
python.trfValidateRootFile.checkFile
def checkFile(fileName, the_type, requireTree)
Definition: trfValidateRootFile.py:242
python.trfValidateRootFile.checkTreeEventWise
def checkTreeEventWise(tree, printInterval=150000)
Definition: trfValidateRootFile.py:60
Generated on Wed Mar 12 2025 21:21:14 for ATLAS Offline Software by   doxygen 1.8.18