ATLAS Offline Software
Functions | Variables
python.trfValidateRootFile Namespace Reference

Functions

def checkBranch (branch)
 
def checkTreeBasketWise (tree)
 
def checkTreeEventWise (tree, printInterval=150000)
 
def checkNTupleEventWise (ntuple, printInterval=150000)
 
def checkNTuplePageWise (ntuple)
 
def checkDirectory (directory, the_type, requireTree, depth)
 
def checkFile (fileName, the_type, requireTree)
 
def checkNEvents (fileName, nEntries)
 
def usage ()
 
def main (argv)
 

Variables

 ROOT
 
 msg
 
 ch
 
 formatter
 
 rc
 

Function Documentation

◆ checkBranch()

def python.trfValidateRootFile.checkBranch (   branch)

Definition at line 21 of file trfValidateRootFile.py.

21 def checkBranch(branch):
22 
23  msg.debug('Checking branch %s ...', branch.GetName())
24 
25  nBaskets=branch.GetWriteBasket()
26 
27  msg.debug('Checking %s baskets ...', nBaskets)
28 
29  for iBasket in range(nBaskets):
30  basket=branch.GetBasket(iBasket)
31  if not basket:
32  msg.warning('Basket %s of branch %s is corrupted.', iBasket, branch.GetName() )
33  return 1
34 
35  listOfSubBranches=branch.GetListOfBranches()
36  msg.debug('Checking %s subbranches ...', listOfSubBranches.GetEntries())
37  for subBranch in listOfSubBranches:
38  if checkBranch(subBranch)==1:
39  return 1
40 
41  msg.debug('Branch %s looks ok.', branch.GetName())
42  return 0
43 
44 

◆ checkDirectory()

def python.trfValidateRootFile.checkDirectory (   directory,
  the_type,
  requireTree,
  depth 
)

Definition at line 144 of file trfValidateRootFile.py.

144 def checkDirectory(directory, the_type, requireTree, depth):
145 
146  from PyUtils import PoolFile
147  nentries = None
148 
149  msg.debug('Checking directory %s ...', directory.GetName())
150 
151  listOfKeys=directory.GetListOfKeys()
152 
153  msg.debug('Checking %s keys ... ', listOfKeys.GetEntries())
154 
155  for key in listOfKeys:
156 
157  msg.debug('Looking at key %s ...', key.GetName())
158  msg.debug('Key is of class %s.', key.GetClassName())
159 
160  the_object=directory.Get(key.GetName())
161  if not the_object:
162  msg.warning("Can't get object of key %s.", key.GetName())
163  return 1
164 
165  if requireTree and not isinstance(the_object, TTree):
166  msg.warning("Object of key %s is not of class TTree!", key.GetName())
167  return 1
168 
169  if isinstance(the_object,TTree):
170 
171  msg.debug('Checking tree %s ...', the_object.GetName())
172 
173  if depth == 0 and PoolFile.PoolOpts.TTreeNames.EventData == the_object.GetName():
174  nentries = the_object.GetEntries()
175  msg.debug(f' contains {nentries} events')
176 
177  if the_type=='event':
178  if checkTreeEventWise(the_object)==1:
179  return 1
180  elif the_type=='basket':
181  if checkTreeBasketWise(the_object)==1:
182  return 1
183 
184  msg.debug('Tree %s looks ok.', the_object.GetName())
185 
186  if isinstance(the_object,RNTuple):
187 
188  msg.debug('Checking ntuple of key %s ...', key.GetName())
189 
190  try:
191  reader=RNTupleReader.Open(the_object)
192  except Exception as err:
193  msg.warning('Could not open ntuple %s: %s', the_object, err)
194  return 1
195 
196  if depth == 0 and PoolFile.PoolOpts.RNTupleNames.EventData == reader.GetDescriptor().GetName():
197  nentries = reader.GetNEntries()
198  msg.debug(f' contains {nentries} events')
199 
200  if the_type=='event':
201  if checkNTupleEventWise(the_object)==1:
202  return 1
203  elif the_type=='basket':
204  if checkNTuplePageWise(the_object)==1:
205  return 1
206 
207  msg.debug('NTuple of key %s looks ok.', key.GetName())
208 
209  if isinstance(the_object, TDirectory):
210  if checkDirectory(the_object, the_type, requireTree, depth + 1)==1:
211  return 1
212 
213  if depth == 0 and checkNEvents(directory.GetName(), nentries)==1:
214  return 1
215  else:
216  msg.debug('Directory %s looks ok.', directory.GetName())
217  return 0
218 
219 

◆ checkFile()

def python.trfValidateRootFile.checkFile (   fileName,
  the_type,
  requireTree 
)

Definition at line 220 of file trfValidateRootFile.py.

220 def checkFile(fileName, the_type, requireTree):
221 
222  msg.info('Checking file %s ...', fileName)
223 
224  isIMTEnabled = ROOT.ROOT.IsImplicitMTEnabled()
225  if not isIMTEnabled and 'TRF_MULTITHREADED_VALIDATION' in os.environ and 'ATHENA_CORE_NUMBER' in os.environ:
226  nThreads = int(os.environ['ATHENA_CORE_NUMBER'])
227  msg.info(f"Setting the number of implicit ROOT threads to {nThreads}")
228  ROOT.ROOT.EnableImplicitMT(nThreads)
229 
230  file_handle=TFile.Open(fileName)
231 
232  if not file_handle:
233  msg.warning("Can't access file %s.", fileName)
234  return 1
235 
236  if not file_handle.IsOpen():
237  msg.warning("Can't open file %s.", fileName)
238  return 1
239 
240  if file_handle.IsZombie():
241  msg.warning("File %s is a zombie.", fileName)
242  file_handle.Close()
243  return 1
244 
245  if file_handle.TestBit(TFile.kRecovered):
246  msg.warning("File %s needed to be recovered.", fileName)
247  file_handle.Close()
248  return 1
249 
250  if checkDirectory(file_handle, the_type, requireTree, 0)==1:
251  msg.warning("File %s is corrupted.", fileName)
252  file_handle.Close()
253  return 1
254 
255  file_handle.Close()
256  msg.info("File %s looks ok.", fileName)
257 
258  if not isIMTEnabled and 'TRF_MULTITHREADED_VALIDATION' in os.environ and 'ATHENA_CORE_NUMBER' in os.environ:
259  ROOT.ROOT.DisableImplicitMT()
260 
261  return 0
262 
263 

◆ checkNEvents()

def python.trfValidateRootFile.checkNEvents (   fileName,
  nEntries 
) 
Check consistency of number of events in file with metadata.

fileName   name of file to check consistency of
nEntries   number of events in fileName (e.g., obtained by examining event data object)
return     0 in case of consistency, 1 otherwise

Definition at line 264 of file trfValidateRootFile.py.

264 def checkNEvents(fileName, nEntries):
265  """Check consistency of number of events in file with metadata.
266 
267  fileName name of file to check consistency of
268  nEntries number of events in fileName (e.g., obtained by examining event data object)
269  return 0 in case of consistency, 1 otherwise
270  """
271  from PyUtils.MetaReader import read_metadata
272 
273  from AthenaCommon.Logging import logging as athlogging
274  from PyUtils.MetaReader import msg as metamsg
275 
276  if msg.getEffectiveLevel() != logging.DEBUG:
277  metamsg.setLevel(athlogging.ERROR)
278 
279  msg.debug('Checking number of events in file %s ...', fileName)
280 
281  meta = read_metadata(fileName, mode='lite')[fileName]
282  msg.debug(' according to metadata: {0}'.format(meta["nentries"]))
283  msg.debug(' according to event data: {0}'.format(nEntries))
284  if meta["nentries"] and nEntries and meta["nentries"] != nEntries \
285  or meta["nentries"] and not nEntries \
286  or not meta["nentries"] and nEntries:
287  msg.warning(f' number of events ({nEntries}) inconsistent with metadata ({meta["nentries"]}) in file {fileName!r}.')
288  return 1
289  else:
290  msg.debug(" looks ok.")
291  return 0
292 

◆ checkNTupleEventWise()

def python.trfValidateRootFile.checkNTupleEventWise (   ntuple,
  printInterval = 150000 
)

Definition at line 76 of file trfValidateRootFile.py.

76 def checkNTupleEventWise(ntuple, printInterval = 150000):
77 
78  try:
79  reader=RNTupleReader.Open(ntuple)
80  except Exception as err:
81  msg.warning('Could not open ntuple %s: %s', ntuple, err)
82  return 1
83 
84  msg.debug('Checking %s entries ...', reader.GetNEntries())
85 
86  for i in reader:
87  try:
88  reader.LoadEntry(i)
89  except Exception as err:
90  msg.warning('Event %s of ntuple %s is corrupted: %s', i, reader.GetDescriptor().GetName(), err)
91  return 1
92 
93  # Show a sign of life for long validation jobs: ATLASJT-433
94  if (i%printInterval)==0 and i>0:
95  msg.info('Validated %s events so far ...', i)
96 
97  return 0
98 

◆ checkNTuplePageWise()

def python.trfValidateRootFile.checkNTuplePageWise (   ntuple) 
Check each page column by column.

Definition at line 99 of file trfValidateRootFile.py.

99 def checkNTuplePageWise(ntuple):
100  """Check each page column by column.
101  """
102 
103  try:
104  pageSource = ntuple.MakePageSource()
105  pageSource.Attach()
106  except Exception as err:
107  msg.warning(f'Could not make page source for ntuple {ntuple!r}: {err}')
108  return 1
109 
110  msg.debug(f'Checking pages of ntuple {pageSource.GetNTupleName()!r}')
111  descriptor=pageSource.GetSharedDescriptorGuard().GetRef()
112  for columnDesc in descriptor.GetColumnIterable():
113  columnPhysicalId = columnDesc.GetPhysicalId()
114  msg.debug(f" Checking column {columnPhysicalId} ...")
115  for cluster in descriptor.GetClusterIterable():
116  if not cluster.ContainsColumn(columnPhysicalId):
117  msg.debug(f" Not in cluster {cluster.GetId()}")
118  continue
119  nElements = cluster.GetColumnRange(columnPhysicalId).fNElements.fValue
120  idxInCluster = 0
121  for (pageNo, pageInfo) in enumerate(cluster.GetPageRange(columnPhysicalId).fPageInfos):
122  msg.debug(f" Page {pageNo}, {pageInfo.fNElements} elements in cluster {cluster.GetId()}")
123  buffer = bytearray(pageInfo.fLocator.fBytesOnStorage)
124  sealedPage = ROOT.Experimental.Detail.RPageStorage.RSealedPage(buffer,\
125  pageInfo.fLocator.fBytesOnStorage,\
126  pageInfo.fNElements)
127  try:
128  pageSource.LoadSealedPage(columnPhysicalId,\
129  ROOT.Experimental.RClusterIndex(cluster.GetId(), idxInCluster),\
130  sealedPage)
131  except Exception as err:
132  msg.warning(f'Could not load SealedPage {sealedPage!r} for page {pageNo} in cluster {cluster.GetId()}'
133  f' of ntuple {pageSource.GetNTupleName()!r}: {err}')
134  return 1
135  idxInCluster += pageInfo.fNElements
136  if idxInCluster == nElements:
137  msg.debug(f" {nElements} elements in cluster {cluster.GetId()}")
138  else:
139  msg.warning(f"NTuple {descriptor.GetName()!r}, cluster {cluster.GetId()}, column {cluster.GetColumnRange(columnPhysicalId).fPhysicalColumnId}: inconsistent meta-data")
140  return 1
141 
142  return 0
143 

◆ checkTreeBasketWise()

def python.trfValidateRootFile.checkTreeBasketWise (   tree)

Definition at line 45 of file trfValidateRootFile.py.

45 def checkTreeBasketWise(tree):
46 
47  listOfBranches=tree.GetListOfBranches()
48 
49  msg.debug('Checking %s branches ...', listOfBranches.GetEntries())
50 
51  for branch in listOfBranches:
52  if checkBranch(branch)==1:
53  msg.warning('Tree %s is corrupted (branch %s ).', tree.GetName(), branch.GetName())
54  return 1
55 
56  return 0
57 
58 

◆ checkTreeEventWise()

def python.trfValidateRootFile.checkTreeEventWise (   tree,
  printInterval = 150000 
)

Definition at line 59 of file trfValidateRootFile.py.

59 def checkTreeEventWise(tree, printInterval = 150000):
60 
61  nEntries=tree.GetEntries()
62 
63  msg.debug('Checking %s entries ...', nEntries)
64 
65  for i in range(nEntries):
66  if tree.GetEntry(i)<0:
67  msg.warning('Event %s of tree %s is corrupted.', i, tree.GetName())
68  return 1
69 
70  # Show a sign of life for long validation jobs: ATLASJT-433
71  if (i%printInterval)==0 and i>0:
72  msg.info('Validated %s events so far ...', i)
73 
74  return 0
75 

◆ main()

def python.trfValidateRootFile.main (   argv)

Definition at line 302 of file trfValidateRootFile.py.

302 def main(argv):
303 
304  clock=TStopwatch()
305 
306  argc=len(argv)
307 
308  if (argc!=5):
309  return usage()
310 
311  fileName=argv[1]
312  the_type=argv[2]
313  requireTree=argv[3]
314  verbosity=argv[4]
315 
316 
317  if the_type!="event" and the_type!="basket":
318  return usage()
319 
320  if requireTree=="true":
321  requireTree=True
322  elif requireTree=="false":
323  requireTree=False
324  else:
325  return usage()
326 
327  if verbosity=="on":
328  msg.setLevel(logging.DEBUG)
329  elif verbosity=="off":
330  msg.setLevel(logging.INFO)
331  else:
332  return usage()
333 
334  rc=checkFile(fileName,the_type, requireTree)
335  msg.debug('Returning %s', rc)
336 
337  clock.Stop()
338  clock.Print()
339 
340  return rc
341 
342 

◆ usage()

def python.trfValidateRootFile.usage ( )

Definition at line 293 of file trfValidateRootFile.py.

293 def usage():
294  print("Usage: validate filename type requireTree verbosity")
295  print("'type' must be either 'event' or 'basket'")
296  print("'requireTree' must be either 'true' or 'false'")
297  print("'verbosity' must be either 'on' or 'off'")
298 
299  return 2
300 
301 

Variable Documentation

◆ ch

python.trfValidateRootFile.ch

Definition at line 345 of file trfValidateRootFile.py.

◆ formatter

python.trfValidateRootFile.formatter

Definition at line 346 of file trfValidateRootFile.py.

◆ msg

python.trfValidateRootFile.msg

Definition at line 19 of file trfValidateRootFile.py.

◆ rc

python.trfValidateRootFile.rc

Definition at line 350 of file trfValidateRootFile.py.

◆ ROOT

python.trfValidateRootFile.ROOT

Definition at line 15 of file trfValidateRootFile.py.

python.trfValidateRootFile.usage
def usage()
Definition: trfValidateRootFile.py:293
vtune_athena.format
format
Definition: vtune_athena.py:14
CaloCellPos2Ntuple.int
int
Definition: CaloCellPos2Ntuple.py:24
python.MetaReader.read_metadata
def read_metadata(filenames, file_type=None, mode='lite', promote=None, meta_key_filter=None, unique_tag_info_values=True, ignoreNonExistingLocalFiles=False)
Definition: MetaReader.py:52
python.trfValidateRootFile.checkTreeBasketWise
def checkTreeBasketWise(tree)
Definition: trfValidateRootFile.py:45
python.trfValidateRootFile.checkDirectory
def checkDirectory(directory, the_type, requireTree, depth)
Definition: trfValidateRootFile.py:144
python.trfValidateRootFile.checkNTuplePageWise
def checkNTuplePageWise(ntuple)
Definition: trfValidateRootFile.py:99
python.trfValidateRootFile.main
def main(argv)
Definition: trfValidateRootFile.py:302
python.trfValidateRootFile.checkNEvents
def checkNEvents(fileName, nEntries)
Definition: trfValidateRootFile.py:264
python.trfValidateRootFile.checkNTupleEventWise
def checkNTupleEventWise(ntuple, printInterval=150000)
Definition: trfValidateRootFile.py:76
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:195
checkFile
Definition: checkFile.py:1
Muon::print
std::string print(const MuPatSegment &)
Definition: MuonTrackSteering.cxx:28
python.trfValidateRootFile.checkBranch
def checkBranch(branch)
Definition: trfValidateRootFile.py:21
python.trfValidateRootFile.checkFile
def checkFile(fileName, the_type, requireTree)
Definition: trfValidateRootFile.py:220
python.trfValidateRootFile.checkTreeEventWise
def checkTreeEventWise(tree, printInterval=150000)
Definition: trfValidateRootFile.py:59
Generated on Fri Jun 28 2024 22:26:51 for ATLAS Offline Software by   doxygen 1.8.18