python.Util Namespace Reference

Functions
	getTableName (name)
	toCSV (fileName, metadata, HLTTriggers, readL1=False)
	toJson (fileName, metadata, L1Triggers, HLTTriggers)
	toROOT (fileName, triggers)
	getMetadata (inputFile)
	populateTriggers (inputFile, metadata, globalGroupDict, filter)
	slice_dictionary (directory, object_key)
	populateScanTriggers (inputFile, metadata)
	getGlobalGroup (inputFile, filter)
	readDBFromAMI (amiTag)
	exploreTree (inputFile, dumpSummary=False, underflowThreshold=0.1, overflowThreshold=0.1, maxRanges=5, skipRanges=-1)
	getWalltime (inputFile, rootName)
	getAlgorithmTotalTime (inputFile, rootName)
	convert (entry)
	getFileName (tableName, rootName)
	getHistogramPrefix (tableName, rootName)

Variables
	log = logging.getLogger('RatesPostProcessing')

Detailed Description

@file Util.py
@brief Utility functions used by RatesPostProcessing

@file Util.py
@brief Helper functions for CostAnalysisPostProcessing script

Function Documentation

convert()

python.Util.convert

(

entry

)

@brief Save entry number in scientific notation

Definition at line 142 of file TrigCostAnalysis/python/Util.py.

def convert(entry):
    ''' @brief Save entry number in scientific notation'''
    if type(entry) is float or type(entry) is int:
        # Avoid scientific notation for small numbers and 0
        if entry == 0:
            return 0
        elif fabs(entry) > 10000 or fabs(entry) < 0.0001:
            return "{:.4e}".format(entry)
        elif int(entry) == entry:
            # Get rid of unnecessary 0
            return int(entry)
        else:
            return "{:.4}".format(entry)
 
    return entry
 
 

exploreTree()

python.Util.exploreTree	(		inputFile,
			dumpSummary = False,
			underflowThreshold = 0.1,
			overflowThreshold = 0.1,
			maxRanges = 5,
			skipRanges = -1 )

 @brief Explore ROOT Tree to find tables with histograms to be saved in csv

Per each found directory TableConstructor object is created.
Expected directory tree:
    rootDir
        table1Dir
            entry1Dir
                hist1
                hist2
                ...
            entry2Dir
                hist1
                ...
        table2Dir
        ...
        walltimeHist

@param[in] inputFile ROOT.TFile object with histograms

Definition at line 18 of file TrigCostAnalysis/python/Util.py.

def exploreTree(inputFile, dumpSummary=False, underflowThreshold=0.1, overflowThreshold=0.1, maxRanges=5, skipRanges=-1):
    ''' @brief Explore ROOT Tree to find tables with histograms to be saved in csv
 
    Per each found directory TableConstructor object is created.
    Expected directory tree:
        rootDir
            table1Dir
                entry1Dir
                    hist1
                    hist2
                    ...
                entry2Dir
                    hist1
                    ...
            table2Dir
            ...
            walltimeHist
    
    @param[in] inputFile ROOT.TFile object with histograms
    '''
 
    processingWarnings = []
    rangeCounter = 0
    rangesToSkip = skipRanges
    for timeRange in inputFile.GetListOfKeys():
        if timeRange.GetName() != "metadata" and rangesToSkip > 0:
            rangesToSkip-=1
            log.debug("Skipping range {0}".format(timeRange.GetName()))
            continue
 
        if maxRanges > 0 and rangeCounter >= maxRanges:
            log.info("{0} ranges were processed - exiting the postprocessing".format(rangeCounter))
            break
        
        rangeObj = timeRange.ReadObj()
        if not rangeObj.IsA().InheritsFrom(ROOT.TDirectory.Class()): continue
        
        walltime = getWalltime(inputFile, timeRange.GetName())
 
        for table in rangeObj.GetListOfKeys():
            tableObj = table.ReadObj()
            if not tableObj.IsA().InheritsFrom(ROOT.TDirectory.Class()): continue
            log.info("Processing Table %s", table.GetName())
            # Find and create Table Constructor for specific Table
            try:
                className = table.GetName() + "_TableConstructor"
                exec("from TrigCostAnalysis." + className + " import " + className)
                t = eval(className + "(tableObj, underflowThreshold, overflowThreshold)")
 
                if table.GetName() == "Chain_HLT" or table.GetName() == "Chain_Algorithm_HLT":
                    t.totalTime = getAlgorithmTotalTime(inputFile, rangeObj.GetName())
 
                if table.GetName() == "Global_HLT":
                    t.lbLength = walltime
 
                if table.GetName() == "Algorithm_HLT":
                    t.dumpSummary = dumpSummary
 
                fileName = getFileName(table.GetName(), timeRange.GetName())
                histPrefix = getHistogramPrefix(table.GetName(), timeRange.GetName())
 
                t.fillTable(histPrefix)
                t.normalizeColumns(walltime)
                t.saveToFile(fileName)
 
                processingWarnings += t.getWarningMsgs()
 
            except (ValueError):
                log.error("Processing of table {0} failed!".format(table.GetName()))
                return []
            except (NameError, ImportError):
                log.warning("Class {0} not defined - directory {1} will not be processed"
                            .format(table.GetName()+"_TableConstructor", table.GetName()))
 
        rangeCounter += 1
        log.debug("Range {0} was processed".format(timeRange.GetName()))
 
    # add smmary of most overflown histograms
    summary = createOverflowSummary(processingWarnings)
    summary["Summary"] += ["Underflow threshold: {0}".format(underflowThreshold), "Overflow threshold: {0}".format(overflowThreshold)]
    return processingWarnings + [summary]
 
 

getAlgorithmTotalTime()

python.Util.getAlgorithmTotalTime	(		inputFile,
			rootName )

 @brief Extract total time [s] of algorithms from histogram

@param[in] inputFile ROOT TFile to look for histogram
@param[in] rootName Name of the root directory to search for tables

@return total execution time [s] value if found else 0

Definition at line 124 of file TrigCostAnalysis/python/Util.py.

def getAlgorithmTotalTime(inputFile, rootName):
    ''' @brief Extract total time [s] of algorithms from histogram
    
    @param[in] inputFile ROOT TFile to look for histogram
    @param[in] rootName Name of the root directory to search for tables
 
    @return total execution time [s] value if found else 0
    '''
 
    totalTime = 0
    alg = inputFile.Get(rootName).Get("Global_HLT").Get("Total")
    hist = alg.Get(rootName + "_Global_HLT_Total_AlgTime_perEvent")
    for i in range(1, hist.GetXaxis().GetNbins()):
        totalTime += hist.GetBinContent(i) * hist.GetXaxis().GetBinCenterLog(i)
 
    return totalTime * 1e-3
 
 

getFileName()

python.Util.getFileName	(		tableName,
			rootName )

@brief Get name of file to save the table

@param[in] tableName Table name
@param[in] rootName  Name of table's root directory

@return Filename for given table

Definition at line 159 of file TrigCostAnalysis/python/Util.py.

def getFileName(tableName, rootName):
    '''@brief Get name of file to save the table
 
    @param[in] tableName Table name
    @param[in] rootName  Name of table's root directory
 
    @return Filename for given table
    '''
    return "Table_" + tableName + "_" + rootName + ".csv"
 
 

getGlobalGroup()

python.Util.getGlobalGroup	(		inputFile,
			filter )

Definition at line 245 of file RatesAnalysis/python/Util.py.

def getGlobalGroup(inputFile, filter):
  for key in inputFile.GetListOfKeys():
    if key.GetName() == 'All':
      for subdirKey in key.ReadObj().GetListOfKeys():
        if not subdirKey.GetName() == "Rate_Group_HLT" : pass
        for globalsKey in subdirKey.ReadObj().GetListOfKeys():
          if filter in globalsKey.GetName():
            groupsDict = slice_dictionary(globalsKey.ReadObj(), "data")
            return groupsDict
 
 

getHistogramPrefix()

python.Util.getHistogramPrefix	(		tableName,
			rootName )

@brief Construct full histogram name

@param[in] tableName Table name
@param[in] rootName  Name of table's root directory

@return Histogram prefix for given table

Definition at line 170 of file TrigCostAnalysis/python/Util.py.

def getHistogramPrefix(tableName, rootName):
    '''@brief Construct full histogram name
 
    @param[in] tableName Table name
    @param[in] rootName  Name of table's root directory
 
    @return Histogram prefix for given table
    '''
 
    return rootName + '_' + tableName + '_'

getMetadata()

python.Util.getMetadata

(

inputFile

)

Get metadata for rates.json file

Definition at line 131 of file RatesAnalysis/python/Util.py.

def getMetadata(inputFile):
  '''Get metadata for rates.json file'''
  metatree = inputFile.Get("metadata")
  if metatree is None:
    return None
 
  metatree.GetEntry(0)
  metadata = {}
 
  metadata['runNumber'] = metatree.runNumber
 
  metadata['targetMu'] = metatree.targetMu
  metadata['targetBunches'] = metatree.targetBunches
  metadata['targetLumi'] = metatree.targetLumi
 
  metadata['masterKey'] = metatree.masterKey
  metadata['lvl1PrescaleKey'] = metatree.lvl1PrescaleKey
  metadata['hltPrescaleKey'] = metatree.hltPrescaleKey
 
  metadata['AtlasProject'] = str(metatree.AtlasProject)
  metadata['AtlasVersion'] = str(metatree.AtlasVersion)
 
  metadata['bunchCrossingRate'] = metatree.bunchCrossingRate
 
  metadata['multiSliceDiJet'] = metatree.multiSliceDiJet
 
  prescales = {}
  lowers = {}
  express = {}
  for i in range(0, metatree.triggers.size()):
    prescale = metatree.prescales.at(i)
    expressPrescale = metatree.express.at(i)
    # Handle group prescale values
    prescales[metatree.triggers.at(i)] = prescale if prescale >= -1 else "Multiple"
    lowers[metatree.triggers.at(i)] = str(metatree.lowers.at(i))
    express[metatree.triggers.at(i)] = expressPrescale if expressPrescale >= -1 else "Multiple"
 
  metadata['prescales'] = prescales
  metadata['lowers'] = lowers
  metadata['express'] = express
  
  chainid = {}
  chaingroup = {}
  for i in range(0, metatree.hltChainIDGroup.size()):
    chainid[metatree.hltChainIDGroup.at(i).at(0)] = metatree.hltChainIDGroup.at(i).at(1)
    chaingroup[metatree.hltChainIDGroup.at(i).at(0)] = metatree.hltChainIDGroup.at(i).at(2)
 
  metadata['chainID'] = chainid
  metadata['chainGroup'] = chaingroup
 
  itemid = {}
  for i in range(0, metatree.l1ItemID.size()):
    itemid[metatree.l1ItemID.at(i).at(0)] = metatree.l1ItemID.at(i).at(1)
 
  metadata['itemID'] = itemid
 
  bunchGroups = []
  for bg in metatree.bunchGroups:
    bunchGroups.append(bg)
  metadata['bunchGroups'] = bunchGroups
 
  return metadata
 
 

getTableName()

python.Util.getTableName

(

name

)

Definition at line 14 of file RatesAnalysis/python/Util.py.

def getTableName(name):
  tabName = "Table_Rate_"
  if name == "HLT" or name == "L1":
    tabName += "Chain" + name
  else:
    tabName += name
 
  tabName += "_HLT_All.csv"
 
  return tabName 
 
 

getWalltime()

python.Util.getWalltime	(		inputFile,
			rootName )

 @brief Extract walltime value from histogram

@param[in] inputFile ROOT TFile to look for histogram
@param[in] rootName Name of the root directory to search for tables

@return walltime value if found else 0 and an error

Definition at line 101 of file TrigCostAnalysis/python/Util.py.

def getWalltime(inputFile, rootName):
    ''' @brief Extract walltime value from histogram
    
    @param[in] inputFile ROOT TFile to look for histogram
    @param[in] rootName Name of the root directory to search for tables
 
    @return walltime value if found else 0 and an error
    '''
 
    dirObj = inputFile.Get(rootName)
    if not dirObj.IsA().InheritsFrom(ROOT.TDirectory.Class()): return 0
    for hist in dirObj.GetListOfKeys():
        if '_walltime' in hist.GetName():
            histObj = hist.ReadObj()
            if histObj.IsA().InheritsFrom(ROOT.TProfile.Class()):
                return histObj.Integral()
            else:
                return histObj.GetBinContent(1)
 
    log.error("Walltime not found")
    return 0
 
 

populateScanTriggers()

python.Util.populateScanTriggers	(		inputFile,
			metadata )

Definition at line 231 of file RatesAnalysis/python/Util.py.

def populateScanTriggers(inputFile, metadata):
  from .RatesScanTrigger import RatesScanTrigger
  triggerList = []
  for key in inputFile.GetListOfKeys():
    if key.GetName() == 'ScanTriggers':
      for scanName in key.ReadObj().GetListOfKeys():
          numerator_dict = slice_dictionary(scanName.ReadObj(), "rateVsThreshold")
          if len(numerator_dict) == 0:
            log.error(f"Empty dictionary in populateScanTriggers for scan {scanName}")
            continue
          else:
            triggerList.append(RatesScanTrigger(scanName.GetName(), metadata, numerator_dict))
  return triggerList
 

populateTriggers()

python.Util.populateTriggers	(		inputFile,
			metadata,
			globalGroupDict,
			filter )

Definition at line 195 of file RatesAnalysis/python/Util.py.

def populateTriggers(inputFile, metadata, globalGroupDict, filter):
  # Fix groups' names that are also not GLOBAL
  def getTriggerName(name, filter):
    if "Group" in filter and "GLOBAL" not in name:
      return name.replace('_', ':', 1)
    else:
      return name
 
  from .RatesTrigger import RatesTrigger
  triggerList = []
  for key in inputFile.GetListOfKeys():
    if key.GetName() == 'All':
      for subdirKey in key.ReadObj().GetListOfKeys():
        if filter not in subdirKey.GetName(): continue
        for triggerKey in subdirKey.ReadObj().GetListOfKeys():
            numeratorDict = slice_dictionary(triggerKey.ReadObj(),"data")
            for suffix, data in numeratorDict.items():
              try:
                triggerList.append(RatesTrigger(getTriggerName(triggerKey.GetName(), filter)+suffix, metadata, data, globalGroupDict[suffix], suffix))
              except ValueError:
                log.error("Cannot create a new trigger for {0}".format(triggerKey.GetName()))
                return []
  return triggerList
 
 

readDBFromAMI()

python.Util.readDBFromAMI

(

amiTag

)

Read used database based on AMI tag 

Definition at line 256 of file RatesAnalysis/python/Util.py.

def readDBFromAMI(amiTag):
  ''' Read used database based on AMI tag '''
  try:
    import pyAMI.client
    import pyAMI.atlas.api as AtlasAPI
  except ModuleNotFoundError:
    log.warning("Unable to import AMIClient from pyAMI. Maybe you didn't do localSetupPyAMI?")
    return ""
 
  amiclient = pyAMI.client.Client('atlas')
  AtlasAPI.init()
 
  command = [ 'AMIGetAMITagInfo', '-amiTag="%s"' % amiTag, '-cached' ]
  amiTagInfo = amiclient.execute(command, format = 'dict_object').get_rows('amiTagInfo')[0]
 
  return amiTagInfo['DBserver'] if "DBserver" in amiTagInfo else None

slice_dictionary()

python.Util.slice_dictionary	(		directory,
			object_key )

Definition at line 220 of file RatesAnalysis/python/Util.py.

def slice_dictionary(directory, object_key):
  slices_dict = {}
  for hist in directory.GetListOfKeys():
    if str(hist.GetName()).startswith(object_key):
      try:
        slice_index = "_"+str(hist.GetName()).split("_")[1]
      except IndexError:
        slice_index = ""
      slices_dict[slice_index] = hist.ReadObj()
  return slices_dict
 

toCSV()

python.Util.toCSV	(		fileName,
			metadata,
			HLTTriggers,
			readL1 = False )

Definition at line 26 of file RatesAnalysis/python/Util.py.

def toCSV(fileName, metadata, HLTTriggers, readL1=False):
  import csv
 
  with open(fileName, mode='w') as outputCSV_file:
    rates_csv_writer = csv.writer(outputCSV_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
 
    rates_csv_writer.writerow(['Name','Active Time [s]','Group','Weighted PS Rate [Hz]','Weighted PS Rate Err [Hz]', \
                                      'Unique Rate [Hz]','Unique Rate Err [Hz]','Express Rate [Hz]','Express Rate Err [Hz]','Prescale','Express Prescale','ID', \
                                      'Raw Active Events','Raw Pass Events','Active Events','Input Rate [Hz]','Pass Fraction after PS [%]','Pass Weighted PS'])
    rates_csv_writer.writerow(['Trigger name','Integrated length of all lumi blocks which contributed events to this rates prediction.','The group this chain belongs to.','Rate after applying all prescale(s) as weights.','Error on rate after applying all prescale(s) as weights','Total rate without this chain rate','Error on unique rate','Express stream rate','Error on express rate','The prescale of this chain. Only displayed for simple combinations.','The prescale of the chain including express prescale','The CPTID or HLT Chain ID','Raw underlying statistics on the number events processed for this chain.','Raw underlying statistics on the number events passed by this chain.','Number of events in which the chain - or at least one chain in the combination - was executed.','Input rate to this chain or combination of chains. At L1 this will be the collision frequency for the bunch pattern.','Fraction of events which pass this trigger after prescale.','Number of events this chain or combination passed after applying prescales as weighting factors.'])
 
    for trig in HLTTriggers:
 
      group_name = chain_id = ""
      if "ChainL1" in fileName:
        group_name = "None"
        chain_id = metadata["itemID"].get(trig.name)
      elif "ChainHLT" in fileName:
        group_name = metadata["chainGroup"].get(trig.name)
        chain_id = metadata["chainID"].get(trig.name)
      elif "Group" in fileName:
        chain_id = 0
        group_name = "All" if "GLOBAL" in trig.name else group_name
 
      if float(trig.rateDenominator)==0:
        print("float(trig.rateDenominator) is ZERO! This shouldn't happen")
      if float(trig.activeWeighted)==0:
        passFrac_afterPS=0
      else:
        passFrac_afterPS=100*float(trig.passWeighted)/float(trig.activeWeighted)
 
      isL1 = trig.name.startswith("L1_")
      rates_csv_writer.writerow([trig.name,"%.4f" % trig.rateDenominator,group_name,"%.4f" % trig.rate,"%.4f" % trig.rateErr, \
          "%.4f" % trig.rateUnique,"%.4f" % trig.rateUniqueErr, ("%.4f" % trig.rateExpress if not isL1 else "-"), ("%.4f" % trig.rateExpressErr if not isL1 else "-"), \
          trig.prescale, (trig.expressPrescale if not isL1 else "-"), chain_id, "%.0f" % trig.activeRaw,"%.0f" % trig.passRaw,"%.4f" % trig.activeWeighted, \
          "%.4f" % (float(trig.activeWeighted)/float(trig.rateDenominator)),"%.4f" % passFrac_afterPS,"%.4f" % trig.passWeighted])
 
 
 

toJson()

python.Util.toJson	(		fileName,
			metadata,
			L1Triggers,
			HLTTriggers )

Definition at line 65 of file RatesAnalysis/python/Util.py.

def toJson(fileName, metadata, L1Triggers, HLTTriggers):
  import json
  l1 = {}
  for trig in L1Triggers:
    trig.export(l1)
  hlt = {}
  for trig in HLTTriggers:
    trig.export(hlt)
 
  level = {}
  level['L1'] = l1
  level['HLT'] = hlt
 
  jsonDict = {}
  jsonDict['PredictionLumi'] = metadata['targetLumi']
  for k,v in metadata.items():
    if k.startswith("n_evts"):
      jsonDict[k] = v
  jsonDict['AtlasProject'] = metadata['AtlasProject']
  jsonDict['AtlasVersion'] = metadata['AtlasVersion']
  jsonDict['triggerMenuSetup'] = metadata['masterKey']
  jsonDict['L1PrescaleSet'] = metadata['lvl1PrescaleKey']
  jsonDict['HLTPrescaleSet'] = metadata['hltPrescaleKey']
  jsonDict['bunchgroup'] = metadata['bunchGroups']
  jsonDict['level'] = level
 
  with open(fileName, 'w') as outFile:
    json.dump(obj=jsonDict, fp=outFile, indent=2, sort_keys=True)
 
 
  metajsonData = [
    {'PredictionLumi' : metadata['targetLumi']},
    {'TargetMu' : metadata['targetMu']},
    {'RunNumber' : metadata['runNumber']},
    {'Details' : metadata['details']},
    {'JIRA' : metadata['JIRA']},
    {'AMITag' : metadata['amiTag']},
    {'SMK' :  metadata['masterKey']},
    {'DB' : readDBFromAMI(metadata['amiTag']) if metadata['amiTag'] else None},
    {'LVL1PSK' :  metadata['lvl1PrescaleKey']},
    {'HLTPSK' :  metadata['hltPrescaleKey']},
    {'AtlasProject' : metadata['AtlasProject']},
    {'AtlasVersion' : metadata['AtlasVersion']}
  ]
  for k,v in metadata.items():
    if k.startswith("n_evts"):
      metajsonData.append({k: v})
 
  metajsonDict = {}
  metajsonDict['text'] = 'metadata'
  metajsonDict['children'] = metajsonData
  
  with open('metadata.json', 'w') as outMetaFile:
    json.dump(obj=metajsonDict, fp=outMetaFile, indent=2, sort_keys=True)
 
 

toROOT()

python.Util.toROOT	(		fileName,
			triggers )

Definition at line 121 of file RatesAnalysis/python/Util.py.

def toROOT(fileName, triggers):
  mydict = {}
  for trigger in triggers:
    trigger.export(mydict)
  from ROOT import TFile
  with TFile.Open(fileName, 'RECREATE') as fout:
    for key, scanDict in mydict.items():
      fout.WriteObject(scanDict['rate'], f"{key}_rate")
 
 

Variable Documentation

log

python.Util.log = logging.getLogger('RatesPostProcessing')

Definition at line 12 of file RatesAnalysis/python/Util.py.