Functions
def	getTableName (name)

def	toCSV (fileName, metadata, HLTTriggers, readL1=False)

def	toJson (fileName, metadata, L1Triggers, HLTTriggers)

def	getMetadata (inputFile)

def	populateTriggers (inputFile, metadata, globalGroup, filter)

def	getGlobalGroup (inputFile, filter)

def	readDBFromAMI (amiTag)

def	exploreTree (inputFile, dumpSummary=False, underflowThreshold=0.1, overflowThreshold=0.1, maxRanges=5, skipRanges=-1)

def	getWalltime (inputFile, rootName)

def	getAlgorithmTotalTime (inputFile, rootName)

def	convert (entry)

def	getFileName (tableName, rootName)

def	getHistogramPrefix (tableName, rootName)

Variables
	log

Function Documentation

◆ convert()

def python.Util.convert ( entry )

@brief Save entry number in scientific notation

Definition at line 142 of file TrigCostAnalysis/python/Util.py.

 def convert(entry):
     ''' @brief Save entry number in scientific notation'''
     if type(entry) is float or type(entry) is int:
         # Avoid scientific notation for small numbers and 0
         if entry == 0:
             return 0
         elif fabs(entry) > 10000 or fabs(entry) < 0.0001:
             return "{:.4e}".format(entry)
         elif int(entry) == entry:
             # Get rid of unnecessary 0
             return int(entry)
         else:
             return "{:.4}".format(entry)
  
     return entry
  
  

◆ exploreTree()

def python.Util.exploreTree	(	inputFile,
		dumpSummary = `False`,
		underflowThreshold = `0.1`,
		overflowThreshold = `0.1`,
		maxRanges = `5`,
		skipRanges = `-1`
	)

@brief Explore ROOT Tree to find tables with histograms to be saved in csv

Per each found directory TableConstructor object is created.
Expected directory tree:
    rootDir
        table1Dir
            entry1Dir
                hist1
                hist2
                ...
            entry2Dir
                hist1
                ...
        table2Dir
        ...
        walltimeHist

@param[in] inputFile ROOT.TFile object with histograms

Definition at line 18 of file TrigCostAnalysis/python/Util.py.

 def exploreTree(inputFile, dumpSummary=False, underflowThreshold=0.1, overflowThreshold=0.1, maxRanges=5, skipRanges=-1):
     ''' @brief Explore ROOT Tree to find tables with histograms to be saved in csv
  
     Per each found directory TableConstructor object is created.
     Expected directory tree:
         rootDir
             table1Dir
                 entry1Dir
                     hist1
                     hist2
                     ...
                 entry2Dir
                     hist1
                     ...
             table2Dir
             ...
             walltimeHist
     
     @param[in] inputFile ROOT.TFile object with histograms
     '''
  
     processingWarnings = []
     rangeCounter = 0
     rangesToSkip = skipRanges
     for timeRange in inputFile.GetListOfKeys():
         if timeRange.GetName() != "metadata" and rangesToSkip > 0:
             rangesToSkip-=1
             log.debug("Skipping range {0}".format(timeRange.GetName()))
             continue
  
         if maxRanges > 0 and rangeCounter >= maxRanges:
             log.info("{0} ranges were processed - exiting the postprocessing".format(rangeCounter))
             break
         
         rangeObj = timeRange.ReadObj()
         if not rangeObj.IsA().InheritsFrom(ROOT.TDirectory.Class()): continue
         
         walltime = getWalltime(inputFile, timeRange.GetName())
  
         for table in rangeObj.GetListOfKeys():
             tableObj = table.ReadObj()
             if not tableObj.IsA().InheritsFrom(ROOT.TDirectory.Class()): continue
             log.info("Processing Table %s", table.GetName())
             # Find and create Table Constructor for specific Table
             try:
                 className = table.GetName() + "_TableConstructor"
                 exec("from TrigCostAnalysis." + className + " import " + className)
                 t = eval(className + "(tableObj, underflowThreshold, overflowThreshold)")
  
                 if table.GetName() == "Chain_HLT" or table.GetName() == "Chain_Algorithm_HLT":
                     t.totalTime = getAlgorithmTotalTime(inputFile, rangeObj.GetName())
  
                 if table.GetName() == "Global_HLT":
                     t.lbLength = walltime
  
                 if table.GetName() == "Algorithm_HLT":
                     t.dumpSummary = dumpSummary
  
                 fileName = getFileName(table.GetName(), timeRange.GetName())
                 histPrefix = getHistogramPrefix(table.GetName(), timeRange.GetName())
  
                 t.fillTable(histPrefix)
                 t.normalizeColumns(walltime)
                 t.saveToFile(fileName)
  
                 processingWarnings += t.getWarningMsgs()
  
             except (ValueError):
                 log.error("Processing of table {0} failed!".format(table.GetName()))
                 return []
             except (NameError, ImportError):
                 log.warning("Class {0} not defined - directory {1} will not be processed"
                             .format(table.GetName()+"_TableConstructor", table.GetName()))
  
         rangeCounter += 1
         log.debug("Range {0} was processed".format(timeRange.GetName()))
  
     # add smmary of most overflown histograms
     summary = createOverflowSummary(processingWarnings)
     summary["Summary"] += ["Underflow threshold: {0}".format(underflowThreshold), "Overflow threshold: {0}".format(overflowThreshold)]
     return processingWarnings + [summary]
  
  

◆ getAlgorithmTotalTime()

def python.Util.getAlgorithmTotalTime	(	inputFile,
		rootName
	)

@brief Extract total time [s] of algorithms from histogram

@param[in] inputFile ROOT TFile to look for histogram
@param[in] rootName Name of the root directory to search for tables

@return total execution time [s] value if found else 0

Definition at line 124 of file TrigCostAnalysis/python/Util.py.

 def getAlgorithmTotalTime(inputFile, rootName):
     ''' @brief Extract total time [s] of algorithms from histogram
     
     @param[in] inputFile ROOT TFile to look for histogram
     @param[in] rootName Name of the root directory to search for tables
  
     @return total execution time [s] value if found else 0
     '''
  
     totalTime = 0
     alg = inputFile.Get(rootName).Get("Global_HLT").Get("Total")
     hist = alg.Get(rootName + "_Global_HLT_Total_AlgTime_perEvent")
     for i in range(1, hist.GetXaxis().GetNbins()):
         totalTime += hist.GetBinContent(i) * hist.GetXaxis().GetBinCenterLog(i)
  
     return totalTime * 1e-3
  
  

◆ getFileName()

def python.Util.getFileName	(	tableName,
		rootName
	)

@brief Get name of file to save the table

@param[in] tableName Table name
@param[in] rootName  Name of table's root directory

@return Filename for given table

Definition at line 159 of file TrigCostAnalysis/python/Util.py.

 def getFileName(tableName, rootName):
     '''@brief Get name of file to save the table
  
     @param[in] tableName Table name
     @param[in] rootName  Name of table's root directory
  
     @return Filename for given table
     '''
     return "Table_" + tableName + "_" + rootName + ".csv"
  
  

◆ getGlobalGroup()

def python.Util.getGlobalGroup	(	inputFile,
		filter
	)

Definition at line 203 of file RatesAnalysis/python/Util.py.

 def getGlobalGroup(inputFile, filter):
   for key in inputFile.GetListOfKeys():
     if key.GetName() == 'All':
       for subdirKey in key.ReadObj().GetListOfKeys():
         if not subdirKey.GetName() == "Rate_Group_HLT" : pass
         for globalsKey in subdirKey.ReadObj().GetListOfKeys():
           if filter in globalsKey.GetName():
             for hist in globalsKey.ReadObj().GetListOfKeys():
               if hist.GetName() == 'data':
                 return hist.ReadObj()
  
  

◆ getHistogramPrefix()

def python.Util.getHistogramPrefix	(	tableName,
		rootName
	)

@brief Construct full histogram name

@param[in] tableName Table name
@param[in] rootName  Name of table's root directory

@return Histogram prefix for given table

Definition at line 170 of file TrigCostAnalysis/python/Util.py.

 def getHistogramPrefix(tableName, rootName):
     '''@brief Construct full histogram name
  
     @param[in] tableName Table name
     @param[in] rootName  Name of table's root directory
  
     @return Histogram prefix for given table
     '''
  
     return rootName + '_' + tableName + '_'

◆ getMetadata()

def python.Util.getMetadata ( inputFile )

Get metadata for rates.json file

Definition at line 118 of file RatesAnalysis/python/Util.py.

 def getMetadata(inputFile):
   '''Get metadata for rates.json file'''
   metatree = inputFile.Get("metadata")
   if metatree is None:
     return None
  
   metatree.GetEntry(0)
   metadata = {}
  
   metadata['runNumber'] = metatree.runNumber
  
   metadata['targetMu'] = metatree.targetMu
   metadata['targetBunches'] = metatree.targetBunches
   metadata['targetLumi'] = metatree.targetLumi
  
   metadata['masterKey'] = metatree.masterKey
   metadata['lvl1PrescaleKey'] = metatree.lvl1PrescaleKey
   metadata['hltPrescaleKey'] = metatree.hltPrescaleKey
  
   metadata['AtlasProject'] = str(metatree.AtlasProject)
   metadata['AtlasVersion'] = str(metatree.AtlasVersion)
  
   prescales = {}
   lowers = {}
   express = {}
   for i in range(0, metatree.triggers.size()):
     prescale = metatree.prescales.at(i)
     expressPrescale = metatree.express.at(i)
     # Handle group prescale values
     prescales[metatree.triggers.at(i)] = prescale if prescale >= -1 else "Multiple"
     lowers[metatree.triggers.at(i)] = str(metatree.lowers.at(i))
     express[metatree.triggers.at(i)] = expressPrescale if expressPrescale >= -1 else "Multiple"
  
   metadata['prescales'] = prescales
   metadata['lowers'] = lowers
   metadata['express'] = express
   
   chainid = {}
   chaingroup = {}
   for i in range(0, metatree.hltChainIDGroup.size()):
     chainid[metatree.hltChainIDGroup.at(i).at(0)] = metatree.hltChainIDGroup.at(i).at(1)
     chaingroup[metatree.hltChainIDGroup.at(i).at(0)] = metatree.hltChainIDGroup.at(i).at(2)
  
   metadata['chainID'] = chainid
   metadata['chainGroup'] = chaingroup
  
   itemid = {}
   for i in range(0, metatree.l1ItemID.size()):
     itemid[metatree.l1ItemID.at(i).at(0)] = metatree.l1ItemID.at(i).at(1)
  
   metadata['itemID'] = itemid
  
   bunchGroups = []
   for bg in metatree.bunchGroups:
     bunchGroups.append(bg)
   metadata['bunchGroups'] = bunchGroups
  
   return metadata
  
  

◆ getTableName()

def python.Util.getTableName ( name )

Definition at line 14 of file RatesAnalysis/python/Util.py.

 def getTableName(name):
   tabName = "Table_Rate_"
   if name == "HLT" or name == "L1":
     tabName += "Chain" + name
   else:
     tabName += name
  
   tabName += "_HLT_All.csv"
  
   return tabName 
  
  

◆ getWalltime()

def python.Util.getWalltime	(	inputFile,
		rootName
	)

@brief Extract walltime value from histogram

@param[in] inputFile ROOT TFile to look for histogram
@param[in] rootName Name of the root directory to search for tables

@return walltime value if found else 0 and an error

Definition at line 101 of file TrigCostAnalysis/python/Util.py.

 def getWalltime(inputFile, rootName):
     ''' @brief Extract walltime value from histogram
     
     @param[in] inputFile ROOT TFile to look for histogram
     @param[in] rootName Name of the root directory to search for tables
  
     @return walltime value if found else 0 and an error
     '''
  
     dirObj = inputFile.Get(rootName)
     if not dirObj.IsA().InheritsFrom(ROOT.TDirectory.Class()): return 0
     for hist in dirObj.GetListOfKeys():
         if '_walltime' in hist.GetName():
             histObj = hist.ReadObj()
             if histObj.IsA().InheritsFrom(ROOT.TProfile.Class()):
                 return histObj.Integral()
             else:
                 return histObj.GetBinContent(1)
  
     log.error("Walltime not found")
     return 0
  
  

◆ populateTriggers()

def python.Util.populateTriggers	(	inputFile,
		metadata,
		globalGroup,
		filter
	)

Definition at line 178 of file RatesAnalysis/python/Util.py.

 def populateTriggers(inputFile, metadata, globalGroup, filter):
   # Fix groups' names that are also not GLOBAL
   def getTriggerName(name, filter):
     if "Group" in filter and "GLOBAL" not in name:
       return name.replace('_', ':', 1)
     else:
       return name
  
   from .RatesTrigger import RatesTrigger
   triggerList = []
   for key in inputFile.GetListOfKeys():
     if key.GetName() == 'All':
       for subdirKey in key.ReadObj().GetListOfKeys():
         if filter not in subdirKey.GetName(): continue
         for triggerKey in subdirKey.ReadObj().GetListOfKeys():
             for hist in triggerKey.ReadObj().GetListOfKeys():
               if hist.GetName() == 'data':
                 try:
                   triggerList.append(RatesTrigger(getTriggerName(triggerKey.GetName(), filter), metadata, hist.ReadObj(), globalGroup))
                 except ValueError:
                   log.error("Cannot create a new trigger for {0}".format(triggerKey.GetName()))
                   return []
   return triggerList
  
  

◆ readDBFromAMI()

def python.Util.readDBFromAMI ( amiTag )

Read used database based on AMI tag

Definition at line 215 of file RatesAnalysis/python/Util.py.

 def readDBFromAMI(amiTag):
   ''' Read used database based on AMI tag '''
   try:
     import pyAMI.client
     import pyAMI.atlas.api as AtlasAPI
   except ModuleNotFoundError:
     log.warning("Unable to import AMIClient from pyAMI. Maybe you didn't do localSetupPyAMI?")
     return ""
  
   amiclient = pyAMI.client.Client('atlas')
   AtlasAPI.init()
  
   command = [ 'AMIGetAMITagInfo', '-amiTag="%s"' % amiTag, '-cached' ]
   amiTagInfo = amiclient.execute(command, format = 'dict_object').get_rows('amiTagInfo')[0]
  
   return amiTagInfo['DBserver'] if "DBserver" in amiTagInfo else None

◆ toCSV()

def python.Util.toCSV	(	fileName,
		metadata,
		HLTTriggers,
		readL1 = `False`
	)

Definition at line 26 of file RatesAnalysis/python/Util.py.

 def toCSV(fileName, metadata, HLTTriggers, readL1=False):
   import csv
  
   with open(fileName, mode='w') as outputCSV_file:
     rates_csv_writer = csv.writer(outputCSV_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
  
     rates_csv_writer.writerow(['Name','Active Time [s]','Group','Weighted PS Rate [Hz]','Weighted PS Rate Err [Hz]', \
                                       'Unique Rate [Hz]','Unique Rate Err [Hz]','Express Rate [Hz]','Express Rate Err [Hz]','Prescale','Express Prescale','ID', \
                                       'Raw Active Events','Raw Pass Events','Active Events','Input Rate [Hz]','Pass Fraction after PS [%]','Pass Weighted PS'])
     rates_csv_writer.writerow(['Trigger name','Integrated length of all lumi blocks which contributed events to this rates prediction.','The group this chain belongs to.','Rate after applying all prescale(s) as weights.','Error on rate after applying all prescale(s) as weights','Total rate without this chain rate','Error on unique rate','Express stream rate','Error on express rate','The prescale of this chain. Only displayed for simple combinations.','The prescale of the chain including express prescale','The CPTID or HLT Chain ID','Raw underlying statistics on the number events processed for this chain.','Raw underlying statistics on the number events passed by this chain.','Number of events in which the chain - or at least one chain in the combination - was executed.','Input rate to this chain or combination of chains. At L1 this will be the collision frequency for the bunch pattern.','Fraction of events which pass this trigger after prescale.','Number of events this chain or combination passed after applying prescales as weighting factors.'])
  
     for trig in HLTTriggers:
  
       group_name = chain_id = ""
       if "ChainL1" in fileName:
         group_name = "None"
         chain_id = metadata["itemID"].get(trig.name)
       elif "ChainHLT" in fileName:
         group_name = metadata["chainGroup"].get(trig.name)
         chain_id = metadata["chainID"].get(trig.name)
       elif "Group" in fileName:
         chain_id = 0
         group_name = "All" if "GLOBAL" in trig.name else group_name
  
       if float(trig.rateDenominator)==0:
         print("float(trig.rateDenominator) is ZERO! This shouldn't happen")
       if float(trig.activeWeighted)==0:
         passFrac_afterPS=0
       else:
         passFrac_afterPS=100*float(trig.passWeighted)/float(trig.activeWeighted)
  
       isL1 = trig.name.startswith("L1_")
       rates_csv_writer.writerow([trig.name,"%.4f" % trig.rateDenominator,group_name,"%.4f" % trig.rate,"%.4f" % trig.rateErr, \
           "%.4f" % trig.rateUnique,"%.4f" % trig.rateUniqueErr, ("%.4f" % trig.rateExpress if not isL1 else "-"), ("%.4f" % trig.rateExpressErr if not isL1 else "-"), \
           trig.prescale, (trig.expressPrescale if not isL1 else "-"), chain_id, "%.0f" % trig.activeRaw,"%.0f" % trig.passRaw,"%.4f" % trig.activeWeighted, \
           "%.4f" % (float(trig.activeWeighted)/float(trig.rateDenominator)),"%.4f" % passFrac_afterPS,"%.4f" % trig.passWeighted])
       
     
  

◆ toJson()

def python.Util.toJson	(	fileName,
		metadata,
		L1Triggers,
		HLTTriggers
	)

Definition at line 65 of file RatesAnalysis/python/Util.py.

 def toJson(fileName, metadata, L1Triggers, HLTTriggers):
   import json
   l1 = {}
   for trig in L1Triggers:
     trig.export(l1)
   hlt = {}
   for trig in HLTTriggers:
     trig.export(hlt)
  
   level = {}
   level['L1'] = l1
   level['HLT'] = hlt
  
   jsonDict = {}
   jsonDict['PredictionLumi'] = metadata['targetLumi']
   jsonDict['n_evts'] = metadata['n_evts']
   jsonDict['AtlasProject'] = metadata['AtlasProject']
   jsonDict['AtlasVersion'] = metadata['AtlasVersion']
   jsonDict['triggerMenuSetup'] = metadata['masterKey']
   jsonDict['L1PrescaleSet'] = metadata['lvl1PrescaleKey']
   jsonDict['HLTPrescaleSet'] = metadata['hltPrescaleKey']
   jsonDict['bunchgroup'] = metadata['bunchGroups']
   jsonDict['level'] = level
  
   with open(fileName, 'w') as outFile:
     json.dump(obj=jsonDict, fp=outFile, indent=2, sort_keys=True)
  
  
   metajsonData = [
     {'PredictionLumi' : metadata['targetLumi']},
     {'TargetMu' : metadata['targetMu']},
     {'RunNumber' : metadata['runNumber']},
     {'NEvents' : metadata['n_evts']},
     {'Details' : metadata['details']},
     {'JIRA' : metadata['JIRA']},
     {'AMITag' : metadata['amiTag']},
     {'SMK' :  metadata['masterKey']},
     {'DB' : readDBFromAMI(metadata['amiTag']) if metadata['amiTag'] else None},
     {'LVL1PSK' :  metadata['lvl1PrescaleKey']},
     {'HLTPSK' :  metadata['hltPrescaleKey']},
     {'AtlasProject' : metadata['AtlasProject']},
     {'AtlasVersion' : metadata['AtlasVersion']}
   ]
  
  
   metajsonDict = {}
   metajsonDict['text'] = 'metadata'
   metajsonDict['children'] = metajsonData
   
   with open('metadata.json', 'w') as outMetaFile:
     json.dump(obj=metajsonDict, fp=outMetaFile, indent=2, sort_keys=True)
  
  

Variable Documentation

◆ log

python.Util.log

Definition at line 12 of file RatesAnalysis/python/Util.py.

Functions

Variables

Function Documentation

◆ convert()

◆ exploreTree()

◆ getAlgorithmTotalTime()

◆ getFileName()

◆ getGlobalGroup()

◆ getHistogramPrefix()

◆ getMetadata()

◆ getTableName()

◆ getWalltime()

◆ populateTriggers()

◆ readDBFromAMI()

◆ toCSV()

◆ toJson()

Variable Documentation

◆ log