covarianceTool Namespace Reference

Functions
def	isFloat (x)
def	convertHistToScatter (ao)
def	renameAOs (fin_name, aoMap)
def	makeBeamer (plotDictionary, outdir)
def	makeSummaryTable (plotDictionary, outfile, hdataName=None)
def	matchAOs (aoList1, aoList2, strategy=0, recursive=True)
def	markupYODAwithCorrelationsFromYODAs (reference, correlationsDir)
def	markupYODAwithCorrelationsFromYAML (reference, correlationsDir)
def	markupYODAwithCorrelations (ref, corrDir, ignoreCorrelations)
def	returnFileContents (filename, inputType='data')
def	getCorrelationInfoFromWeb (name=None, dlformat="yoda")

Variables
	parser = optparse.OptionParser(usage="%prog [options]")
	help
	dest
	default
	False
	action
	opts
	args
dictionary	histograms
	ntoys
	verbosity
	filter
	ignore_corrs_data
	ignore_corrs_mc
dictionary	mcNames = {}
dictionary	plotDictionary = {'error-analysis': {}, 'covariance-matrix': {}, 'correlation-matrix': {}, 'data-vs-mc': {}, 'chi2-contribs': {}, 'chi2-value': {}, 'covDetails': {}, 'summary-plot': {}, 'summary-table': {}}
	data
string	outdir = "outputs/%s/data/plots" % (opts.data.replace(".yoda", ""))
	plotparser = rivet.mkStdPlotParser([], [])
	headers = plotparser.getHeaders(hdata.path())
	XLabel
	Title
	xLabel
	title
	mcName = mc.split(":")[1]
	mc = mc.split(":")[0]
def	mcnew = markupYODAwithCorrelations(mc, opts.corr_mc, opts.ignore_corrs_mc)
def	aoMap = matchAOs(histograms['data'], histograms['models'][mcnew])
def	newmc = renameAOs(mcnew, aoMap)
	dataSuperAO = ct.makeSuperAO(histograms['data'])
	mcSuperAO = ct.makeSuperAO(histograms['models'][mc])
dictionary	mcResults = {}
bool	passFilter = False
def	hmc = aoMap[hdata]
string	covDetailsData = "Size of uncertainties across range, and Data Correlation Matrix"
	covData = ct.makeCovarianceMatrixFromToys(hdata, opts.ntoys, opts.ignore_corrs_data)
string	covDetailsMC = "MC Covariance Matrix"
	covMC = ct.makeCovarianceMatrixFromToys(hmc, opts.ntoys, opts.ignore_corrs_mc)
	covTotal = covData + covMC
	chi2
	ndf
	prob
	chi2contribs
	chi2ContribsByRow = ct.chi2ContribsByRow(chi2contribs)
	chi2ContribsByRowYAML = yaml.dump(chi2ContribsByRow, default_flow_style=True, default_style='', width=1e6)
dictionary	res = {'%s' % opts.data: '[Data]', '%s' % model: '[%s (# chi^2=%.2f/%d)]' % (mcName, chi2, ndf)}
string	outdirplots = outdir + "/data-vs-%s/plots/" % mcName
	plots = st.makeSystematicsPlotsWithROOT(res, outdirplots, nominalName='Data', ratioZoom=None, regexFilter=".*%s.*" % hmc.name(), regexVeto=None)
	pathName = hdata.path().replace("/REF", "")
dictionary	mcR = mcResults[model]
def	beamerPath = makeBeamer(plotDictionary, outdir)

Function Documentation

convertHistToScatter()

def covarianceTool.convertHistToScatter

(

ao

)

Definition at line 69 of file covarianceTool.py.

def convertHistToScatter(ao):
  print("convertHistToScatter: This is a placeholder, need to implement")
  exit(1)
 
 

getCorrelationInfoFromWeb()

def covarianceTool.getCorrelationInfoFromWeb	(		name = None,
			dlformat = "yoda"
	)

Definition at line 479 of file covarianceTool.py.

def getCorrelationInfoFromWeb(name=None, dlformat="yoda"):
  # try to get the YAML files for the corresponding HEPData record
  # download and untar them...
  if (name is None): name = opts.data
  inspireID = [t for t in name.split("_") if "I" in t]
  inspireID = inspireID[0].replace("I", "")
  hdurl = "http://www.hepdata.net/record/ins%s?format=%s" % (inspireID, dlformat)
  print("[INFO] Trying to download information from %s", hdurl)
  if hdurl:
    response = urllib2.urlopen(hdurl)
    download = response.read()
    if not download or "<html" in download:
        print(("Problem encountered when getting data from HepData (%s). No reference data file written." % hdurl))
    else:
        tar = tarfile.open(mode="r:gz", fileobj=StringIO.StringIO(download))
        fnames = tar.getnames()
        if len(fnames) > 0 and dlformat == 'yaml':
            tar.extractall()
            os.system("mkdir -p corrInfo/ins%s" % (inspireID))
            os.system("mv %s/* corrInfo/ins%s/." % (fnames[0], inspireID))
            os.system("rm -r %s" % (fnames[0]))
            print("[INFO] successfully downloaded YAML files from HEPMC, will see if they can be used for correlations")
        if len(fnames) == 1 and dlformat == 'yoda':
            tar.extractall()
            os.system("mkdir -p corrInfo/ins%s" % (inspireID))
            os.system("mv %s corrInfo/ins%s/." % (fnames[0], inspireID))
            print("[INFO] successfully downloaded YODA files from HEPMC, will see if they can be used for correlations")
    opts.corr_data = "corrInfo/ins%s" % inspireID
    response.close()
 
 

isFloat()

def covarianceTool.isFloat

(

x

)

Definition at line 60 of file covarianceTool.py.

def isFloat(x):
  # check if the object can be cast as a float
  try:
    float(x)
    return True
  except ValueError:
    return False
 
 

makeBeamer()

def covarianceTool.makeBeamer	(		plotDictionary,
			outdir
	)

Definition at line 93 of file covarianceTool.py.

def makeBeamer(plotDictionary, outdir):
  # produce summary slides using beamer
  frames = []
  dataDir = os.environ["SYSTTOOLSPATH"] + "/data/"
  os.system("mkdir -p %s " % outdir)
  samples = plotDictionary['chi2-value'].keys()
  histos = plotDictionary['chi2-value'][samples[0]].keys()
  os.system('cp %s/title.tex %s/title.tex' % (dataDir, outdir))
  os.system('cp %s/title.tex %s/title.tex' % (dataDir, outdir))
  os.system("sed -i -e 's|!TITLE!|%s|g' %s/title.tex" % (opts.data.replace('.yoda', '').replace('_', ' '), outdir))
  frames.append("%s/title.tex" % outdir)
  for histo in histos:
     if 'superAO' in histo: continue
     os.system('cp %s/section.tex %s/%s_section.tex' % (dataDir, outdir, histo))
     os.system("sed -i -e 's|!AOREF!|%s|g' %s/%s_section.tex" % (histo, outdir, histo))
     frames.append('%s/%s_section.tex' % (outdir, histo))
     os.system('cp %s/errorAnalysis.tex %s/%s_%s_errorAnalysis.tex' % (dataDir, outdir, 'data', histo))
     os.system("sed -i -e 's|!AOREF!|%s|g' %s/%s_%s_errorAnalysis.tex" % (histo, outdir, 'data', histo))
     os.system("sed -i -e 's|!SAMPLE!|%s|g' %s/%s_%s_errorAnalysis.tex" % ('data', outdir, 'data', histo))
     os.system("sed -i -e 's|!ERRORBREAKDOWN!|%s|g' %s/%s_%s_errorAnalysis.tex" % (plotDictionary['error-analysis']['data'][histo], outdir, 'data', histo))
     os.system("sed -i -e 's|!COVMATRIX!|%s|g' %s/%s_%s_errorAnalysis.tex" % (plotDictionary['correlation-matrix']['data'][histo], outdir, 'data', histo))
     os.system("sed -i -e 's|!COVDETAILS!|%s|g' %s/%s_%s_errorAnalysis.tex" % (plotDictionary['covDetails']['data'][histo], outdir, 'data', histo))
     frames.append("%s/%s_%s_errorAnalysis.tex" % (outdir, 'data', histo))
     for sample in samples:
       sampleNameClean = sample.replace("_", " ").replace("-", " ")
       os.system('cp %s/dataMCComparison.tex %s/%s_%s_dataMCComparison.tex' % (dataDir, outdir, sample, histo))
       os.system("sed -i -e 's|!AOREF!|%s|g' %s/%s_%s_dataMCComparison.tex" % (histo, outdir, sample, histo))
       os.system("sed -i -e 's|!SAMPLE!|%s|g' %s/%s_%s_dataMCComparison.tex" % (sampleNameClean, outdir, sample, histo))
       os.system("sed -i -e 's|!DATAVSMC!|%s|g' %s/%s_%s_dataMCComparison.tex" % (plotDictionary['data-vs-mc'][sample][histo], outdir, sample, histo))
       os.system("sed -i -e 's|!CHI2CONTRIBS!|%s|g' %s/%s_%s_dataMCComparison.tex" % (plotDictionary['chi2-contribs'][sample][histo], outdir, sample, histo))
       frames.append("%s/%s_%s_dataMCComparison.tex" % (outdir, sample, histo))
     # summary
     os.system('cp %s/sectionSummary.tex %s/%s_sectionSummary.tex' % (dataDir, outdir, histo))
     os.system("sed -i -e 's|!AOREF!|%s|g' %s/%s_sectionSummary.tex" % (histo, outdir, histo))
     os.system("sed -i -e 's|!SUMMARYPLOT!|%s|g' %s/%s_sectionSummary.tex" % (plotDictionary['summary-plot'][histo], outdir, histo))
     os.system("sed -i -e 's|!SUMMARYTABLE!|%s|g' %s/%s_sectionSummary.tex" % (plotDictionary['summary-table'][histo], outdir, histo))
     frames.append("%s/%s_sectionSummary.tex" % (outdir, histo))
  os.system('cp %s/section.tex %s/%s_section.tex' % (dataDir, outdir, 'overall'))
  os.system("sed -i -e 's|!AOREF!|%s|g' %s/%s_section.tex" % ('overall summary', outdir, 'overall'))
  frames.append('%s/%s_section.tex' % (outdir, 'overall'))
  os.system('cp %s/overallSummary.tex %s/%s_overallSummary.tex' % (dataDir, outdir, 'overall'))
  os.system("sed -i -e 's|!AOREF!|%s|g' %s/%s_overallSummary.tex" % ('overall summary', outdir, 'overall'))
  os.system("sed -i -e 's|!SUMMARYTABLE!|%s|g' %s/%s_overallSummary.tex" % (plotDictionary['summary-table']['all'], outdir, 'overall'))
  frames.append("%s/%s_overallSummary.tex" % (outdir, 'overall'))
  frames.append("%s/end.tex" % dataDir)
  catLine = " ".join(frames)
  os.system(" cat %s > %s/slides_%s.tex" % (catLine, outdir, opts.data.replace('.yoda', '')))
  return " %s/slides_%s.tex" % (outdir, opts.data.replace('.yoda', ''))
 
 

makeSummaryTable()

def covarianceTool.makeSummaryTable	(		plotDictionary,
			outfile,
			hdataName = None
	)

Definition at line 143 of file covarianceTool.py.

def makeSummaryTable(plotDictionary, outfile, hdataName=None):
   # make a summary latex table of the chi2 values for each AO
   outf = open(outfile, 'w')
   samples = plotDictionary['chi2-value'].keys()
   if (hdataName):
     histos = [hdataName]
   else:
     histos = sorted(plotDictionary['chi2-value'][samples[0]].keys())
 
   line = "\\begin{tabular}{|l|"
   for hist in histos:
    line += "c|"
   line += "}"
   outf.write(line + '\n')
   outf.write("\\hline " + '\n')
   line = " $\\chi^2$ / ndof "
   for hist in histos:
     if 'superAO' in hist: hist = 'global agreement'
     line += " & %s " % hist
   line += "\\\\"
   outf.write(line + '\n')
   outf.write("\\hline" + '\n')
   for sample in samples:
     line = " %s " % sample.replace("-", " ").replace("_", " ")
     for hist in histos:
       line += " & %s" % plotDictionary['chi2-value'][sample][hist]  
     line += "\\\\"
     outf.write(line + '\n')
   outf.write("\\hline" + '\n')
   outf.write("\\end{tabular}" + '\n')
   outf.close()
 
 

markupYODAwithCorrelations()

def covarianceTool.markupYODAwithCorrelations	(		ref,
			corrDir,
			ignoreCorrelations
	)

Definition at line 419 of file covarianceTool.py.

def markupYODAwithCorrelations(ref, corrDir, ignoreCorrelations):
  # produce a new YODA file where the AOs have been marked up with
  # covariance info from an uncertainty breakdown provided as
  # either YAML or YODA files
  isYoda = False
  if ignoreCorrelations:
     return markupYODAwithCorrelationsFromYODAs(ref, 'self')
  else:
    if (corrDir is None): return ref
    for f in os.listdir(corrDir):
      if '.yoda' in f: isYoda = True
  if isYoda:
     return markupYODAwithCorrelationsFromYODAs(ref, corrDir)
  else:
     return markupYODAwithCorrelationsFromYAML(ref, corrDir)
 
 

markupYODAwithCorrelationsFromYAML()

def covarianceTool.markupYODAwithCorrelationsFromYAML	(		reference,
			correlationsDir
	)

Definition at line 292 of file covarianceTool.py.

def markupYODAwithCorrelationsFromYAML(reference, correlationsDir):
  # produce a new YODA file where the AOs have been marked up with
  # covariance info from an uncertainty breakdown provided as
  # one or more YAML files from HEPData
  reference_out = reference.replace(".yoda", "_corrs.yoda")
  reference_outf = open(reference_out, 'w')
  infiles = {}
  for f in os.listdir(correlationsDir):
    if 'submission' in f: continue
    if 'comb' in f: continue
    if "syst" in f:
      infiles.setdefault(correlationsDir + "/" + f.replace('syst', 'xsect'), [])
    else:  
      infiles.setdefault(correlationsDir + "/" + f, [])
 
  for f in infiles.keys():
    stream = open(f, "r")
    doc = yaml.load(stream)
    binLabels = {}
    binValues = {}
    binErrors = {}
    for itemtype, itemarray in doc.items():
      if itemtype == "independent_variables":  # bin labels
        for line in itemarray:
          binLabels['name'] = itemarray[0]['header']['name']
          counter = 0
          for entry in itemarray[0]['values']:
            if ('low' in entry.keys() and 'high' in entry.keys()):
              binLabels[counter] = "%f - %f" % (entry['low'], entry['high'])
            else:
              binLabels[counter] = "%f" % (entry['value'])
            counter += 1
 
      if itemtype == "dependent_variables":  # values
        for line in itemarray:
          binValues['name'] = "%s [%s]" % (line['header']['name'], '')
          binErrors['name'] = "uncertainties"
        counter = 0
        for entry in line['values']:
           binValues[counter] = entry['value']
           binErrors[counter] = {}
           if 'errors' in entry.keys():
             for error in entry['errors']:
               binErrors[counter][error['label']] = {}
               if 'symerror' in error.keys():
                 if not isFloat(error['symerror']) and '%' in (error['symerror']):
                   binErrors[counter][error['label']]['up'] = binValues[counter] * 0.01 * float(error['symerror'].replace('%', ''))
                   binErrors[counter][error['label']]['dn'] = binValues[counter] * -0.01 * float(error['symerror'].replace('%', ''))
                 else:
                   binErrors[counter][error['label']]['up'] = float(error['symerror'])
                   binErrors[counter][error['label']]['dn'] = -1 * float(error['symerror'])
               elif 'asymerror' in error.keys():
                 if not isFloat(error['asymerror']['plus']) and '%' in error['asymerror']['plus']:
                   binErrors[counter][error['label']]['up'] = binValues[counter] * 0.01 * float(error['asymerror']['plus'].replace('%', ''))
                   binErrors[counter][error['label']]['dn'] = binValues[counter] * 0.01 * float(error['asymerror']['minus'].replace('%', ''))
                 else:
                   binErrors[counter][error['label']]['up'] = float(error['asymerror']['plus'])
                   binErrors[counter][error['label']]['dn'] = 1 * float(error['asymerror']['minus'])
               else:
                 print('[ERROR] errors are neither symmetric or asymmetric... exit!')
                 exit(1)
           counter += 1
 
    if 'xsect' in f:
      systNames = binErrors[0].keys()
      binErrorsGranular = {}
      systFile = f.replace("xsect", "syst")
      streamsyst = open(systFile, "r")
      docsyst = yaml.load(streamsyst)
      binErrorsGranular['name'] = "uncertainties"
      for itemtype, itemarray in docsyst.items():
        if "variables" not in itemtype: continue
        if (itemtype == "independent_variables"):
          systNames = [v['value'] for v in itemarray[0]['values']]
        else:
          binCounter = -1
          for line in itemarray:
            binCounter += 1
            binErrorsGranular[binCounter] = {}
            if 'values' not in line.keys(): continue
            systCounter = -1
            for error in line['values']:
              systCounter += 1
              binErrorsGranular[binCounter][systNames[systCounter]] = {}
              binErrorsGranular[binCounter][systNames[systCounter]]['up'] = float(error['value']) * binValues[binCounter] * 0.01
              binErrorsGranular[binCounter][systNames[systCounter]]['dn'] = -1 * float(error['value']) * binValues[binCounter] * 0.01
      for i in range(len(binErrorsGranular.keys()) - 1):
        binErrorsGranular[i]['stat'] = {}
        binErrorsGranular[i]['stat']['up'] = binErrors[i]['stat']['up']
        binErrorsGranular[i]['stat']['dn'] = binErrors[i]['stat']['dn']
      binErrors = binErrorsGranular
    nBins = len(binLabels) - 1
    infiles[f] = [binValues, binErrors]
 
  hists = yoda.read(reference, unpatterns=".*RAW.*")
  for name in hists:
    nearestMatch = ""
    smallestDiff = 999.
    nearestMatchErrors = ""
    nominal = [(hists[name].points()[i].y) for i in range(hists[name].numPoints())]
    for f, corrs in infiles.items():
      binValues = corrs[0]
      binErrors = corrs[1]
      nBins = len(binValues) - 1
      if (len(nominal) != nBins): continue
      totalDiff = 0
      for i in range(nBins):
        totalDiff += abs(binValues[i] - nominal[i])
      if (totalDiff < smallestDiff):
        nearestMatch = f
        smallestDiff = totalDiff
        nearestMatchErrors = binErrors
    if (opts.verbosity > 1): print("[DEBUG] candidate", nominal, " nearest match is ", nearestMatch)
    if smallestDiff < 1.0:
      if (opts.verbosity > 1): print("[INFO] mapping", name, " ---> ", nearestMatch, (smallestDiff))
      corrs = yaml.dump(nearestMatchErrors, default_flow_style=True, default_style='', width=1e6)
      hists[name].setAnnotation("ErrorBreakdown", corrs)
      yoda.writeYODA(hists[name], reference_outf)
    else:
      print("[WARNING] Warning, no match found for", name)
  print("[INFO] %s has been marked up with correlation info from %s in this file:" % (reference, correlationsDir))
  print("[INFO] ---> %s " % (reference_out))
  print("[INFO] next time, you can use this file as an input instead of ", reference)
  reference_outf.close()
  return reference_out
 
 

markupYODAwithCorrelationsFromYODAs()

def covarianceTool.markupYODAwithCorrelationsFromYODAs	(		reference,
			correlationsDir
	)

Definition at line 228 of file covarianceTool.py.

def markupYODAwithCorrelationsFromYODAs(reference, correlationsDir):
  # produce a new YODA file where the AOs have been marked up with
  # covariance info from an uncertainty breakdown provided as
  # a separate YODA file for each systematic variation
  reference_out = reference.replace(".yoda", "_corrs.yoda")
  reference_outf = open(reference_out, 'w')
  infiles = {}
  if correlationsDir == 'self':
    infiles.setdefault(reference, [])
  else:
    for f in os.listdir(correlationsDir):
      if "all" in f: continue
      infiles.setdefault(correlationsDir + "/" + f, [])
  nBins = -1
  binLabels = {}
  binValues = {}
  binErrors = {}
 
  histos = [name for name in yoda.read(reference, unpatterns=".*RAW.*")]
  for histname in histos:
    hAll = yoda.read(reference, unpatterns=".*RAW.*")[histname]
    if type(hAll) is yoda.core.Histo1D: hAll = hAll.mkScatter()
    for f in infiles.keys():
      systname = f.split("/")[-1].replace(".yoda", "")
      h = yoda.read(f, unpatterns=".*RAW.*")[histname]
      if type(h) is yoda.core.Scatter2D:
        nBins = h.numPoints()
        for ipt in range(nBins):
           binValues[ipt] = h.points()[ipt].y()
           binLabels[ipt] = "%f - %f" % (h.points()[ipt].xMin(), h.points()[ipt].xMax())
           errs = h.points()[ipt].yErrs()
           errAv = (abs(errs[1]) + abs(errs[0])) * 0.5
           binErrors.setdefault(ipt, {}).setdefault(systname, {})['up'] = errs[1]
           binErrors.setdefault(ipt, {}).setdefault(systname, {})['dn'] = -1 * errs[0]
      elif type(h) is yoda.core.Histo1D:
        nBins = h.numBins
        for ipt in range(nBins):
           binValues[ipt] = h.bins()[ipt].sumW()
           binLabels[ipt] = "%f - %f" % (h.bins()[ipt].xMin(), h.bins()[ipt].xMax())
           errAv = h.bins()[ipt].sumW2()
           binErrors.setdefault(ipt, {}).setdefault(systname, {})['up'] = errAv
           binErrors.setdefault(ipt, {}).setdefault(systname, {})['dn'] = -1 * errAv
      else: continue
    if (reference != ""):
      smallestDiff = 999.
      for f in reference.split(","):
        hists = yoda.read(f, unpatterns=".*RAW.*")
        for name in hists:
            if not type(hists[name]) is yoda.core.Scatter2D: continue
            nominal = [(hists[name].points()[i].y()) for i in range(hists[name].numPoints())]
            if (len(nominal) != nBins): continue
            totalDiff = 0
            for i in range(nBins):
              totalDiff += abs(binValues[i] - nominal[i])
            if (totalDiff < smallestDiff):
              smallestDiff = totalDiff
 
    corrs = yaml.dump(binErrors, default_flow_style=True, default_style='', width=1e6)
    hAll.setAnnotation("ErrorBreakdown", corrs)
    yoda.writeYODA(hAll, reference_outf)
  reference_outf.close()
  return reference_out
 
 

matchAOs()

def covarianceTool.matchAOs	(		aoList1,
			aoList2,
			strategy = 0,
			recursive = True
	)

Definition at line 176 of file covarianceTool.py.

def matchAOs(aoList1, aoList2, strategy=0, recursive=True):
  # try to establish a 1-1 mapping between AOs in two YODA files
  # eg in case of different naming conventions
  aoMap = {}
  for ao1 in aoList1:
   aoMap.setdefault(ao1, [])
   for ao2 in aoList2:
     nBinsMatch = False
     namesMatch = False
     namesPartialMatch = False
     binWidthsMatch = False
     if (ao2.name() in ao1.name()): namesMatch = True
     if (ao1.name() in ao2.name()): namesMatch = True
     ao1NameTokens = ao1.name().split("/")[-1].split("-")
     ao2NameTokens = ao2.name().split("/")[-1].split("-")
     if len(list(set(ao1NameTokens).intersection(ao2NameTokens))) >= 2: namesPartialMatch = True
     if (ao1.numPoints() == ao2.numPoints()):
       nBinsMatch = True
       for ipt in range(ao1.numPoints()):
         ao1BinLabels = "%.2f - %.2f" % (ao1.points()[ipt].xMin(), ao1.points()[ipt].xMax())
         ao2BinLabels = "%.2f - %.2f" % (ao2.points()[ipt].xMin(), ao2.points()[ipt].xMax())
         if (ao1BinLabels == ao2BinLabels): binWidthsMatch = True
     if strategy == 0:
       if (namesMatch and nBinsMatch and binWidthsMatch):
         aoMap.setdefault(ao1, []).append(ao2)
     elif strategy == 1:
       if (nBinsMatch and binWidthsMatch):
         aoMap.setdefault(ao1, []).append(ao2)
     elif strategy == 2:
       if (nBinsMatch and binWidthsMatch and namesPartialMatch):
         aoMap.setdefault(ao1, []).append(ao2)
 
  if (len(aoMap.keys()) == len(aoList1)) and ([len(aoMap[ao]) for ao in aoList1]) == ([1 for ao in aoList1]):
    if (opts.verbosity > 0): print("[INFO] found 1-1 mapping between aoLists using strategy %d:" % strategy)
    for ao1 in aoList1:
        aoMap[ao1] = aoMap[ao1][0]
        if (opts.verbosity > 1): print("%s --> %s" % (ao1.name(), aoMap[ao1].name()))
  elif (recursive and strategy < 3):
    if (opts.verbosity > 0):
      print("[WARNING] Could not establish 1-1 mapping between aoLists using strategy", strategy, ", try strategy ", strategy + 1)
    for ao1 in aoList1:
        if (opts.verbosity > 1): print("%s --> %s" % (ao1.name(), aoMap[ao1]))
    strategy += 1
    aoMap = matchAOs(aoList1, aoList2, strategy)
  else:
    print("[ERROR] could not match AOS in the Data and MC files. Please make sure the AOs have the same binning/ number of bins (and name if possible!)")
    print("[ERROR] aos from list 1", aoList1)
    print("[ERROR] aos from list 2", aoList2)
    exit(1)
  return aoMap
 
 

renameAOs()

def covarianceTool.renameAOs	(		fin_name,
			aoMap
	)

Definition at line 74 of file covarianceTool.py.

def renameAOs(fin_name, aoMap):
  # make a new YODA file where the AOs are renamed as per the reference file
  fout_name = fin_name.replace(".yoda", ".renamed.yoda")
  fout = open(fout_name, 'w')
  aos = yoda.read(fin_name, unpatterns=".*RAW.*")
  for aopath, ao in aos.items():
    foundMatch = False
    for ao1, ao2 in aoMap.items():
      if ao1.name() == ao.name():
        ao.setAnnotation("Path", ao2.path().replace("REF/", ""))
        foundMatch = True
      if ao2.name() == ao.name():
        ao.setAnnotation("Path", ao1.path().replace("REF/", ""))
        foundMatch = True
    if foundMatch: yoda.writeYODA(ao, fout)
  fout.close()
  return fout_name
 
 

returnFileContents()

def covarianceTool.returnFileContents	(		filename,
			inputType = 'data'
	)

Definition at line 436 of file covarianceTool.py.

def returnFileContents(filename, inputType='data'):
    result = []
    aos = yoda.read(filename, unpatterns=".*RAW.*")
    for aopath, ao in aos.items():
      aoType = type(ao)
      if aoType in [yoda.Histo1D, yoda.Histo2D]: ao = ao.mkScatter()
      if aoType in [yoda.Counter]: continue
      if 'all' not in opts.filter and inputType == 'data':
        for f in opts.filter:
          if f in aopath:
            result.append(ao)
      else:
        result.append(ao)
 
    covInfoMissing = []
    for ao in result:
      corr = ao.annotation("ErrorBreakdown")
      if (corr is None):
        covInfoMissing.append(ao)
    if (len(covInfoMissing) and inputType == 'data' and not opts.ignore_corrs and not opts.corr_data):
      print("[WARNING] These DATA analysis objects are missing the correlation/covariance information:", covInfoMissing)
      print("  --> To add correlation information, you can use the eg options: --get_corr_from_web, --corr_data ")
      print("  --> You can also proceed by adding the --ignore_corrs_data in which case the chi2 is calculated")
      print("  --> from the total uncertainties and assumed to be bin-bin uncorrelated.")
      inspireID = [t for t in opts.data.split("_") if "I" in t]
      inspireID = inspireID[0].replace("I", "")
      if os.path().isdir("corrInfo/ins%s" % inspireID):
        print("[INFO] Attempting to use the information in corrInfo/ins%s to build correlation info " % inspireID)
        opts.corr_data = "corrInfo/ins%s" % inspireID
      else:
        getCorrelationInfoFromWeb()
 
    if (len(covInfoMissing) and inputType == 'mc' and not opts.ignore_corrs and not opts.corr_mc):
      print("[WARNING] These MC analysis objects are missing the correlation/covariance information:", covInfoMissing)
      print("  --> To add correlation information, you can use the eg options: --corr_mc ")
      print("  --> You can also proceed by adding the --ignore_corrs_mc in which case the chi2 is calculated")
      print("  --> from the total uncertainties and assumed to be bin-bin uncorrelated.")
      if os.path.isdir("corrInfoMC/%s" % filename.replace(".yoda", "")):
        print("[INFO] Attempting to use the information in corrInfoMC/%s to build correlation info " % filename.replace(".yoda", ""))
        opts.corr_mc = "corrInfoMC/%s" % filename.replace(".yoda", "")
    return result
 
 

Variable Documentation

action

covarianceTool.action

Definition at line 34 of file covarianceTool.py.

aoMap

def covarianceTool.aoMap = matchAOs(histograms['data'], histograms['models'][mcnew])

Definition at line 564 of file covarianceTool.py.

args

covarianceTool.args

Definition at line 49 of file covarianceTool.py.

beamerPath

def covarianceTool.beamerPath = makeBeamer(plotDictionary, outdir)

Definition at line 730 of file covarianceTool.py.

chi2

covarianceTool.chi2

Definition at line 678 of file covarianceTool.py.

chi2contribs

covarianceTool.chi2contribs

Definition at line 678 of file covarianceTool.py.

chi2ContribsByRow

covarianceTool.chi2ContribsByRow = ct.chi2ContribsByRow(chi2contribs)

Definition at line 683 of file covarianceTool.py.

chi2ContribsByRowYAML

covarianceTool.chi2ContribsByRowYAML = yaml.dump(chi2ContribsByRow, default_flow_style=True, default_style='', width=1e6)

Definition at line 684 of file covarianceTool.py.

covData

covarianceTool.covData = ct.makeCovarianceMatrixFromToys(hdata, opts.ntoys, opts.ignore_corrs_data)

Definition at line 621 of file covarianceTool.py.

covDetailsData

string covarianceTool.covDetailsData = "Size of uncertainties across range, and Data Correlation Matrix"

Definition at line 618 of file covarianceTool.py.

covDetailsMC

string covarianceTool.covDetailsMC = "MC Covariance Matrix"

Definition at line 638 of file covarianceTool.py.

covMC

covarianceTool.covMC = ct.makeCovarianceMatrixFromToys(hmc, opts.ntoys, opts.ignore_corrs_mc)

Definition at line 641 of file covarianceTool.py.

covTotal

covarianceTool.covTotal = covData + covMC

Definition at line 654 of file covarianceTool.py.

data

covarianceTool.data

Definition at line 529 of file covarianceTool.py.

dataSuperAO

covarianceTool.dataSuperAO = ct.makeSuperAO(histograms['data'])

Definition at line 585 of file covarianceTool.py.

default

covarianceTool.default

Definition at line 32 of file covarianceTool.py.

dest

covarianceTool.dest

Definition at line 32 of file covarianceTool.py.

False

covarianceTool.False

Definition at line 34 of file covarianceTool.py.

filter

covarianceTool.filter

Definition at line 514 of file covarianceTool.py.

headers

covarianceTool.headers = plotparser.getHeaders(hdata.path())

Definition at line 537 of file covarianceTool.py.

help

covarianceTool.help

Definition at line 32 of file covarianceTool.py.

histograms

dictionary covarianceTool.histograms

Initial value:

=  {
    'data': None,
    'covariance': {},
    'models': {},
}

Definition at line 53 of file covarianceTool.py.

hmc

def covarianceTool.hmc = aoMap[hdata]

Definition at line 610 of file covarianceTool.py.

ignore_corrs_data

covarianceTool.ignore_corrs_data

Definition at line 516 of file covarianceTool.py.

ignore_corrs_mc

covarianceTool.ignore_corrs_mc

Definition at line 517 of file covarianceTool.py.

mc

covarianceTool.mc = mc.split(":")[0]

Definition at line 554 of file covarianceTool.py.

mcName

dictionary covarianceTool.mcName = mc.split(":")[1]

Definition at line 553 of file covarianceTool.py.

mcNames

dictionary covarianceTool.mcNames = {}

Definition at line 519 of file covarianceTool.py.

mcnew

def covarianceTool.mcnew = markupYODAwithCorrelations(mc, opts.corr_mc, opts.ignore_corrs_mc)

Definition at line 558 of file covarianceTool.py.

mcR

dictionary covarianceTool.mcR = mcResults[model]

Definition at line 709 of file covarianceTool.py.

mcResults

dictionary covarianceTool.mcResults = {}

Definition at line 602 of file covarianceTool.py.

mcSuperAO

covarianceTool.mcSuperAO = ct.makeSuperAO(histograms['models'][mc])

Definition at line 589 of file covarianceTool.py.

ndf

covarianceTool.ndf

Definition at line 678 of file covarianceTool.py.

newmc

covarianceTool.newmc = renameAOs(mcnew, aoMap)

Definition at line 565 of file covarianceTool.py.

ntoys

covarianceTool.ntoys

Definition at line 512 of file covarianceTool.py.

opts

covarianceTool.opts

Definition at line 49 of file covarianceTool.py.

outdir

string covarianceTool.outdir = "outputs/%s/data/plots" % (opts.data.replace(".yoda", ""))

Definition at line 532 of file covarianceTool.py.

outdirplots

string covarianceTool.outdirplots = outdir + "/data-vs-%s/plots/" % mcName

Definition at line 697 of file covarianceTool.py.

parser

covarianceTool.parser = optparse.OptionParser(usage="%prog [options]")

Definition at line 31 of file covarianceTool.py.

passFilter

covarianceTool.passFilter = False

Definition at line 604 of file covarianceTool.py.

pathName

covarianceTool.pathName = hdata.path().replace("/REF", "")

Definition at line 704 of file covarianceTool.py.

plotDictionary

dictionary covarianceTool.plotDictionary = {'error-analysis': {}, 'covariance-matrix': {}, 'correlation-matrix': {}, 'data-vs-mc': {}, 'chi2-contribs': {}, 'chi2-value': {}, 'covDetails': {}, 'summary-plot': {}, 'summary-table': {}}

Definition at line 520 of file covarianceTool.py.

plotparser

covarianceTool.plotparser = rivet.mkStdPlotParser([], [])

Definition at line 536 of file covarianceTool.py.

plots

covarianceTool.plots = st.makeSystematicsPlotsWithROOT(res, outdirplots, nominalName='Data', ratioZoom=None, regexFilter=".*%s.*" % hmc.name(), regexVeto=None)

Definition at line 698 of file covarianceTool.py.

prob

covarianceTool.prob

Definition at line 678 of file covarianceTool.py.

res

dictionary covarianceTool.res = {'%s' % opts.data: '[Data]', '%s' % model: '[%s (# chi^2=%.2f/%d)]' % (mcName, chi2, ndf)}

Definition at line 696 of file covarianceTool.py.

Title

covarianceTool.Title

Definition at line 540 of file covarianceTool.py.

title

covarianceTool.title

Definition at line 542 of file covarianceTool.py.

verbosity

covarianceTool.verbosity

Definition at line 513 of file covarianceTool.py.

XLabel

covarianceTool.XLabel

Definition at line 538 of file covarianceTool.py.

xLabel

covarianceTool.xLabel

Definition at line 542 of file covarianceTool.py.