TRTCalib_last_Skeleton.py

Go to the documentation of this file.

# Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
 
import sys, os, glob, subprocess, tarfile, fnmatch, smtplib
 
def nextstep(text):
    print("\n"+"#"*100)
    print("#")
    print("#    %s" % (text))
    print("#")
    print("#"*100,"\n")
    
def tryError(command, error):
    try:
        print(" Running: %s\n" % (command))
        stdout, stderr = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
        print("OUTPUT: \n%s" % (stdout.decode('ascii')))
        print("ERRORS: %s" % ("NONE" if stderr.decode('ascii')=='' else "\n"+stderr.decode('ascii')))
        if stderr:
            exit(1)        
    except OSError as e:
        print(error,e)
        sys.exit(e.errno)       
        
    
def send_statusmail(itera, runNumber, mto, outdir, a, b, c, d, e, f, g, h) :
    mserver = 'cernmx.cern.ch'
    mfrom   = 'no-reply@cern.ch'
    msubject = "TRT CALIB TIER0 - Exit Status for Run %d" % (runNumber)
 
    # assemble mail body
    mbody  = " Calibration job finished for run %d, iteration: %s \n\n" % (runNumber, itera)
    mbody += "   Here are the residuals obtained at detector level with currently used constants: \n"
    mbody += "   Residual-Barrel A: %s, Time-Residual-Barrel A: %s \n" % (a, b)
    mbody += "   Residual-Barrel C: %s, Time-Residual-Barrel C: %s \n" % (c, d)
    mbody += "   Residual-Endcap A: %s, Time-Residual-Endcap A: %s \n" % (e, f)
    mbody += "   Residual-Endcap C: %s, Time-Residual-Endcap C: %s \n\n" % (g, h)      
    mbody += " Expect Residual 140 +- 10mu in the barrel (Ar) and 130 +- 10mu in EC (Xe). Expect Time-Residual 0.0 +- 0.5ns.\n\n" 
    mbody += " Please check the histograms of the current run, to decide whether to upload new constants.\n" 
    mbody += " Histograms can be found on AFS, directory %s \n\n" % (outdir)
    
    print("Email body:\n\n",mbody)
    print("Email sent to:")
    
    try :
        con = smtplib.SMTP(mserver)
        if isinstance(mto, str) :
            print("\t- %s"% (mto))
            con.sendmail(mfrom, mto, 'Subject:' + str(msubject) + '\n\n' + str(mbody))
        elif isinstance(mto, list) :
            for onemto in mto :
                print("\t- %s"% (onemto))
                con.sendmail(mfrom, onemto, 'Subject:' + str(msubject) + '\n\n' + str(mbody))
        con.quit()
    except OSError as e:
        print("ERROR: Failed sending email notification\n",e)
        exit(e.errno)
 
 
def fromRunArgs(runArgs):
    
    
    nextstep("UNTAR files")
    
    
    print("Uncompressing files:")
    try:
        for file in runArgs.inputTARFile:
            print("\t-",file)
            tarfile.open(file).extractall(".") 
    except OSError as e:
        print("ERROR: Failed uncompressing TAR file\n",e)
        sys.exit(e.errno)    
    
    
    nextstep("Filtering files")
    
    
    # RT files (nor merging barrel)
    files_list_rt =[item for item in glob.glob("*_rt.txt") if "barrel" not in item]
    files_list_t0 =[item for item in glob.glob("*_t0.txt") if "barrel" not in item]
    files_list_cal=[item for item in glob.glob("*calibout.root") if "barrel" not in item]
    # tracks
    files_list_trk=glob.glob("*tracktuple.root")
    # straws
    files_list_stw=glob.glob("*merged.straw.txt")
    # constants
    files_list_ctn=glob.glob("*calib_constants_out.txt")
    
    def listFiles(v,txt=""):
        print("Number of files for %s: %i" % (txt,len(v)))
        
        if not v:
            print("ERROR: list of files for %s is empty" % (txt) )
            sys.exit(1)
            
        for i in v:
            print("\t-%s: %s" % (txt,i))
            
    listFiles(files_list_rt ,"RT")
    listFiles(files_list_t0 ,"T0")
    listFiles(files_list_cal,"Cal")
    listFiles(files_list_trk,"Track")
    listFiles(files_list_stw,"Straws")
    listFiles(files_list_ctn,"Constants")
    
    
    nextstep("Merging Rt and t0 files")
    
    
    command  = "touch calibout.txt ;"  
    command += "  echo '# Fileformat=2' >> calibout.txt ; "
    command += "  echo '# RtRelation'   >> calibout.txt ; "
    
    # Merging Rt files
    for _f in files_list_rt :
        command += "cat %s >> calibout.txt ; " % (_f)
        
    command += "  echo '# StrawT0' >> calibout.txt ; "
    
    # Merging t0 files
    for _f in files_list_t0 :      
        command += "cat %s >> calibout.txt ; " % (_f)
 
    # add footer
    command += "  echo '#GLOBALOFFSET 0.0000' >> calibout.txt ; "
    
    if os.path.isfile("calibout.txt"):
        print("calibout.txt.. already exists. Removed.")
        os.remove("calibout.txt")
            
    tryError(command,"ERROR: Not able to merge the Rt and t0 files\n") 
      
    
    
    nextstep("Merging constants files")
    
 
    command = " touch oldt0s.txt ;"
    for _f in files_list_ctn :
        command += "cat %s >> oldt0s.txt ; " % (_f)    
 
    if os.path.isfile("oldt0s.txt"):
        print("oldt0s.txt.. already exists. Removed.")
        os.remove("oldt0s.txt")
 
    tryError(command,"ERROR: Not able to merge the Rt and t0 files\n")   
          
    
    
    nextstep("Shift text file")
    
    
    command = 'TRTCalib_cfilter.py calibout.txt calibout.txt oldt0s.txt shiftrt'  
      
    tryError(command,"ERROR: Failed in process TRTCalib_cfilter\n")    
 
    
    nextstep("Run hadd to merge root files")
    
    
    # Removing the output file if it exists to avoid problems
    if os.path.isfile("merge.root"):
        print("merge.root.. already exists. Removed.")
        os.remove("merge.root")  
        
    command  = 'hadd merge.root '
    # merge all the files
    for _f in files_list_cal :
        command += "%s " % (_f)     
    # and track tuple, we just need the first one since they are all the same
    command += "%s " % files_list_trk[0]       
    
    tryError(command,"ERROR: Failed in process hadd\n")         
    
    
    nextstep("Rename root files")
    
    
    outputFile = runArgs.outputTAR_MERGEDFile
    runNumber = int(runArgs.inputTARFile[0].split('.')[1])
    
    # Rename calibout.txt and merge.root
    command  = "mv -v calibout.txt %s.calibout.txt ; " % outputFile 
    command += "mv -v merge.root %s.merge.root ; " % outputFile 
    # only one straw file is needed
    command += "mv -v %s straws.%d.txt ; " % (files_list_stw[0], runNumber)
    # copy also db const
    command += "mv -v dbconst.txt %s.dbconst.txt" % outputFile    
    
    tryError(command,"ERROR: Failed in process hadd\n")  
 
    
    nextstep("Make all plots")
    
 
    # generating the .ps file
    command  = "TRTCalib_makeplots itersum %s.merge.root %s/lastconfigfile" % (outputFile, os.path.abspath('.'))
    
    tryError(command,"ERROR: Failed in creating plots (itersum.ps file)\n")  
 
    
    nextstep("converting ps to pdf")
    
    
    command  = "ps2pdf itersum.ps %s.itersum.pdf" % (outputFile)
    
    tryError(command,"ERROR: Failed in creating itersum.pdf from itersum.ps)\n")
 
    
    nextstep("Straw status Report")
    
    
    command  = "mkdir -p -v output/ ; TRTCalib_StrawStatusReport %d" % (runNumber)
    
    tryError(command,"ERROR: Failed running TRTCalib_StrawStatusReport.cxx\n")
 
    
    nextstep("Straw status plots (root macro)")
    
       
    from ROOT import PathResolver
    command  = "root -l -b -q %s" % (PathResolver.FindCalibFile("TRT_CalibAlgs/TRTCalib_StrawStatusReport.C"))
    
    tryError(command,"ERROR: Failed running root macro TRTCalib_StrawStatusReport.C\n")
 
    
    nextstep("TAR'ing files")
    
       
    try:
        # Getting list of files to be compressed
        files_list=glob.glob(outputFile+".*")
        # Compressing
        tar = tarfile.open(outputFile, "w:gz")
        print("\nCompressing files in %s output file:" % outputFile)
        for file in files_list:
            print("\t-",file)
            tar.add(file)
        tar.close()
    except OSError as e:
        print("ERROR: Failed compressing the output files\n",e)
        sys.exit(e.errno)   
 
 
    
    nextstep("Copying files to AFS Directory")
    
    
    outDIR = "%s/run_%d" % (runArgs.attrtcal_dir, runNumber)
    
    # Extracting iteration from Tier0 and for emails
    outputFile_split = outputFile.split('.')
    itera = '9999' # default iteration for testing only
    if len(outputFile_split) > 5:
        if "iter" in outputFile_split[5]:
            itera = (outputFile_split[5])[-1:]    # e.g. data14_cos.00247236.express_express.trtcal.TXT.iter1 --> '1'
    
    command  = "mkdir -p -v %s ; " % (outDIR)
    command += "cp -v %s.merge.root %s/trtcalib_0%s_histograms.root ; " % (outputFile, outDIR, itera)
    command += "cp -v %s.calibout.txt %s/calibout.%d.NoShifted.txt ; " % (outputFile, outDIR, runNumber)
    command += "cp -v %s.itersum.pdf %s/Plots.%d.pdf ; " % (outputFile, outDIR, runNumber)
    command += "cp -v %s.dbconst.txt %s/dbconst.%d.txt ; " % (outputFile, outDIR, runNumber)
    command += "cp -v %s.dbconst.txt %s/calibout.%d.txt ; " % (outputFile, outDIR, runNumber)
    command += "cp -v straws.%d.txt %s/straws.%d.txt; " % (runNumber, outDIR, runNumber)
    command += "cp -v TRT_StrawStatusReport.txt %s/TRT_StrawStatusReport.%d.txt; " % (outDIR, runNumber)
    command += "cp -v allPlots.pdf %s/TRT_StrawStatusReport.%d.pdf; " % (outDIR, runNumber)
    
    tryError(command,"ERROR: Files cannot be copied to the chosen directory\n")
    
    
    
    
    nextstep("Extractor information")
    
    
    command  = "python -m TRT_CalibAlgs.TRTCalib_Extractor %s/trtcalib_0%s_histograms.root > %s/extraction.txt" % (outDIR, itera, outDIR)
    
    tryError(command,"ERROR: Extracting information for email notification\n")
    
    
    nextstep("Email notification")
    
 
    res_ba  = 0 ; tres_ba = 0
    res_bc  = 0 ; tres_bc = 0
    res_ea  = 0 ; tres_ea = 0
    res_ec  = 0 ; tres_ec = 0
    
    try:
        with open("%s/extraction.txt" % (outDIR)) as exfile:
            for line in exfile :
                if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part 1*') :
                        res_ba = line.split()[6]  
                if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part 1*') :
                        tres_ba = line.split()[6]
                if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part -1*') :
                        res_bc = line.split()[6]
                if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part -1*') :
                        tres_bc = line.split()[6]
                if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part 2*') :
                        res_ea = line.split()[6]
                if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part 2*') :
                        tres_ea = line.split()[6]
                if fnmatch.fnmatch(line,'* res *') and fnmatch.fnmatch(line, '*part -2*') :
                        res_ec = line.split()[6]
                if fnmatch.fnmatch(line,'* tresmean *') and fnmatch.fnmatch(line, '*part -2*') :
                        tres_ec = line.split()[6]
    except OSError as e:
        print("ERROR: Failed reading %s/extraction.txt file\n" % (outDIR) ,e)
        sys.exit(e.errno)    
 
    if runArgs.sendNotification and runArgs.emailList :
        send_statusmail(itera, runNumber, runArgs.emailList, outDIR, res_ba, tres_ba, res_bc, tres_bc, res_ea, tres_ea, res_ec, tres_ec)
    else:
        print("INFO: No email notification sent since --sendNotification=%r or empty --emailList=" % (runArgs.sendNotification), runArgs.emailList)