CscCalibQuery Namespace Reference

Functions
def	updateList (oldListPath, newList)
def	runCalib (calType, runNumber, workDir, castorCopyCmd)

Variables
int	numFilesToRun = 3
string	responsiblePerson = "youzhou@email.arizona.edu"
string	maillist = responsiblePerson
bool	CoolMergeByDefault = False
	calType = sys.argv[1]
	Main program#############################. More...
string	calibFileDir = '/castor/cern.ch/user/l/lampen/CalibRunTest/slope/'
string	oldListFilePath = '/afs/cern.ch/user/m/muoncali/CSC/run/pulserList.txt'
string	outputDir = '/afs/cern.ch/user/m/muoncali/w0/CSC/runs/pulser/pulser'
	calibRe = re.compile('data1.*_calib.*calibration_pedCSC\\.daq\\.RAW.*\\.data')
	testFile = glob.glob("/afs/cern.ch/user/m/muoncali/CSC/run/runningCalibration*")
	currentLs = os.popen('rfdir ' + calibFileDir).read()
	inFile = open(oldListFilePath, 'r')
	oldLs = inFile.read()
	now = datetime.datetime.now()
	today = now.day
	currentDirList = currentLs.split('\n')
	oldDirList = oldLs.split('\n')
bool	updateRunList = True
string	runningDir = ""
	splitDir = Dir.split()
	day = int(splitDir[-3])
	DirName = splitDir[-1]
	timediff = today - day
string	cmd = 'rfdir ' + calibFileDir + DirName
	fileList = os.popen(cmd).read().split('\n')
int	nFiles = len(fileList) -1
	runNumber = DirName
string	outputDirFull = outputDir + 'Run_' + runNumber
bool	ThisCalibDir = False
string	castorCopyCmd = "mkdir ${bytestreamDir}"
tuple	fileName = (file.split(' '))[-1]
string	fullPath = calibFileDir + DirName + '/' + fileName

Function Documentation

runCalib()

def CscCalibQuery.runCalib	(		calType,
			runNumber,
			workDir,
			castorCopyCmd
	)

Definition at line 47 of file CscCalibQuery.py.

def runCalib(calType, runNumber,workDir,castorCopyCmd):
  #print ('running calib')
  #initialize based on calibration type
  scriptDir = '${HOME}/CSC/run/'
  if calType == 'pulser':
    calibScript =  scriptDir + 'CscCalcSlopeMon.py'
    dbScript = scriptDir + 'cscWritePSlopeCool.py'
    webDir = '${HOME}/www/csc/pulser'
    runListFile = "pulserRunList.pickle"
    #extractScript = scriptDir +'CscExtractPulser.py'
  elif( calType == 'ped'):
    calibScript = scriptDir + 'CscCalcPedMon.py'
    dbScript = scriptDir + 'cscWritePedRefCool.py'
    onlDbFile = scriptDir + 'online.cal'
    webDir = '${HOME}/www/csc/ped'
    webPageUrl = 'https://atlas-csc-calib.web.cern.ch/atlas-csc-calib/ped/pedRun_' +runNumber
    runListFile = "pedRunList.pickle"
    #extractScript = scriptDir + 'CscExtractPed.py'
 
  outputDir = workDir +"/CalibResults"
  bsDir = workDir +"/Bytestream"
 
  print ('outputDir = ' + outputDir)
 
 
  #Setup finished email message
  emailMessage = 'Finished ' + calType + ' calibration on run number ' + runNumber
  emailMessage += '. Output in ' + outputDir + '.'
  emailMessage += '\\nAnd website at:\\n'
  emailMessage += webPageUrl
  
  #Setup finished email subjects
  goodEmailSubject = '[CSC CALIB PROC]: SUCCESS with ' + calType + 'calib run' + runNumber
  badEmailSubject = '[CSC CALIB PROC]: PROBLEMS with ' + calType + 'calib run' + runNumber
 
  #Prepare bash script for batch system
  bashFilePath = workDir+'/CscCalib_' + calType + '_' + runNumber + '.sh'
 
  bsubCmd ='cd ' + outputDir + ';bsub -q 2nd -R "type==SLC5_64&&mem>420" ' + bashFilePath
 
  bashFileContents = "#!/bin/bash\n"
  bashFileContents += "#To resubmit this job, submit it to the atlasmuonqueu like so:\n#"
  bashFileContents += bsubCmd +"\n"
  bashFileContents += "source ~/CSC/CscSetup.sh\n"
  bashFileContents += "\n"
  bashFileContents += "resultDir=\"" + outputDir + "\"\n"
  bashFileContents += 'bytestreamDir="' + bsDir + '"\n'
  bashFileContents += 'maillist="' + maillist + '"\n'
  bashFileContents += 'webSiteDir="' + webDir + '"\n'
 
  calFilePrefix= "${resultDir}/" + runNumber
  inputPattern = "${bytestreamDir}/*.data"
 
  calibCommand = 'echo "Running calibration"\n' \
      + 'mkdir ${resultDir}\n' \
      + 'athena.py -c "outputPre=\'' + calFilePrefix \
      + '\';inputOnlCalibFile=\'' +onlDbFile \
      + '\';inputPat=\'' + inputPattern \
      + '\';reportPrefix=\'' + emailMessage \
      + '\';" ' \
      +  calibScript
 
 
  goodEmailCommand = ' mail -s "' + goodEmailSubject + '" $maillist < ' + calFilePrefix + "CalibReport.txt"
  badEmailCommand = ' mail -s "' + badEmailSubject + '" $maillist < ' + calFilePrefix + "CalibReport.txt"
  
  #For reference tag, we actually want the IOV to start just after the LAST run number
  #Get old run numbers
  infile = open(runListFile,"rb")
  runList = pickle.load(infile)
  runList.sort()
  print ("got runs")
  print (runList)
  infile.close()
 
 
  if(runNumber in runList):
    print ("Mailing message")
    message =["mail","-s",\
        '"New castor run directory found for previously processed run ' + str(runNumber) + '"',\
        responsiblePerson,\
        "<",\
        "runAlreadyProcessed.mail"]
    print (message)
    subprocess.call(message)
    sys.exit()
 
 
  highestRun = runList[-1]
 
  isRunNumberConflict = False
 
  
  if(highestRun > runNumber):
    #Something odd happening. The Input run number is lower than the last run number
    #this script (thinks it) processed
    #Notify someone important, and don't add to cool when done...
    subprocess.call(["mail","-s",\
        "[CSC CALIB PROC] Wrong run number ordering on run " + str(runNumber) \
        + "! Human intervension required!",\
        responsiblePerson,\
        "<",\
        "runNumberConflict.mail"]\
        )
    isRunNumberConflict = True
  else:
    #No problem, update run list
    runList += [runNumber]
    outfile = open(runListFile,"wb")
    pickle.dump(runList,outfile)
    outfile.close()
 
    
  #Label that we're working, so that other instances of CscCalibQuery won't run
  subprocess.call(['touch','/afs/cern.ch/user/m/muoncali/CSC/run/runningCalibration' + runNumber])
 
  #Command to create .db file
  DbCommand = 'athena.py -c "input=\'' + calFilePrefix+'.cal\';output=\'' \
      + calFilePrefix + '.db\';IOVRunStart=int(\'' + highestRun + '\')" ' + dbScript
 
  UploadCommand = ''
  #Noexec prevents actual execution of cool 
  #UploadCommand = '/afs/cern.ch/user/a/atlcond/utils/AtlCoolMerge.py ' \
  #    + "--batch --noexec --comment=\'Automated UPD1 update from " + calType  \
  #    + ' run ' + runNumber +'\' ' \
  #    + calFilePrefix + '.db'   \
  #    + ' COMP200 ATONR_COOL ATLAS_COOLONL_CSC_W PASSWORD '
  #UploadCommand += "\n"
 
 
 
 
 
  #Command to upload .db file to database
  UploadCommand += '/afs/cern.ch/user/a/atlcond/utils/AtlCoolMerge.py ' \
      + "--batch "
  if(not CoolMergeByDefault or isRunNumberConflict):
    UploadCommand += " --noexec "
    
  UploadCommand += "--comment=\'Automated reference update from " + calType  \
    + ' run ' + runNumber + ' to IOV starting at ' + highestRun + "' " \
    + calFilePrefix + '.db'   \
    + ' COMP200 ATLAS_COOLWRITE ATLAS_COOLOFL_CSC_W WCOOLOFL4CSC17 '
 
  WebSiteCommand = '\nfs sa ${resultDir} webserver:afs read\n'
  WebSiteCommand += 'cd $resultDir\n'
  WebSiteCommand += 'ln -s ${resultDir} ${webSiteDir}/pedRun_' + runNumber + '\n'
  WebSiteCommand += 'MakeCscPedSite.exe ' + runNumber + '.root ' + runNumber + '.cal_online\n'
  
  t1 = '\t'
 
  #Run calibration. If no problems detected, go ahead and upload.
  bashFileContents += '\ncd ' + workDir + '\n' \
      + "#Copying files from castor#\n" \
      + castorCopyCmd \
      + '\n'\
      + "#Running calibration (and calib monitoring)\n"\
      + "#, resulting in .cal file and status report\n"\
      + calibCommand +'\n' \
      + "\n"\
      +  "#Athena job to transform *.cal file to *.db file.\n"\
      +  DbCommand + '\n' \
      +  "#Python utility to upload *.db file to database. When entering by\n"\
      +  "#hand, I recomend removing '--batch' flag so you can check puposed\n"\
      +  "#operations before submision to database\n"\
      +  UploadCommand + '\n' \
      + '#Check if AllCalibMonGood file was created, which means that\n'\
      + '#this run passed acceptable criteria in the calibration monitoring\n'\
      + 'if [ -a AllCalibMonGood ]; then\n' \
      + t1 + "#Email list that the calibration looks good\n"\
      + t1 + goodEmailCommand +'\n' \
      + t1 + "################################################################\n"\
      + t1 + "#Execute next two commands if you want to submit database entry#\n"\
      + t1 + "#Useful if these steps were skipped due to suspicious behaviour#\n"\
      + t1 + "#during calibration.                                           #\n"\
      + t1 + "###############################################################\n"\
      + t1 + "\n"\
      + t1 + '\n'\
      + 'else\n' \
      + t1 + "#Suspicious behaviour in calibration. Notify mail list of this fact\n"\
      + t1 + badEmailCommand + '\n' \
      + 'fi\n'\
      + '\n'\
      + '#Always create website'\
      + WebSiteCommand + '\n' \
      + 'rm -rf $bytestreamDir\n' \
      + 'rm -rf ' + scriptDir +"runningCalibration" + runNumber +'\n'
 
  #Write bashfile
  print ("Printing bash file to: " +bashFilePath)
  bashFile = open(bashFilePath, 'w')
  bashFile.write(bashFileContents)
  bashFile.close()
 
  #Submit script
  os.system('chmod +x ' + bashFilePath)
  bsubsMessage = os.popen(bsubCmd).read()
 
  #Send alert email
  emailMessage = 'Starting ' + calType + ' calibration on run number ' + runNumber
  emailSubject = '[CSC CALIB PROC]: ' +  emailMessage
  emailMessage += '\nbsubs output:\n' + bsubsMessage
  os.system('echo "' + emailMessage + '" | mail -s "' + emailSubject + '" ' + maillist)
 

updateList()

def CscCalibQuery.updateList	(		oldListPath,
			newList
	)

Definition at line 37 of file CscCalibQuery.py.

def updateList(oldListPath,newList):
  print ('updating file list')
  #update the old file list
  outFile = open(oldListPath, 'w')
  outFile.write(newList)
  outFile.close()
 
#runs calibration on input file.
#It creates a bash script which can setup the job, run it, and do post job
#processing. The bash script is submitted to lxbatch

Variable Documentation

calibFileDir

string CscCalibQuery.calibFileDir = '/castor/cern.ch/user/l/lampen/CalibRunTest/slope/'

Definition at line 259 of file CscCalibQuery.py.

calibRe

CscCalibQuery.calibRe = re.compile('data1.*_calib.*calibration_pedCSC\\.daq\\.RAW.*\\.data')

Definition at line 266 of file CscCalibQuery.py.

calType

CscCalibQuery.calType = sys.argv[1]

Main program#############################.

Definition at line 257 of file CscCalibQuery.py.

castorCopyCmd

string CscCalibQuery.castorCopyCmd = "mkdir ${bytestreamDir}"

Definition at line 346 of file CscCalibQuery.py.

cmd

string CscCalibQuery.cmd = 'rfdir ' + calibFileDir + DirName

Definition at line 329 of file CscCalibQuery.py.

CoolMergeByDefault

bool CscCalibQuery.CoolMergeByDefault = False

Definition at line 32 of file CscCalibQuery.py.

currentDirList

CscCalibQuery.currentDirList = currentLs.split('\n')

Definition at line 301 of file CscCalibQuery.py.

currentLs

CscCalibQuery.currentLs = os.popen('rfdir ' + calibFileDir).read()

Definition at line 282 of file CscCalibQuery.py.

day

CscCalibQuery.day = int(splitDir[-3])

Definition at line 312 of file CscCalibQuery.py.

DirName

CscCalibQuery.DirName = splitDir[-1]

Definition at line 313 of file CscCalibQuery.py.

fileList

CscCalibQuery.fileList = os.popen(cmd).read().split('\n')

Definition at line 330 of file CscCalibQuery.py.

fileName

tuple CscCalibQuery.fileName = (file.split(' '))[-1]

Definition at line 348 of file CscCalibQuery.py.

fullPath

string CscCalibQuery.fullPath = calibFileDir + DirName + '/' + fileName

Definition at line 360 of file CscCalibQuery.py.

inFile

CscCalibQuery.inFile = open(oldListFilePath, 'r')

Definition at line 287 of file CscCalibQuery.py.

maillist

string CscCalibQuery.maillist = responsiblePerson

Definition at line 30 of file CscCalibQuery.py.

nFiles

int CscCalibQuery.nFiles = len(fileList) -1

Definition at line 332 of file CscCalibQuery.py.

now

CscCalibQuery.now = datetime.datetime.now()

Definition at line 295 of file CscCalibQuery.py.

numFilesToRun

int CscCalibQuery.numFilesToRun = 3

Definition at line 28 of file CscCalibQuery.py.

oldDirList

CscCalibQuery.oldDirList = oldLs.split('\n')

Definition at line 302 of file CscCalibQuery.py.

oldListFilePath

string CscCalibQuery.oldListFilePath = '/afs/cern.ch/user/m/muoncali/CSC/run/pulserList.txt'

Definition at line 260 of file CscCalibQuery.py.

oldLs

CscCalibQuery.oldLs = inFile.read()

Definition at line 288 of file CscCalibQuery.py.

outputDir

string CscCalibQuery.outputDir = '/afs/cern.ch/user/m/muoncali/w0/CSC/runs/pulser/pulser'

Definition at line 261 of file CscCalibQuery.py.

outputDirFull

string CscCalibQuery.outputDirFull = outputDir + 'Run_' + runNumber

Definition at line 338 of file CscCalibQuery.py.

responsiblePerson

string CscCalibQuery.responsiblePerson = "youzhou@email.arizona.edu"

Definition at line 29 of file CscCalibQuery.py.

runningDir

CscCalibQuery.runningDir = ""

Definition at line 305 of file CscCalibQuery.py.

runNumber

CscCalibQuery.runNumber = DirName

Definition at line 335 of file CscCalibQuery.py.

splitDir

CscCalibQuery.splitDir = Dir.split()

Definition at line 311 of file CscCalibQuery.py.

testFile

CscCalibQuery.testFile = glob.glob("/afs/cern.ch/user/m/muoncali/CSC/run/runningCalibration*")

Definition at line 274 of file CscCalibQuery.py.

ThisCalibDir

CscCalibQuery.ThisCalibDir = False

Definition at line 345 of file CscCalibQuery.py.

timediff

CscCalibQuery.timediff = today - day

Definition at line 316 of file CscCalibQuery.py.

today

CscCalibQuery.today = now.day

Definition at line 296 of file CscCalibQuery.py.

updateRunList

bool CscCalibQuery.updateRunList = True

Definition at line 304 of file CscCalibQuery.py.