ATLAS Offline Software
Loading...
Searching...
No Matches
Functions | Variables
CscCalibQuery Namespace Reference

Functions

 updateList (oldListPath, newList)
 runCalib (calType, runNumber, workDir, castorCopyCmd)

Variables

int numFilesToRun = 3
str responsiblePerson = "youzhou@email.arizona.edu"
str maillist = responsiblePerson
bool CoolMergeByDefault = False
 calType = sys.argv[1]
 Main program#############################.
str calibFileDir = '/castor/cern.ch/user/l/lampen/CalibRunTest/slope/'
str oldListFilePath = '/afs/cern.ch/user/m/muoncali/CSC/run/pulserList.txt'
str outputDir = '/afs/cern.ch/user/m/muoncali/w0/CSC/runs/pulser/pulser'
 calibRe = re.compile('data1.*_calib.*calibration_pedCSC\\.daq\\.RAW.*\\.data')
 testFile = glob.glob("/afs/cern.ch/user/m/muoncali/CSC/run/runningCalibration*")
 currentLs = os.popen('rfdir ' + calibFileDir).read()
 inFile = open(oldListFilePath, 'r')
 oldLs = inFile.read()
 now = datetime.datetime.now()
 today = now.day
 currentDirList = currentLs.split('\n')
 oldDirList = oldLs.split('\n')
bool updateRunList = True
str runningDir = ""
 splitDir = Dir.split()
 day = int(splitDir[-3])
 DirName = splitDir[-1]
 timediff = today - day
str cmd = 'rfdir ' + calibFileDir + DirName
 fileList = os.popen(cmd).read().split('\n')
int nFiles = len(fileList) -1
 runNumber = DirName
str outputDirFull = outputDir + 'Run_' + runNumber
bool ThisCalibDir = False
str castorCopyCmd = "mkdir ${bytestreamDir}"
tuple fileName = (file.split(' '))[-1]
str fullPath = calibFileDir + DirName + '/' + fileName

Function Documentation

◆ runCalib()

CscCalibQuery.runCalib ( calType,
runNumber,
workDir,
castorCopyCmd )

Definition at line 46 of file CscCalibQuery.py.

46def runCalib(calType, runNumber,workDir,castorCopyCmd):
47 #print ('running calib')
48 #initialize based on calibration type
49 scriptDir = '${HOME}/CSC/run/'
50 if calType == 'pulser':
51 calibScript = scriptDir + 'CscCalcSlopeMon.py'
52 dbScript = scriptDir + 'cscWritePSlopeCool.py'
53 webDir = '${HOME}/www/csc/pulser'
54 runListFile = "pulserRunList.pickle"
55 #extractScript = scriptDir +'CscExtractPulser.py'
56 elif( calType == 'ped'):
57 calibScript = scriptDir + 'CscCalcPedMon.py'
58 dbScript = scriptDir + 'cscWritePedRefCool.py'
59 onlDbFile = scriptDir + 'online.cal'
60 webDir = '${HOME}/www/csc/ped'
61 webPageUrl = 'https://atlas-csc-calib.web.cern.ch/atlas-csc-calib/ped/pedRun_' +runNumber
62 runListFile = "pedRunList.pickle"
63 #extractScript = scriptDir + 'CscExtractPed.py'
64
65 outputDir = workDir +"/CalibResults"
66 bsDir = workDir +"/Bytestream"
67
68 print ('outputDir = ' + outputDir)
69
70
71 #Setup finished email message
72 emailMessage = 'Finished ' + calType + ' calibration on run number ' + runNumber
73 emailMessage += '. Output in ' + outputDir + '.'
74 emailMessage += '\\nAnd website at:\\n'
75 emailMessage += webPageUrl
76
77 #Setup finished email subjects
78 goodEmailSubject = '[CSC CALIB PROC]: SUCCESS with ' + calType + 'calib run' + runNumber
79 badEmailSubject = '[CSC CALIB PROC]: PROBLEMS with ' + calType + 'calib run' + runNumber
80
81 #Prepare bash script for batch system
82 bashFilePath = workDir+'/CscCalib_' + calType + '_' + runNumber + '.sh'
83
84 bsubCmd ='cd ' + outputDir + ';bsub -q 2nd -R "type==SLC5_64&&mem>420" ' + bashFilePath
85
86 bashFileContents = "#!/bin/bash\n"
87 bashFileContents += "#To resubmit this job, submit it to the atlasmuonqueu like so:\n#"
88 bashFileContents += bsubCmd +"\n"
89 bashFileContents += "source ~/CSC/CscSetup.sh\n"
90 bashFileContents += "\n"
91 bashFileContents += "resultDir=\"" + outputDir + "\"\n"
92 bashFileContents += 'bytestreamDir="' + bsDir + '"\n'
93 bashFileContents += 'maillist="' + maillist + '"\n'
94 bashFileContents += 'webSiteDir="' + webDir + '"\n'
95
96 calFilePrefix= "${resultDir}/" + runNumber
97 inputPattern = "${bytestreamDir}/*.data"
98
99 calibCommand = 'echo "Running calibration"\n' \
100 + 'mkdir ${resultDir}\n' \
101 + 'athena.py -c "outputPre=\'' + calFilePrefix \
102 + '\';inputOnlCalibFile=\'' +onlDbFile \
103 + '\';inputPat=\'' + inputPattern \
104 + '\';reportPrefix=\'' + emailMessage \
105 + '\';" ' \
106 + calibScript
107
108
109 goodEmailCommand = ' mail -s "' + goodEmailSubject + '" $maillist < ' + calFilePrefix + "CalibReport.txt"
110 badEmailCommand = ' mail -s "' + badEmailSubject + '" $maillist < ' + calFilePrefix + "CalibReport.txt"
111
112 #For reference tag, we actually want the IOV to start just after the LAST run number
113 #Get old run numbers
114 infile = open(runListFile,"rb")
115 runList = pickle.load(infile)
116 runList.sort()
117 print ("got runs")
118 print (runList)
119 infile.close()
120
121
122 if(runNumber in runList):
123 print ("Mailing message")
124 message =["mail","-s",\
125 '"New castor run directory found for previously processed run ' + str(runNumber) + '"',\
126 responsiblePerson,\
127 "<",\
128 "runAlreadyProcessed.mail"]
129 print (message)
130 subprocess.call(message)
131 sys.exit()
132
133
134 highestRun = runList[-1]
135
136 isRunNumberConflict = False
137
138
139 if(highestRun > runNumber):
140 #Something odd happening. The Input run number is lower than the last run number
141 #this script (thinks it) processed
142 #Notify someone important, and don't add to cool when done...
143 subprocess.call(["mail","-s",\
144 "[CSC CALIB PROC] Wrong run number ordering on run " + str(runNumber) \
145 + "! Human intervension required!",\
146 responsiblePerson,\
147 "<",\
148 "runNumberConflict.mail"]\
149 )
150 isRunNumberConflict = True
151 else:
152 #No problem, update run list
153 runList += [runNumber]
154 outfile = open(runListFile,"wb")
155 pickle.dump(runList,outfile)
156 outfile.close()
157
158
159 #Label that we're working, so that other instances of CscCalibQuery won't run
160 subprocess.call(['touch','/afs/cern.ch/user/m/muoncali/CSC/run/runningCalibration' + runNumber])
161
162 #Command to create .db file
163 DbCommand = 'athena.py -c "input=\'' + calFilePrefix+'.cal\';output=\'' \
164 + calFilePrefix + '.db\';IOVRunStart=int(\'' + highestRun + '\')" ' + dbScript
165
166 UploadCommand = ''
167 #Noexec prevents actual execution of cool
168 #UploadCommand = '/afs/cern.ch/user/a/atlcond/utils/AtlCoolMerge.py ' \
169 # + "--batch --noexec --comment=\'Automated UPD1 update from " + calType \
170 # + ' run ' + runNumber +'\' ' \
171 # + calFilePrefix + '.db' \
172 # + ' COMP200 ATONR_COOL ATLAS_COOLONL_CSC_W PASSWORD '
173 #UploadCommand += "\n"
174
175
176
177
178
179 #Command to upload .db file to database
180 UploadCommand += '/afs/cern.ch/user/a/atlcond/utils/AtlCoolMerge.py ' \
181 + "--batch "
182 if(not CoolMergeByDefault or isRunNumberConflict):
183 UploadCommand += " --noexec "
184
185 UploadCommand += "--comment=\'Automated reference update from " + calType \
186 + ' run ' + runNumber + ' to IOV starting at ' + highestRun + "' " \
187 + calFilePrefix + '.db' \
188 + ' COMP200 ATLAS_COOLWRITE ATLAS_COOLOFL_CSC_W WCOOLOFL4CSC17 '
189
190 WebSiteCommand = '\nfs sa ${resultDir} webserver:afs read\n'
191 WebSiteCommand += 'cd $resultDir\n'
192 WebSiteCommand += 'ln -s ${resultDir} ${webSiteDir}/pedRun_' + runNumber + '\n'
193 WebSiteCommand += 'MakeCscPedSite.exe ' + runNumber + '.root ' + runNumber + '.cal_online\n'
194
195 t1 = '\t'
196
197 #Run calibration. If no problems detected, go ahead and upload.
198 bashFileContents += '\ncd ' + workDir + '\n' \
199 + "#Copying files from castor#\n" \
200 + castorCopyCmd \
201 + '\n'\
202 + "#Running calibration (and calib monitoring)\n"\
203 + "#, resulting in .cal file and status report\n"\
204 + calibCommand +'\n' \
205 + "\n"\
206 + "#Athena job to transform *.cal file to *.db file.\n"\
207 + DbCommand + '\n' \
208 + "#Python utility to upload *.db file to database. When entering by\n"\
209 + "#hand, I recomend removing '--batch' flag so you can check puposed\n"\
210 + "#operations before submision to database\n"\
211 + UploadCommand + '\n' \
212 + '#Check if AllCalibMonGood file was created, which means that\n'\
213 + '#this run passed acceptable criteria in the calibration monitoring\n'\
214 + 'if [ -a AllCalibMonGood ]; then\n' \
215 + t1 + "#Email list that the calibration looks good\n"\
216 + t1 + goodEmailCommand +'\n' \
217 + t1 + "################################################################\n"\
218 + t1 + "#Execute next two commands if you want to submit database entry#\n"\
219 + t1 + "#Useful if these steps were skipped due to suspicious behaviour#\n"\
220 + t1 + "#during calibration. #\n"\
221 + t1 + "###############################################################\n"\
222 + t1 + "\n"\
223 + t1 + '\n'\
224 + 'else\n' \
225 + t1 + "#Suspicious behaviour in calibration. Notify mail list of this fact\n"\
226 + t1 + badEmailCommand + '\n' \
227 + 'fi\n'\
228 + '\n'\
229 + '#Always create website'\
230 + WebSiteCommand + '\n' \
231 + 'rm -rf $bytestreamDir\n' \
232 + 'rm -rf ' + scriptDir +"runningCalibration" + runNumber +'\n'
233
234 #Write bashfile
235 print ("Printing bash file to: " +bashFilePath)
236 bashFile = open(bashFilePath, 'w')
237 bashFile.write(bashFileContents)
238 bashFile.close()
239
240 #Submit script
241 os.system('chmod +x ' + bashFilePath)
242 bsubsMessage = os.popen(bsubCmd).read()
243
244 #Send alert email
245 emailMessage = 'Starting ' + calType + ' calibration on run number ' + runNumber
246 emailSubject = '[CSC CALIB PROC]: ' + emailMessage
247 emailMessage += '\nbsubs output:\n' + bsubsMessage
248 os.system('echo "' + emailMessage + '" | mail -s "' + emailSubject + '" ' + maillist)
249
if
if(febId1==febId2)
Definition LArRodBlockPhysicsV0.cxx:567
read
IovVectorMap_t read(const Folder &theFolder, const SelectionCriterion &choice, const unsigned int limit=10)
Definition openCoraCool.cxx:569

◆ updateList()

CscCalibQuery.updateList ( oldListPath,
newList )

Definition at line 36 of file CscCalibQuery.py.

36def updateList(oldListPath,newList):
37 print ('updating file list')
38 #update the old file list
39 outFile = open(oldListPath, 'w')
40 outFile.write(newList)
41 outFile.close()
42
43#runs calibration on input file.
44#It creates a bash script which can setup the job, run it, and do post job
45#processing. The bash script is submitted to lxbatch

Variable Documentation

◆ calibFileDir

str CscCalibQuery.calibFileDir = '/castor/cern.ch/user/l/lampen/CalibRunTest/slope/'

Definition at line 258 of file CscCalibQuery.py.

◆ calibRe

CscCalibQuery.calibRe = re.compile('data1.*_calib.*calibration_pedCSC\\.daq\\.RAW.*\\.data')

Definition at line 265 of file CscCalibQuery.py.

◆ calType

CscCalibQuery.calType = sys.argv[1]

Main program#############################.

Definition at line 256 of file CscCalibQuery.py.

◆ castorCopyCmd

str CscCalibQuery.castorCopyCmd = "mkdir ${bytestreamDir}"

Definition at line 345 of file CscCalibQuery.py.

◆ cmd

str CscCalibQuery.cmd = 'rfdir ' + calibFileDir + DirName

Definition at line 328 of file CscCalibQuery.py.

◆ CoolMergeByDefault

bool CscCalibQuery.CoolMergeByDefault = False

Definition at line 31 of file CscCalibQuery.py.

◆ currentDirList

CscCalibQuery.currentDirList = currentLs.split('\n')

Definition at line 300 of file CscCalibQuery.py.

◆ currentLs

CscCalibQuery.currentLs = os.popen('rfdir ' + calibFileDir).read()

Definition at line 281 of file CscCalibQuery.py.

◆ day

CscCalibQuery.day = int(splitDir[-3])

Definition at line 311 of file CscCalibQuery.py.

◆ DirName

CscCalibQuery.DirName = splitDir[-1]

Definition at line 312 of file CscCalibQuery.py.

◆ fileList

CscCalibQuery.fileList = os.popen(cmd).read().split('\n')

Definition at line 329 of file CscCalibQuery.py.

◆ fileName

tuple CscCalibQuery.fileName = (file.split(' '))[-1]

Definition at line 347 of file CscCalibQuery.py.

◆ fullPath

str CscCalibQuery.fullPath = calibFileDir + DirName + '/' + fileName

Definition at line 359 of file CscCalibQuery.py.

◆ inFile

CscCalibQuery.inFile = open(oldListFilePath, 'r')

Definition at line 286 of file CscCalibQuery.py.

◆ maillist

str CscCalibQuery.maillist = responsiblePerson

Definition at line 29 of file CscCalibQuery.py.

◆ nFiles

int CscCalibQuery.nFiles = len(fileList) -1

Definition at line 331 of file CscCalibQuery.py.

◆ now

CscCalibQuery.now = datetime.datetime.now()

Definition at line 294 of file CscCalibQuery.py.

◆ numFilesToRun

int CscCalibQuery.numFilesToRun = 3

Definition at line 27 of file CscCalibQuery.py.

◆ oldDirList

CscCalibQuery.oldDirList = oldLs.split('\n')

Definition at line 301 of file CscCalibQuery.py.

◆ oldListFilePath

str CscCalibQuery.oldListFilePath = '/afs/cern.ch/user/m/muoncali/CSC/run/pulserList.txt'

Definition at line 259 of file CscCalibQuery.py.

◆ oldLs

CscCalibQuery.oldLs = inFile.read()

Definition at line 287 of file CscCalibQuery.py.

◆ outputDir

str CscCalibQuery.outputDir = '/afs/cern.ch/user/m/muoncali/w0/CSC/runs/pulser/pulser'

Definition at line 260 of file CscCalibQuery.py.

◆ outputDirFull

str CscCalibQuery.outputDirFull = outputDir + 'Run_' + runNumber

Definition at line 337 of file CscCalibQuery.py.

◆ responsiblePerson

str CscCalibQuery.responsiblePerson = "youzhou@email.arizona.edu"

Definition at line 28 of file CscCalibQuery.py.

◆ runningDir

CscCalibQuery.runningDir = ""

Definition at line 304 of file CscCalibQuery.py.

◆ runNumber

CscCalibQuery.runNumber = DirName

Definition at line 334 of file CscCalibQuery.py.

◆ splitDir

CscCalibQuery.splitDir = Dir.split()

Definition at line 310 of file CscCalibQuery.py.

◆ testFile

CscCalibQuery.testFile = glob.glob("/afs/cern.ch/user/m/muoncali/CSC/run/runningCalibration*")

Definition at line 273 of file CscCalibQuery.py.

◆ ThisCalibDir

CscCalibQuery.ThisCalibDir = False

Definition at line 344 of file CscCalibQuery.py.

◆ timediff

CscCalibQuery.timediff = today - day

Definition at line 315 of file CscCalibQuery.py.

◆ today

CscCalibQuery.today = now.day

Definition at line 295 of file CscCalibQuery.py.

◆ updateRunList

bool CscCalibQuery.updateRunList = True

Definition at line 303 of file CscCalibQuery.py.

Generated on for ATLAS Offline Software by doxygen 1.14.0