11 """Run on multiple pedestal files
13 #Used to run on multiple pedestal bytestream files located somewhere on disk.
14 #Puts results into seperate directories by run-number. It extracts pertinent
15 #information from the file names, which are expected to be of the form:
17 Quickie folowup when BulkRun failed on a few. This will re-enter existing directories
18 of particular run numbers. It should be given a sepearte processedFiles.list
20 data10_calib.00157081.calibration_pedCSC.daq.RAW._lb0000._CSC-EB._0001.data
22 run() - run over all runs
23 ReadProcessedFilesList() - find all files already run on from a local text file
24 AddProcessedFiles() - save newly run on files to disk
25 FindFiles() - Find pattern and run number of a set of files that have not yet been run on
26 RunAthena() - run athena job for a set of runs
32 inputPattern = "/raid02/schernau/ped/csc/*.data",
33 processedFilesList = "ProcessedFiles_Followup.list",
34 outputDirBase = "/raid02/lampen/datasets/csc/PedProcessing",
36 allowDirOverwrite = False,
38 """initialize internal variables"""
46 def run(self, numToRun = 10) :
48 Run over all run numbers, find files for each, and submit each set to
52 print (
"Running on " +
str(numToRun) +
" runsSet.")
56 for runCnt
in range(numToRun) :
57 print (
">>>>Running on runSet " +
str(runCnt+1) +
" of " +
str(numToRun))
62 runNumbers += [runNumber]
66 print (
"Skipping file from runNumber " +
str(runNumber) +
" (Not allowed)")
68 newProcessedFiles = glob(pattern)
72 print (
"No more unprocessed files. Congrats!")
73 print (
"N runs done: " +
str(runCnt +1))
76 print (
"finished all " +
str(numToRun) )
77 print (
"Run numbers include:")
80 print (
"All Processed files:" )
90 ProcessedFiles = pickle.load(f)
93 print (
'Processed String: ')
94 print (ProcessedFiles)
99 """Save new processed files to disk"""
101 print (
"Got old processed files")
102 print (ProcessedFiles)
103 ProcessedFiles += newFiles
104 print (
"New version:")
105 print (ProcessedFiles)
107 pickle.dump(ProcessedFiles,f)
117 FoundUnprocessedFile =
False
127 print (
"Searching for file")
134 for file
in inputFiles:
135 if not ProcessedFiles.count(file):
137 index = file.find(
"data10")
139 index = file.find(
"data11")
141 print (
"ERROR! Index of -1 searching for data1X prefix in filename:")
143 raise Exception(
"Index error")
144 FoundUnprocessedFile =
True
145 pattern = file[0:index + 22] +
"*"
146 runNumber = file[index + 13: index + 21]
148 if(
not FoundUnprocessedFile):
150 print (
"Did not find unprocessed file")
154 print (
"Found unprocessed file with pattern: " + pattern)
155 print (
"This includes files:")
156 print (glob(pattern))
158 return pattern, runNumber
161 """Run athena on a particular set of files matching pattern"""
165 subprocess.call(
"rm -rf " + outputDirPath)
167 print (
"Making directory" + outputDirPath)
169 os.mkdir(outputDirPath,0o755)
172 athOpt =
"outputPre='" + outputDirPath +
"/" + runNumber
173 athOpt +=
"';inputPat='" + pattern
174 athOpt +=
"';reportPrefix='runNumber = "
175 athOpt += runNumber +
"'"
178 athArgs = [
"athena.py",
"-c", athOpt,
"CscCalcPedMon.py"]
181 logFile =
open(outputDirPath +
"/run2.log",
"w")
184 print (
"**************************************")
185 print (
"Starting running on run " +
str(runNumber))
187 subprocess.Popen(athArgs,stdout=logFile,stderr=subprocess.STDOUT).wait()
188 print (
"Finished run " +
str(runNumber))
189 print (
"**************************************")
195 newProcessedFiles = glob(pattern)