ATLAS Offline Software
BulkRunFollowup.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
2 
3 
4 from glob import glob
5 import subprocess
6 import os
7 import pickle
8 import sys
9 
11  """Run on multiple pedestal files
12 
13  #Used to run on multiple pedestal bytestream files located somewhere on disk.
14  #Puts results into seperate directories by run-number. It extracts pertinent
15  #information from the file names, which are expected to be of the form:
16 
17  Quickie folowup when BulkRun failed on a few. This will re-enter existing directories
18  of particular run numbers. It should be given a sepearte processedFiles.list
19 
20  data10_calib.00157081.calibration_pedCSC.daq.RAW._lb0000._CSC-EB._0001.data
21 
22  run() - run over all runs
23  ReadProcessedFilesList() - find all files already run on from a local text file
24  AddProcessedFiles() - save newly run on files to disk
25  FindFiles() - Find pattern and run number of a set of files that have not yet been run on
26  RunAthena() - run athena job for a set of runs
27 
28  """
29  def __init__(
30  self,
31  allowedRuns,
32  inputPattern = "/raid02/schernau/ped/csc/*.data",
33  processedFilesList = "ProcessedFiles_Followup.list",
34  outputDirBase = "/raid02/lampen/datasets/csc/PedProcessing",
35  debug = False,
36  allowDirOverwrite = False,
37  ):
38  """initialize internal variables"""
39  self.AllowedRuns = allowedRuns
40  self.InputPattern = inputPattern
41  self.ProcessedFilesList = processedFilesList
42  self.OutputDirBase = outputDirBase
43  self.debug = debug
44  self.AllowDirOverwrite = allowDirOverwrite
45 
46  def run(self, numToRun = 10) :
47  """
48  Run over all run numbers, find files for each, and submit each set to
49  CscCalcPedMon.py
50  """
51 
52  print ("Running on " + str(numToRun) + " runsSet.")
53 
54 
55  runNumbers = []
56  for runCnt in range(numToRun) :
57  print (">>>>Running on runSet " + str(runCnt+1) + " of " + str(numToRun))
58  pattern,runNumber = self.FindFiles()
59  if(pattern != ""):
60  #have files to run on
61  if runNumber in self.AllowedRuns:
62  runNumbers += [runNumber]
63  self.RunAthena(pattern,runNumber)
64  else:
65  if(self.debug):
66  print ("Skipping file from runNumber " + str(runNumber) + " (Not allowed)")
67  #Add files we're skipping into file list
68  newProcessedFiles = glob(pattern)
69  self.AddProcessedFiles(newProcessedFiles)
70 
71  else:
72  print ("No more unprocessed files. Congrats!")
73  print ("N runs done: " + str(runCnt +1))
74  print (runNumbers)
75  return
76  print ("finished all " + str(numToRun) )
77  print ("Run numbers include:")
78  print (runNumbers)
79  print()
80  print ("All Processed files:" )
81  print (self.ReadProcessedFilesList())
82 
83  #Read list of previously processed files
85 
86  ProcessedFiles = []
87 
88  #Get processed files
89  f = open(self.ProcessedFilesList,"rb")
90  ProcessedFiles = pickle.load(f)
91  f.close()
92 
93  print ('Processed String: ')
94  print (ProcessedFiles)
95 
96  return ProcessedFiles
97 
98  def AddProcessedFiles(self,newFiles):
99  """Save new processed files to disk"""
100  ProcessedFiles = self.ReadProcessedFilesList()
101  print ("Got old processed files")
102  print (ProcessedFiles)
103  ProcessedFiles += newFiles
104  print ("New version:")
105  print (ProcessedFiles)
106  f = open(self.ProcessedFilesList,"wb")
107  pickle.dump(ProcessedFiles,f)
108  f.close()
109 
110  return True
111 
112 
113  #Read list of previously processed files
114  def FindFiles(self):
115 
116  #Initial setup
117  FoundUnprocessedFile = False
118 
119  #Get processed file list
120  ProcessedFiles = self.ReadProcessedFilesList()
121 
122  #Get list of files in input dir
123  inputFiles = glob(self.InputPattern)
124 
125  if(self.debug):
126  print()
127  print ("Searching for file")
128  print ("InputPattern: " + self.InputPattern)
129 
130  pattern = ""
131  runNumber = ""
132 
133  #Loop through list until find file that is
134  for file in inputFiles:
135  if not ProcessedFiles.count(file):
136 
137  index = file.find("data10")
138  if(index ==-1):
139  index = file.find("data11")
140  if(index == -1):
141  print ("ERROR! Index of -1 searching for data1X prefix in filename:")
142  print (file)
143  raise Exception("Index error")
144  FoundUnprocessedFile = True
145  pattern = file[0:index + 22] + "*" #includes run number
146  runNumber = file[index + 13: index + 21]
147 
148  if(not FoundUnprocessedFile):
149  if(self.debug):
150  print ("Did not find unprocessed file")
151  return "",""
152 
153  if(self.debug) :
154  print ("Found unprocessed file with pattern: " + pattern)
155  print ("This includes files:")
156  print (glob(pattern))
157 
158  return pattern, runNumber
159 
160  def RunAthena(self, pattern, runNumber):
161  """Run athena on a particular set of files matching pattern"""
162  outputDirPath = self.OutputDirBase + "/" + runNumber
163 
164  if(self.AllowDirOverwrite):
165  subprocess.call("rm -rf " + outputDirPath)
166 
167  print ("Making directory" + outputDirPath)
168  #Create output directory
169  os.mkdir(outputDirPath,0o755)
170 
171  #Athena options
172  athOpt = "outputPre='" + outputDirPath +"/" + runNumber
173  athOpt += "';inputPat='" + pattern
174  athOpt += "';reportPrefix='runNumber = "
175  athOpt += runNumber + "'"
176 
177  #athena arguments
178  athArgs = ["athena.py", "-c", athOpt, "CscCalcPedMon.py"]
179 
180  #Output log file
181  logFile = open(outputDirPath + "/run2.log","w")
182 
183  print()
184  print ("**************************************")
185  print ("Starting running on run " + str(runNumber))
186  sys.stdout.flush()
187  subprocess.Popen(athArgs,stdout=logFile,stderr=subprocess.STDOUT).wait()
188  print ("Finished run " + str(runNumber))
189  print ("**************************************")
190  print()
191 
192  logFile.close()
193 
194  #Add files we just ran on to file list
195  newProcessedFiles = glob(pattern)
196  self.AddProcessedFiles(newProcessedFiles)
BulkRunFollowup.BulkRunFollowup.RunAthena
def RunAthena(self, pattern, runNumber)
Definition: BulkRunFollowup.py:160
BulkRunFollowup.BulkRunFollowup
Definition: BulkRunFollowup.py:10
BulkRunFollowup.BulkRunFollowup.OutputDirBase
OutputDirBase
Definition: BulkRunFollowup.py:34
BulkRunFollowup.BulkRunFollowup.ProcessedFilesList
ProcessedFilesList
Definition: BulkRunFollowup.py:33
BulkRunFollowup.BulkRunFollowup.debug
debug
Definition: BulkRunFollowup.py:35
BulkRunFollowup.BulkRunFollowup.AddProcessedFiles
def AddProcessedFiles(self, newFiles)
Definition: BulkRunFollowup.py:98
BulkRunFollowup.BulkRunFollowup.InputPattern
InputPattern
Definition: BulkRunFollowup.py:32
BulkRunFollowup.BulkRunFollowup.run
def run(self, numToRun=10)
Definition: BulkRunFollowup.py:46
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:194
BulkRunFollowup.BulkRunFollowup.AllowDirOverwrite
AllowDirOverwrite
Definition: BulkRunFollowup.py:36
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:25
BulkRunFollowup.BulkRunFollowup.AllowedRuns
AllowedRuns
Definition: BulkRunFollowup.py:31
Trk::open
@ open
Definition: BinningType.h:40
BulkRunFollowup.BulkRunFollowup.ReadProcessedFilesList
def ReadProcessedFilesList(self)
Definition: BulkRunFollowup.py:84
if
if(febId1==febId2)
Definition: LArRodBlockPhysicsV0.cxx:567
BulkRunFollowup.BulkRunFollowup.__init__
def __init__(self, allowedRuns, inputPattern="/raid02/schernau/ped/csc/*.data", processedFilesList="ProcessedFiles_Followup.list", outputDirBase="/raid02/lampen/datasets/csc/PedProcessing", debug=False, allowDirOverwrite=False)
Definition: BulkRunFollowup.py:29
str
Definition: BTagTrackIpAccessor.cxx:11
BulkRunFollowup.BulkRunFollowup.FindFiles
def FindFiles(self)
Definition: BulkRunFollowup.py:114