ATLAS Offline Software
Public Member Functions | Public Attributes | List of all members
python.PostProcessing.PostProcessingStep Class Reference
Collaboration diagram for python.PostProcessing.PostProcessingStep:

Public Member Functions

def __init__ (self, taskman, taskDict, oldStatus, previousSteps, postprocLib, jobName=None)
 
def __del__ (self)
 
def log (self, msg=None, text=None, noTime=False, doPrint=False)
 
def logExec (self, cmd, doPrint=False, checkStatusCode=True, errorMsg='', abortOnError=True)
 
def getFileName (self, postFix, stepName='', absPath=False)
 
def jobList (self)
 
def taskFileList (self, fileNamePattern, statusName='POSTPROCESSING', jobName=None)
 
def hadd (self, dir, outputFileName, inputFiles, maxFilesPerHadd=100)
 
def getJobConfig (self, jobName=None)
 
def addResult (self, resultFileName)
 

Public Attributes

 stepName
 
 executedSteps
 
 taskman
 
 taskDict
 
 dsName
 
 taskName
 
 taskDir
 
 oldStatus
 
 postprocLib
 
 jobName
 
 baseName
 
 baseNameStep
 
 logfile
 

Detailed Description

Definition at line 146 of file PostProcessing.py.

Constructor & Destructor Documentation

◆ __init__()

def python.PostProcessing.PostProcessingStep.__init__ (   self,
  taskman,
  taskDict,
  oldStatus,
  previousSteps,
  postprocLib,
  jobName = None 
)

Definition at line 148 of file PostProcessing.py.

148  def __init__(self,taskman,taskDict,oldStatus,previousSteps,postprocLib,jobName=None):
149  self.stepName = self.__class__.__name__
150  self.executedSteps = previousSteps
151  self.executedSteps.append( self.stepName )
152  self.taskman = taskman
153  self.taskDict = taskDict
154  self.dsName = taskDict['DSNAME']
155  self.taskName = taskDict['TASKNAME']
156  self.taskDir = '/'.join([self.dsName,self.taskName])
157  self.oldStatus = oldStatus
158  self.postprocLib = postprocLib
159  self.jobName = jobName
160  self.baseName = '%s-%s' % (self.dsName,self.taskName)
161  self.baseNameStep = '%s-%s.%s' % (self.dsName,self.taskName,self.stepName)
162  self.logfile = open('%s/%s.log' % (self.taskDir,self.baseNameStep), 'w')
163  self.log('Starting postprocessing: %s' % self.__class__.__name__)
164  self.log('* Data set name = %s' % self.dsName, noTime=True)
165  self.log('* Task name = %s' % self.taskName, noTime=True)
166  self.log('* Old status = %s' % getKey(TaskManager.StatusCodes,self.oldStatus), noTime=True)
167  self.log('* Job name = %s' % self.jobName, noTime=True)
168  self.log('* User name = %s' % getUserName(), noTime=True)
169  self.log('* Host name = %s\n' % os.uname()[1], noTime=True)
170  pass
171 

◆ __del__()

def python.PostProcessing.PostProcessingStep.__del__ (   self)

Definition at line 172 of file PostProcessing.py.

172  def __del__(self):
173  self.log('Done')
174  self.logfile.close()
175 

Member Function Documentation

◆ addResult()

def python.PostProcessing.PostProcessingStep.addResult (   self,
  resultFileName 
)

Definition at line 268 of file PostProcessing.py.

268  def addResult(self,resultFileName):
269  # FIXME: strip leading dirs from resultFilename, so I can pass an absolute file name?
270  if not resultFileName:
271  return
272  resultFiles = self.taskDict['RESULTFILES']
273  if resultFiles is None:
274  resultFiles = ''
275  if resultFileName not in resultFiles.split():
276  resultFiles = ' '.join([resultFiles,resultFileName])
277  self.taskman.setValue(self.dsName,self.taskName,'RESULTFILES',resultFiles)
278  self.taskDict['RESULTFILES'] = resultFiles

◆ getFileName()

def python.PostProcessing.PostProcessingStep.getFileName (   self,
  postFix,
  stepName = '',
  absPath = False 
)

Definition at line 209 of file PostProcessing.py.

209  def getFileName(self,postFix,stepName='',absPath=False):
210  if stepName:
211  name = '%s.%s%s' % (self.baseName,stepName,postFix)
212  else:
213  name = self.baseNameStep+postFix
214  if absPath:
215  name = '%s/%s' % (self.taskDir,name)
216  return name
217 

◆ getJobConfig()

def python.PostProcessing.PostProcessingStep.getJobConfig (   self,
  jobName = None 
)

Definition at line 257 of file PostProcessing.py.

257  def getJobConfig(self,jobName=None):
258  config = {}
259  if not jobName:
260  jobName = '*'
261  configFile = glob.glob('%s/%s/%s' % (self.taskDir,jobName,'*.config.py.final.py'))
262  if not configFile:
263  configFile = glob.glob('%s/%s/%s' % (self.taskDir,jobName,'*.config.py'))
264  if configFile:
265  exec(open(configFile[0],'r'),config) # Eval config file and put defs into config dict
266  return config['jobConfig']
267 

◆ hadd()

def python.PostProcessing.PostProcessingStep.hadd (   self,
  dir,
  outputFileName,
  inputFiles,
  maxFilesPerHadd = 100 
)
Execute staged hadd for many files.

Definition at line 240 of file PostProcessing.py.

240  def hadd(self,dir,outputFileName,inputFiles,maxFilesPerHadd=100):
241  """Execute staged hadd for many files."""
242  if len(inputFiles)<=maxFilesPerHadd:
243  self.logExec('cd %s; hadd -f %s %s' % (dir,outputFileName,' '.join(inputFiles)))
244  else:
245  nSteps = int(math.ceil(float(len(inputFiles))/maxFilesPerHadd))
246  partFiles = []
247  for index in range(nSteps):
248  minIndex = index*maxFilesPerHadd
249  maxIndex = min((index+1)*maxFilesPerHadd,len(inputFiles))
250  stepInputFiles = inputFiles[minIndex:maxIndex]
251  stepOutputFileName = '%s.part%i' % (outputFileName,index)
252  self.logExec('cd %s; hadd -f %s %s' % (dir,stepOutputFileName,' '.join(stepInputFiles)))
253  partFiles.append(stepOutputFileName)
254  self.logExec('cd %s; hadd -f %s %s' % (dir,outputFileName,' '.join(partFiles)))
255  self.logExec('cd %s; rm -f %s' % (dir,' '.join(partFiles)))
256 

◆ jobList()

def python.PostProcessing.PostProcessingStep.jobList (   self)

Definition at line 218 of file PostProcessing.py.

218  def jobList(self):
219  try:
220  l = os.listdir(self.taskDir)
221  except Exception:
222  l = []
223  return l
224 

◆ log()

def python.PostProcessing.PostProcessingStep.log (   self,
  msg = None,
  text = None,
  noTime = False,
  doPrint = False 
)

Definition at line 176 of file PostProcessing.py.

176  def log(self,msg=None,text=None,noTime=False,doPrint=False):
177  outputFiles = [self.logfile]
178  if doPrint:
179  outputFiles.append(sys.stdout)
180  for out in outputFiles:
181  if msg:
182  if noTime:
183  out.write(' '*31)
184  out.write(msg)
185  out.write('\n')
186  else:
187  out.write('\n%-30s %s\n' % (time.strftime('%a %b %d %X %Z %Y'),msg))
188  if text:
189  #out.write('\n')
190  out.write(text)
191  out.write('\n')
192 

◆ logExec()

def python.PostProcessing.PostProcessingStep.logExec (   self,
  cmd,
  doPrint = False,
  checkStatusCode = True,
  errorMsg = '',
  abortOnError = True 
)

Definition at line 193 of file PostProcessing.py.

193  def logExec(self,cmd,doPrint=False,checkStatusCode=True,errorMsg='',abortOnError=True):
194  (status,output) = subprocess.getstatusoutput(cmd)
195  status = status >> 8 # Convert to standard Unix exit code
196  self.log('Executing: %s' % cmd, output)
197  if doPrint or status:
198  print (output)
199  print()
200  if status and checkStatusCode:
201  if not errorMsg:
202  errorMsg = 'ERROR in postprocessing step %s while executing:\n\n%s\n' % (self.stepName,cmd)
203  self.log(text='\nERROR: exit status = %i' % (status))
204  self.log(text=errorMsg)
205  if abortOnError:
206  raise PostProcessingError(errorMsg,self.executedSteps)
207  return status
208 

◆ taskFileList()

def python.PostProcessing.PostProcessingStep.taskFileList (   self,
  fileNamePattern,
  statusName = 'POSTPROCESSING',
  jobName = None 
)
Get list of files with specific names from all jobs in a given state.

Definition at line 225 of file PostProcessing.py.

225  def taskFileList(self,fileNamePattern,statusName='POSTPROCESSING',jobName=None):
226  """Get list of files with specific names from all jobs in a given state."""
227  oldDir = os.getcwd()
228  os.chdir(self.taskDir)
229  if jobName:
230  statusFiles = glob.glob('%s/*.status.%s' % (jobName,statusName))
231  else:
232  statusFiles = glob.glob('*/*.status.%s' % (statusName))
233  files = []
234  for s in statusFiles:
235  b = '/'.join(s.split('/')[:-1])
236  files.extend(glob.glob('%s/%s' % (b,fileNamePattern)))
237  os.chdir(oldDir)
238  return sorted(files)
239 

Member Data Documentation

◆ baseName

python.PostProcessing.PostProcessingStep.baseName

Definition at line 160 of file PostProcessing.py.

◆ baseNameStep

python.PostProcessing.PostProcessingStep.baseNameStep

Definition at line 161 of file PostProcessing.py.

◆ dsName

python.PostProcessing.PostProcessingStep.dsName

Definition at line 154 of file PostProcessing.py.

◆ executedSteps

python.PostProcessing.PostProcessingStep.executedSteps

Definition at line 150 of file PostProcessing.py.

◆ jobName

python.PostProcessing.PostProcessingStep.jobName

Definition at line 159 of file PostProcessing.py.

◆ logfile

python.PostProcessing.PostProcessingStep.logfile

Definition at line 162 of file PostProcessing.py.

◆ oldStatus

python.PostProcessing.PostProcessingStep.oldStatus

Definition at line 157 of file PostProcessing.py.

◆ postprocLib

python.PostProcessing.PostProcessingStep.postprocLib

Definition at line 158 of file PostProcessing.py.

◆ stepName

python.PostProcessing.PostProcessingStep.stepName

Definition at line 149 of file PostProcessing.py.

◆ taskDict

python.PostProcessing.PostProcessingStep.taskDict

Definition at line 153 of file PostProcessing.py.

◆ taskDir

python.PostProcessing.PostProcessingStep.taskDir

Definition at line 156 of file PostProcessing.py.

◆ taskman

python.PostProcessing.PostProcessingStep.taskman

Definition at line 152 of file PostProcessing.py.

◆ taskName

python.PostProcessing.PostProcessingStep.taskName

Definition at line 155 of file PostProcessing.py.


The documentation for this class was generated from the following file:
CaloCellPos2Ntuple.int
int
Definition: CaloCellPos2Ntuple.py:24
python.TaskManager.getJobConfig
def getJobConfig(jobDir, dsName, taskName, jobName=' *')
Definition: TaskManager.py:103
min
constexpr double min()
Definition: ap_fixedTest.cxx:26
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
RCU::hadd
void hadd(const std::string &output_file, const std::vector< std::string > &input_files, unsigned max_files)
effects: perform the hadd functionality guarantee: basic failures: out of memory III failures: i/o er...
Definition: hadd.cxx:28
LArG4FSStartPointFilter.exec
exec
Definition: LArG4FSStartPointFilter.py:103
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:195
DerivationFramework::TriggerMatchingUtils::sorted
std::vector< typename T::value_type > sorted(T begin, T end)
Helper function to create a sorted vector from an unsorted one.
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:25
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
python.processes.powheg.ZZ.ZZ.__init__
def __init__(self, base_directory, **kwargs)
Constructor: all process options are set here.
Definition: ZZ.py:18
Trk::open
@ open
Definition: BinningType.h:40
python.AthDsoLogger.__del__
def __del__(self)
Definition: AthDsoLogger.py:82
python.TrfUtils.getFileName
def getFileName(name, sep='#')
Definition: TrfUtils.py:45
python.CaloCondTools.log
log
Definition: CaloCondTools.py:20
pool::getKey
std::string getKey(const std::string &key, const std::string &encoded)
readCCLHist.float
float
Definition: readCCLHist.py:83
python.Utils.getUserName
def getUserName(default='UNKNOWN')
Definition: InnerDetector/InDetExample/InDetBeamSpotExample/python/Utils.py:48