python.PostProcessing.PostProcessingStep Class Reference

Collaboration diagram for python.PostProcessing.PostProcessingStep:

Public Member Functions
	__init__ (self, taskman, taskDict, oldStatus, previousSteps, postprocLib, jobName=None)
	__del__ (self)
	log (self, msg=None, text=None, noTime=False, doPrint=False)
	logExec (self, cmd, doPrint=False, checkStatusCode=True, errorMsg='', abortOnError=True)
	getFileName (self, postFix, stepName='', absPath=False)
	jobList (self)
	taskFileList (self, fileNamePattern, statusName='POSTPROCESSING', jobName=None)
	hadd (self, dir, outputFileName, inputFiles, maxFilesPerHadd=100)
	getJobConfig (self, jobName=None)
	addResult (self, resultFileName)

Public Attributes
	stepName = self.__class__.__name__
	executedSteps = previousSteps
	taskman = taskman
	taskDict = taskDict
	dsName = taskDict['DSNAME']
	taskName = taskDict['TASKNAME']
str	taskDir = '/'.join([self.dsName,self.taskName])
	oldStatus = oldStatus
	postprocLib = postprocLib
	jobName = jobName
str	baseName = '%s-%s' % (self.dsName,self.taskName)
str	baseNameStep = '%s-%s.%s' % (self.dsName,self.taskName,self.stepName)
	logfile = open('%s/%s.log' % (self.taskDir,self.baseNameStep), 'w')

Detailed Description

Definition at line 148 of file PostProcessing.py.

Constructor & Destructor Documentation

__init__()

python.PostProcessing.PostProcessingStep.__init__	(		self,
			taskman,
			taskDict,
			oldStatus,
			previousSteps,
			postprocLib,
			jobName = None )

Definition at line 150 of file PostProcessing.py.

    def __init__(self,taskman,taskDict,oldStatus,previousSteps,postprocLib,jobName=None):
        self.stepName = self.__class__.__name__
        self.executedSteps = previousSteps
        self.executedSteps.append( self.stepName )
        self.taskman = taskman
        self.taskDict = taskDict
        self.dsName = taskDict['DSNAME']
        self.taskName = taskDict['TASKNAME']
        self.taskDir = '/'.join([self.dsName,self.taskName])
        self.oldStatus = oldStatus
        self.postprocLib = postprocLib
        self.jobName = jobName
        self.baseName     = '%s-%s' % (self.dsName,self.taskName)
        self.baseNameStep = '%s-%s.%s' % (self.dsName,self.taskName,self.stepName)
        self.logfile = open('%s/%s.log' % (self.taskDir,self.baseNameStep), 'w')
        self.log('Starting postprocessing:  %s' % self.__class__.__name__)
        self.log('* Data set name = %s' % self.dsName, noTime=True)
        self.log('* Task name     = %s' % self.taskName, noTime=True)
        self.log('* Old status    = %s' % getKey(TaskManager.StatusCodes,self.oldStatus), noTime=True)
        self.log('* Job name      = %s' % self.jobName, noTime=True)
        self.log('* User name     = %s' % getUserName(), noTime=True)
        self.log('* Host name     = %s\n' % os.uname()[1], noTime=True)
        pass
 

__del__()

python.PostProcessing.PostProcessingStep.__del__

(

self

)

Definition at line 174 of file PostProcessing.py.

    def __del__(self):
        self.log('Done')
        self.logfile.close()
 

Member Function Documentation

addResult()

python.PostProcessing.PostProcessingStep.addResult	(		self,
			resultFileName )

Definition at line 270 of file PostProcessing.py.

    def addResult(self,resultFileName):
        # FIXME: strip leading dirs from resultFilename, so I can pass an absolute file name?
        if not resultFileName:
            return
        resultFiles = self.taskDict['RESULTFILES']
        if resultFiles is None:
            resultFiles = ''
        if resultFileName not in resultFiles.split():
            resultFiles = ' '.join([resultFiles,resultFileName])
            self.taskman.setValue(self.dsName,self.taskName,'RESULTFILES',resultFiles)
            self.taskDict['RESULTFILES'] = resultFiles

getFileName()

python.PostProcessing.PostProcessingStep.getFileName	(		self,
			postFix,
			stepName = '',
			absPath = False )

Definition at line 211 of file PostProcessing.py.

    def getFileName(self,postFix,stepName='',absPath=False):
        if stepName:
            name = '%s.%s%s' % (self.baseName,stepName,postFix)
        else:
            name = self.baseNameStep+postFix
        if absPath:
            name = '%s/%s' % (self.taskDir,name)
        return name
 

getJobConfig()

python.PostProcessing.PostProcessingStep.getJobConfig	(		self,
			jobName = None )

Definition at line 259 of file PostProcessing.py.

    def getJobConfig(self,jobName=None):
        config = {}
        if not jobName:
            jobName = '*'
        configFile = glob.glob('%s/%s/%s' % (self.taskDir,jobName,'*.config.py.final.py'))
        if not configFile:
            configFile = glob.glob('%s/%s/%s' % (self.taskDir,jobName,'*.config.py'))
        if configFile:
            exec(open(configFile[0],'r'),config)    # Eval config file and put defs into config dict
        return config['jobConfig']
 

hadd()

python.PostProcessing.PostProcessingStep.hadd	(		self,
			dir,
			outputFileName,
			inputFiles,
			maxFilesPerHadd = 100 )

Execute staged hadd for many files.

Definition at line 242 of file PostProcessing.py.

    def hadd(self,dir,outputFileName,inputFiles,maxFilesPerHadd=100):
        """Execute staged hadd for many files."""
        if len(inputFiles)<=maxFilesPerHadd:
            self.logExec('cd %s; hadd -f %s %s' % (dir,outputFileName,' '.join(inputFiles)))
        else:
            nSteps = int(math.ceil(float(len(inputFiles))/maxFilesPerHadd))
            partFiles = []
            for index in range(nSteps):
                minIndex = index*maxFilesPerHadd
                maxIndex = min((index+1)*maxFilesPerHadd,len(inputFiles))
                stepInputFiles = inputFiles[minIndex:maxIndex]
                stepOutputFileName = '%s.part%i' % (outputFileName,index)
                self.logExec('cd %s; hadd -f %s %s' % (dir,stepOutputFileName,' '.join(stepInputFiles)))
                partFiles.append(stepOutputFileName)
            self.logExec('cd %s; hadd -f %s %s' % (dir,outputFileName,' '.join(partFiles)))
            self.logExec('cd %s; rm -f %s' % (dir,' '.join(partFiles)))            
 

jobList()

python.PostProcessing.PostProcessingStep.jobList

(

self

)

Definition at line 220 of file PostProcessing.py.

    def jobList(self):
        try:
            l = os.listdir(self.taskDir)
        except Exception:
            l = []
        return l
 

log()

python.PostProcessing.PostProcessingStep.log	(		self,
			msg = None,
			text = None,
			noTime = False,
			doPrint = False )

Definition at line 178 of file PostProcessing.py.

    def log(self,msg=None,text=None,noTime=False,doPrint=False):
        outputFiles = [self.logfile]
        if doPrint:
            outputFiles.append(sys.stdout)
        for out in outputFiles:
            if msg:
                if noTime:
                    out.write(' '*31)
                    out.write(msg)
                    out.write('\n')
                else:
                    out.write('\n%-30s %s\n' % (time.strftime('%a %b %d %X %Z %Y'),msg))
            if text:
                #out.write('\n')
                out.write(text)
                out.write('\n')
 

logExec()

python.PostProcessing.PostProcessingStep.logExec	(		self,
			cmd,
			doPrint = False,
			checkStatusCode = True,
			errorMsg = '',
			abortOnError = True )

Definition at line 195 of file PostProcessing.py.

    def logExec(self,cmd,doPrint=False,checkStatusCode=True,errorMsg='',abortOnError=True):
        (status,output) = subprocess.getstatusoutput(cmd)
        status = status >> 8   # Convert to standard Unix exit code 
        self.log('Executing: %s' % cmd, output)
        if doPrint or status:
            print (output)
            print()
        if status and checkStatusCode:
            if not errorMsg:
                errorMsg = 'ERROR in postprocessing step %s while executing:\n\n%s\n' % (self.stepName,cmd)
            self.log(text='\nERROR: exit status = %i' % (status))
            self.log(text=errorMsg)
            if abortOnError:
                raise PostProcessingError(errorMsg,self.executedSteps)
        return status
 

taskFileList()

python.PostProcessing.PostProcessingStep.taskFileList	(		self,
			fileNamePattern,
			statusName = 'POSTPROCESSING',
			jobName = None )

Get list of files with specific names from all jobs in a given state.

Definition at line 227 of file PostProcessing.py.

    def taskFileList(self,fileNamePattern,statusName='POSTPROCESSING',jobName=None):
        """Get list of files with specific names from all jobs in a given state."""
        oldDir = os.getcwd()
        os.chdir(self.taskDir)
        if jobName:
            statusFiles = glob.glob('%s/*.status.%s' % (jobName,statusName))
        else:
            statusFiles = glob.glob('*/*.status.%s' % (statusName))
        files = []
        for s in statusFiles:
            b = '/'.join(s.split('/')[:-1])
            files.extend(glob.glob('%s/%s' % (b,fileNamePattern)))
        os.chdir(oldDir)
        return sorted(files)
 

Member Data Documentation

baseName

str python.PostProcessing.PostProcessingStep.baseName = '%s-%s' % (self.dsName,self.taskName)

Definition at line 162 of file PostProcessing.py.

baseNameStep

str python.PostProcessing.PostProcessingStep.baseNameStep = '%s-%s.%s' % (self.dsName,self.taskName,self.stepName)

Definition at line 163 of file PostProcessing.py.

dsName

python.PostProcessing.PostProcessingStep.dsName = taskDict['DSNAME']

Definition at line 156 of file PostProcessing.py.

executedSteps

python.PostProcessing.PostProcessingStep.executedSteps = previousSteps

Definition at line 152 of file PostProcessing.py.

jobName

python.PostProcessing.PostProcessingStep.jobName = jobName

Definition at line 161 of file PostProcessing.py.

logfile

python.PostProcessing.PostProcessingStep.logfile = open('%s/%s.log' % (self.taskDir,self.baseNameStep), 'w')

Definition at line 164 of file PostProcessing.py.

oldStatus

python.PostProcessing.PostProcessingStep.oldStatus = oldStatus

Definition at line 159 of file PostProcessing.py.

postprocLib

python.PostProcessing.PostProcessingStep.postprocLib = postprocLib

Definition at line 160 of file PostProcessing.py.

stepName

python.PostProcessing.PostProcessingStep.stepName = self.__class__.__name__

Definition at line 151 of file PostProcessing.py.

taskDict

python.PostProcessing.PostProcessingStep.taskDict = taskDict

Definition at line 155 of file PostProcessing.py.

taskDir

python.PostProcessing.PostProcessingStep.taskDir = '/'.join([self.dsName,self.taskName])

Definition at line 158 of file PostProcessing.py.

taskman

python.PostProcessing.PostProcessingStep.taskman = taskman

Definition at line 154 of file PostProcessing.py.

taskName

python.PostProcessing.PostProcessingStep.taskName = taskDict['TASKNAME']

Definition at line 157 of file PostProcessing.py.

---

The documentation for this class was generated from the following file:

• PostProcessing.py