python.PostProcessing.PostProcessingStep Class Reference

Collaboration diagram for python.PostProcessing.PostProcessingStep:

Public Member Functions
def	__init__ (self, taskman, taskDict, oldStatus, previousSteps, postprocLib, jobName=None)
def	__del__ (self)
def	log (self, msg=None, text=None, noTime=False, doPrint=False)
def	logExec (self, cmd, doPrint=False, checkStatusCode=True, errorMsg='', abortOnError=True)
def	getFileName (self, postFix, stepName='', absPath=False)
def	jobList (self)
def	taskFileList (self, fileNamePattern, statusName='POSTPROCESSING', jobName=None)
def	hadd (self, dir, outputFileName, inputFiles, maxFilesPerHadd=100)
def	getJobConfig (self, jobName=None)
def	addResult (self, resultFileName)

Public Attributes
	stepName
	executedSteps
	taskman
	taskDict
	dsName
	taskName
	taskDir
	oldStatus
	postprocLib
	jobName
	baseName
	baseNameStep
	logfile

Detailed Description

Definition at line 148 of file PostProcessing.py.

Constructor & Destructor Documentation

__init__()

def python.PostProcessing.PostProcessingStep.__init__	(		self,
			taskman,
			taskDict,
			oldStatus,
			previousSteps,
			postprocLib,
			jobName = None
	)

Definition at line 150 of file PostProcessing.py.

    def __init__(self,taskman,taskDict,oldStatus,previousSteps,postprocLib,jobName=None):
        self.stepName = self.__class__.__name__
        self.executedSteps = previousSteps
        self.executedSteps.append( self.stepName )
        self.taskman = taskman
        self.taskDict = taskDict
        self.dsName = taskDict['DSNAME']
        self.taskName = taskDict['TASKNAME']
        self.taskDir = '/'.join([self.dsName,self.taskName])
        self.oldStatus = oldStatus
        self.postprocLib = postprocLib
        self.jobName = jobName
        self.baseName     = '%s-%s' % (self.dsName,self.taskName)
        self.baseNameStep = '%s-%s.%s' % (self.dsName,self.taskName,self.stepName)
        self.logfile = open('%s/%s.log' % (self.taskDir,self.baseNameStep), 'w')
        self.log('Starting postprocessing:  %s' % self.__class__.__name__)
        self.log('* Data set name = %s' % self.dsName, noTime=True)
        self.log('* Task name     = %s' % self.taskName, noTime=True)
        self.log('* Old status    = %s' % getKey(TaskManager.StatusCodes,self.oldStatus), noTime=True)
        self.log('* Job name      = %s' % self.jobName, noTime=True)
        self.log('* User name     = %s' % getUserName(), noTime=True)
        self.log('* Host name     = %s\n' % os.uname()[1], noTime=True)
        pass
 

__del__()

def python.PostProcessing.PostProcessingStep.__del__

(

self

)

Definition at line 174 of file PostProcessing.py.

    def __del__(self):
        self.log('Done')
        self.logfile.close()
 

Member Function Documentation

addResult()

def python.PostProcessing.PostProcessingStep.addResult	(		self,
			resultFileName
	)

Definition at line 270 of file PostProcessing.py.

    def addResult(self,resultFileName):
        # FIXME: strip leading dirs from resultFilename, so I can pass an absolute file name?
        if not resultFileName:
            return
        resultFiles = self.taskDict['RESULTFILES']
        if resultFiles is None:
            resultFiles = ''
        if resultFileName not in resultFiles.split():
            resultFiles = ' '.join([resultFiles,resultFileName])
            self.taskman.setValue(self.dsName,self.taskName,'RESULTFILES',resultFiles)
            self.taskDict['RESULTFILES'] = resultFiles

getFileName()

def python.PostProcessing.PostProcessingStep.getFileName	(		self,
			postFix,
			stepName = '',
			absPath = False
	)

Definition at line 211 of file PostProcessing.py.

    def getFileName(self,postFix,stepName='',absPath=False):
        if stepName:
            name = '%s.%s%s' % (self.baseName,stepName,postFix)
        else:
            name = self.baseNameStep+postFix
        if absPath:
            name = '%s/%s' % (self.taskDir,name)
        return name
 

getJobConfig()

def python.PostProcessing.PostProcessingStep.getJobConfig	(		self,
			jobName = None
	)

Definition at line 259 of file PostProcessing.py.

    def getJobConfig(self,jobName=None):
        config = {}
        if not jobName:
            jobName = '*'
        configFile = glob.glob('%s/%s/%s' % (self.taskDir,jobName,'*.config.py.final.py'))
        if not configFile:
            configFile = glob.glob('%s/%s/%s' % (self.taskDir,jobName,'*.config.py'))
        if configFile:
            exec(open(configFile[0],'r'),config)    # Eval config file and put defs into config dict
        return config['jobConfig']
 

hadd()

def python.PostProcessing.PostProcessingStep.hadd	(		self,
			dir,
			outputFileName,
			inputFiles,
			maxFilesPerHadd = 100
	)

Execute staged hadd for many files.

Definition at line 242 of file PostProcessing.py.

    def hadd(self,dir,outputFileName,inputFiles,maxFilesPerHadd=100):
        """Execute staged hadd for many files."""
        if len(inputFiles)<=maxFilesPerHadd:
            self.logExec('cd %s; hadd -f %s %s' % (dir,outputFileName,' '.join(inputFiles)))
        else:
            nSteps = int(math.ceil(float(len(inputFiles))/maxFilesPerHadd))
            partFiles = []
            for index in range(nSteps):
                minIndex = index*maxFilesPerHadd
                maxIndex = min((index+1)*maxFilesPerHadd,len(inputFiles))
                stepInputFiles = inputFiles[minIndex:maxIndex]
                stepOutputFileName = '%s.part%i' % (outputFileName,index)
                self.logExec('cd %s; hadd -f %s %s' % (dir,stepOutputFileName,' '.join(stepInputFiles)))
                partFiles.append(stepOutputFileName)
            self.logExec('cd %s; hadd -f %s %s' % (dir,outputFileName,' '.join(partFiles)))
            self.logExec('cd %s; rm -f %s' % (dir,' '.join(partFiles)))            
 

jobList()

def python.PostProcessing.PostProcessingStep.jobList

(

self

)

Definition at line 220 of file PostProcessing.py.

    def jobList(self):
        try:
            l = os.listdir(self.taskDir)
        except Exception:
            l = []
        return l
 

log()

def python.PostProcessing.PostProcessingStep.log	(		self,
			msg = None,
			text = None,
			noTime = False,
			doPrint = False
	)

Definition at line 178 of file PostProcessing.py.

    def log(self,msg=None,text=None,noTime=False,doPrint=False):
        outputFiles = [self.logfile]
        if doPrint:
            outputFiles.append(sys.stdout)
        for out in outputFiles:
            if msg:
                if noTime:
                    out.write(' '*31)
                    out.write(msg)
                    out.write('\n')
                else:
                    out.write('\n%-30s %s\n' % (time.strftime('%a %b %d %X %Z %Y'),msg))
            if text:
                #out.write('\n')
                out.write(text)
                out.write('\n')
 

logExec()

def python.PostProcessing.PostProcessingStep.logExec	(		self,
			cmd,
			doPrint = False,
			checkStatusCode = True,
			errorMsg = '',
			abortOnError = True
	)

Definition at line 195 of file PostProcessing.py.

    def logExec(self,cmd,doPrint=False,checkStatusCode=True,errorMsg='',abortOnError=True):
        (status,output) = subprocess.getstatusoutput(cmd)
        status = status >> 8   # Convert to standard Unix exit code 
        self.log('Executing: %s' % cmd, output)
        if doPrint or status:
            print (output)
            print()
        if status and checkStatusCode:
            if not errorMsg:
                errorMsg = 'ERROR in postprocessing step %s while executing:\n\n%s\n' % (self.stepName,cmd)
            self.log(text='\nERROR: exit status = %i' % (status))
            self.log(text=errorMsg)
            if abortOnError:
                raise PostProcessingError(errorMsg,self.executedSteps)
        return status
 

taskFileList()

def python.PostProcessing.PostProcessingStep.taskFileList	(		self,
			fileNamePattern,
			statusName = 'POSTPROCESSING',
			jobName = None
	)

Get list of files with specific names from all jobs in a given state.

Definition at line 227 of file PostProcessing.py.

    def taskFileList(self,fileNamePattern,statusName='POSTPROCESSING',jobName=None):
        """Get list of files with specific names from all jobs in a given state."""
        oldDir = os.getcwd()
        os.chdir(self.taskDir)
        if jobName:
            statusFiles = glob.glob('%s/*.status.%s' % (jobName,statusName))
        else:
            statusFiles = glob.glob('*/*.status.%s' % (statusName))
        files = []
        for s in statusFiles:
            b = '/'.join(s.split('/')[:-1])
            files.extend(glob.glob('%s/%s' % (b,fileNamePattern)))
        os.chdir(oldDir)
        return sorted(files)
 

Member Data Documentation

baseName

python.PostProcessing.PostProcessingStep.baseName

Definition at line 162 of file PostProcessing.py.

baseNameStep

python.PostProcessing.PostProcessingStep.baseNameStep

Definition at line 163 of file PostProcessing.py.

dsName

python.PostProcessing.PostProcessingStep.dsName

Definition at line 156 of file PostProcessing.py.

executedSteps

python.PostProcessing.PostProcessingStep.executedSteps

Definition at line 152 of file PostProcessing.py.

jobName

python.PostProcessing.PostProcessingStep.jobName

Definition at line 161 of file PostProcessing.py.

logfile

python.PostProcessing.PostProcessingStep.logfile

Definition at line 164 of file PostProcessing.py.

oldStatus

python.PostProcessing.PostProcessingStep.oldStatus

Definition at line 159 of file PostProcessing.py.

postprocLib

python.PostProcessing.PostProcessingStep.postprocLib

Definition at line 160 of file PostProcessing.py.

stepName

python.PostProcessing.PostProcessingStep.stepName

Definition at line 151 of file PostProcessing.py.

taskDict

python.PostProcessing.PostProcessingStep.taskDict

Definition at line 155 of file PostProcessing.py.

taskDir

python.PostProcessing.PostProcessingStep.taskDir

Definition at line 158 of file PostProcessing.py.

taskman

python.PostProcessing.PostProcessingStep.taskman

Definition at line 154 of file PostProcessing.py.

taskName

python.PostProcessing.PostProcessingStep.taskName

Definition at line 157 of file PostProcessing.py.

---

The documentation for this class was generated from the following file:

• PostProcessing.py