python.getFileLists.findTCTFiles Class Reference

Collaboration diagram for python.getFileLists.findTCTFiles:

Public Member Functions
def	__init__ (self, rDir, vDir, checkAge=False)
def	checkFileAge (self, path)
def	hasLogfile (self, ref, p, fl)
def	addNew (self, fileList, file)
def	findBetween (self, s, first, last)
def	getTCTChainInfo (self, tci)
def	getChains (self)
def	getCommonChains (self)
def	findFilesInDir (self, dir, filename)
def	findFiles (self, pattern)
def	extractCastorPath (self, rttxmlfile, pattern)

Private Attributes
	_checkAge
	_rDir
	_vDir
	_commonDirs
	_vFiles
	_rFiles

Detailed Description

Definition at line 32 of file getFileLists.py.

Constructor & Destructor Documentation

__init__()

def python.getFileLists.findTCTFiles.__init__	(		self,
			rDir,
			vDir,
			checkAge = False
	)

Definition at line 33 of file getFileLists.py.

    def __init__(self, rDir, vDir, checkAge = False):
        self._checkAge = checkAge
        self._rDir = rDir
        self._vDir = vDir
        self._commonDirs = dict()
        self._vFiles = []
        self._rFiles = []
        

Member Function Documentation

addNew()

def python.getFileLists.findTCTFiles.addNew	(		self,
			fileList,
			file
	)

Definition at line 73 of file getFileLists.py.

    def addNew(self,fileList,file):
        newSplit=file.split("/")
        newStream=newSplit[-3]
        newFile=newSplit[-1]
        for old in fileList:
            oldSplit=old.split("/")
            if oldSplit[-1]==newFile and oldSplit[-3]==newStream:
                return
        #print ("Adding",file)
        fileList+=[file,]
        return
 

checkFileAge()

def python.getFileLists.findTCTFiles.checkFileAge	(		self,
			path
	)

Definition at line 41 of file getFileLists.py.

    def checkFileAge(self,path):
        try:
            fileTime = os.stat(path)[8]
        except Exception:
            return
        age = time() - fileTime
 
        if age > 86400: #More than 24h
            print ("WARNING! File %s is more than %d day(s) old" % (path, int(age/86400.)))
        return
 
 

extractCastorPath()

def python.getFileLists.findTCTFiles.extractCastorPath	(		self,
			rttxmlfile,
			pattern
	)

Definition at line 247 of file getFileLists.py.

    def extractCastorPath(self,rttxmlfile,pattern):
        res=[]
        dom=parse(rttxmlfile)
        archfiles=dom.getElementsByTagName("archivefile")
        for af in archfiles:
            cpEle=af.getElementsByTagName("destination")
            
            if not cpEle:
                s = af.firstChild.nodeValue
                json_acceptable_string = s.replace("'", "\"")
                d = json.loads(json_acceptable_string)
                castorpath=d['src']
            else:
                castorpath=cpEle[0].childNodes[0].data.strip()
            #print (castorpath)
            if len(pattern.findall(castorpath)):
                res+=[castorpath,]
        del dom
        return res
 
 
 

findBetween()

def python.getFileLists.findTCTFiles.findBetween	(		self,
			s,
			first,
			last
	)

Definition at line 85 of file getFileLists.py.

    def findBetween(self, s, first, last ):
        try:
            start = s.index( first ) + len( first )
            end = s.index( last, start )
            return s[start:end]
        except ValueError:
            return ""
 

findFiles()

def python.getFileLists.findTCTFiles.findFiles	(		self,
			pattern
	)

Definition at line 222 of file getFileLists.py.

    def findFiles(self,pattern):
        result=dict()
        if len(self._commonDirs)==0:
            self.getCommonChains()
        #print (self._commonDirs)
        
        for (name,(ref,val)) in six.iteritems (self._commonDirs):
            reffiles=self.findFilesInDir(ref.directory,pattern)
            valfiles=self.findFilesInDir(val.directory,pattern)
 
            result[name]=()
            #Find pairs:
            for rf in reffiles:
                rfN=rf.split("/")[-1]
                #print ("Checking:",rfN)
                for vf in valfiles:
                    vfN=vf.split("/")[-1]
                    if (vfN == rfN):
                        result[name]+=((rf,vf),)
                        del vf
                        break
        return result
 
 
 

findFilesInDir()

def python.getFileLists.findTCTFiles.findFilesInDir	(		self,
			dir,
			filename
	)

Definition at line 207 of file getFileLists.py.

    def findFilesInDir(self,dir,filename):
        #print ("Searching dir",dir)
        pattern=re.compile(filename)
        result=[]
        ls=os.listdir(dir)
        for f in ls:
            if len(pattern.findall(f)):
                #print ("Found",f)
                self.addNew(result,dir+"/"+f)
        if "rttjobinfo.xml" in ls:
            castorfiles=self.extractCastorPath(dir+"/rttjobinfo.xml",pattern)
            for f in castorfiles:
                self.addNew(result,f)
        return result
            

getChains()

def python.getFileLists.findTCTFiles.getChains

(

self

)

Definition at line 124 of file getFileLists.py.

    def getChains(self):
        print ("Input directory:",self._rDir)
        print ("Searching for TCT sub-directories")
        os.path.walk(self._rDir,self.hasLogfile,True)
        print ("Found ",len(self._commonDirs),"directories")
        return self._commonDirs
              
        

getCommonChains()

def python.getFileLists.findTCTFiles.getCommonChains

(

self

)

Definition at line 132 of file getFileLists.py.

    def getCommonChains(self):
        print ("Searching for compatible TCT directories ..." )
        allEvents=0
        
        os.path.walk(self._rDir, self.hasLogfile, True)   # Reference directory
        os.path.walk(self._vDir, self.hasLogfile, False)  # Validation directory
        
        names = self._commonDirs.keys()
        for tctname in names:
            if (tctname.startswith("LatestRun") or tctname.endswith("_MP") or tctname.endswith("IDCosmic0") or tctname.endswith("_PHYSVAL") or tctname.endswith("Derived_Outputs")):
                print ("skipping "+tctname)
                self._commonDirs.pop(tctname)
                continue
 
            tcis = self._commonDirs[tctname]
 
            if len(tcis) != 2:
                self._commonDirs.pop(tctname)
                continue
 
            ref = tcis[0]
            val = tcis[1]
 
            formats = ["RDO", "ESD", "AOD", "TAG"] # or anything matching them, though more than one will cause problems! /CO
            
            refEvents = self.getTCTChainInfo(ref)
            if refEvents is None or len(refEvents) == 0:
                print ("No events found in",ref.logfile)
                self._commonDirs.pop(tctname)
                continue
            
            valEvents = self.getTCTChainInfo(val)
            if valEvents is None or len(valEvents) == 0:
                print ("No events found in", val.logfile)
                continue
 
            if (valEvents == refEvents):
                for file in refEvents:
                    allEvents += refEvents[file]
                print ("TCT %s output seems compatible for ref and chk:" % (tctname))
                for format in refEvents:
                    print ("%-70s: ref: %d events, val: %d events" % (format, refEvents[format], valEvents[format]))
            else:
                # workaround for when names of test output files change between two rels 
                print ("The (names of the) output files differ in some way:")
                print (refEvents)
                print (valEvents)
                print ("Will now attempt to match the files by type")
                matchFound = False
                for refFormat in refEvents:
                    if matchFound:
                        break # exit loop if a match was found
                    valFormat = "MOCK"
                    for vFormat in valEvents:
                        #print (vFormat)
                        for f in formats:
                            if matchFound:
                                break
                            if f in refFormat and f in vFormat:
                                valFormat = vFormat
                                print ("Both are %s: %s, %s" % (f, refFormat, valFormat))
                                matchFound = True
                    print ("  %s, ref: %d, val: %d" % (format, refEvents[refFormat], valEvents[valFormat]))
                if not matchFound:
                    # don't compare the files for this then!
                    self._commonDirs.pop(tctname)
                    print ("TCT %s is NOT compatible, outputs different number of events for at least one format:" % tctname)
                    print ()
                
        print ("Found %i compatible TCT chains with at total of %i events" % (len(self._commonDirs), allEvents))
        #rint "Done"
        #sys.exit(0)
        return self._commonDirs
            
 

getTCTChainInfo()

def python.getFileLists.findTCTFiles.getTCTChainInfo	(		self,
			tci
	)

Definition at line 93 of file getFileLists.py.

    def getTCTChainInfo(self,tci):
        validationStartLine = "INFO Validating output files"
        
        try:
            lf = open(tci.logfile,"r")
        except Exception:
            return None
 
        # loop through the log file and find all the output files being validated
        foundFileValidationStart = False
        nextLineHasEventCount = False
        lastOutputFileName = ""
        for l in lf:
            tci.loglines += 1
            if not foundFileValidationStart:
                if validationStartLine in l:
                    foundFileValidationStart = True
            elif "Testing event count..." in l:                
                format = self.findBetween(l, "INFO", "Testing event count...").strip()
                tci.outputDict[format] = -1
                nextLineHasEventCount = True
                lastOutputFileName = format
            elif nextLineHasEventCount:
                tci.outputDict[lastOutputFileName] = int(self.findBetween(l, "Event counting test passed (", " events)."))
                nextLineHasEventCount = False
        #print ("Done parsing %s, found the following output files:" % tci.logfile)
        #for file in tci.outputDict:
        #    print ("  %s : %d events" % (file, tci.outputDict[format]))
        lf.close()
        return tci.outputDict
 

hasLogfile()

def python.getFileLists.findTCTFiles.hasLogfile	(		self,
			ref,
			p,
			fl
	)

Definition at line 53 of file getFileLists.py.

    def hasLogfile(self,ref,p,fl):
        #print ("Checking",p)
        for f in fl:
            logpath=p+'/'+f
            #print ("\t",f)
            if os.path.isfile(logpath) and f.endswith("_log") and not f.endswith("_script_log"):
                #name=f[:-4].rstrip(digits) #Was needed for old rtt version, job index number appended
                name=f[:-4]
                tci=TCTChainInfo(p,logpath)
                if ref: #Reference TCT (the first one)
                    if name in self._commonDirs:
                        print ("ERROR: Duplicate directory name",name)
                    else:
                        self._commonDirs[name]=(tci,)
                else: #validation (the second one)
                    if name in self._commonDirs: #exits also in reference set
                        self._commonDirs[name]+=(tci,)
                return
                  
            

Member Data Documentation

_checkAge

python.getFileLists.findTCTFiles._checkAge

private

Definition at line 34 of file getFileLists.py.

_commonDirs

python.getFileLists.findTCTFiles._commonDirs

private

Definition at line 37 of file getFileLists.py.

_rDir

python.getFileLists.findTCTFiles._rDir

private

Definition at line 35 of file getFileLists.py.

_rFiles

python.getFileLists.findTCTFiles._rFiles

private

Definition at line 39 of file getFileLists.py.

_vDir

python.getFileLists.findTCTFiles._vDir

private

Definition at line 36 of file getFileLists.py.

_vFiles

python.getFileLists.findTCTFiles._vFiles

private

Definition at line 38 of file getFileLists.py.

---

The documentation for this class was generated from the following file:

• getFileLists.py