python.getFileLists.findTCTFiles Class Reference

Collaboration diagram for python.getFileLists.findTCTFiles:

Public Member Functions
	__init__ (self, rDir, vDir, checkAge=False)
	checkFileAge (self, path)
	hasLogfile (self, ref, p, fl)
	addNew (self, fileList, file)
	findBetween (self, s, first, last)
	getTCTChainInfo (self, tci)
	getChains (self)
	getCommonChains (self)
	findFilesInDir (self, dir, filename)
	findFiles (self, pattern)
	extractCastorPath (self, rttxmlfile, pattern)

Public Attributes
	hasLogfile

Protected Attributes
	_checkAge = checkAge
	_rDir = rDir
	_vDir = vDir
#exits also in reference set	_commonDirs = dict()
list	_vFiles = []
list	_rFiles = []

Detailed Description

Definition at line 32 of file getFileLists.py.

Constructor & Destructor Documentation

__init__()

python.getFileLists.findTCTFiles.__init__	(		self,
			rDir,
			vDir,
			checkAge = False )

Definition at line 33 of file getFileLists.py.

    def __init__(self, rDir, vDir, checkAge = False):
        self._checkAge = checkAge
        self._rDir = rDir
        self._vDir = vDir
        self._commonDirs = dict()
        self._vFiles = []
        self._rFiles = []
        

Member Function Documentation

addNew()

python.getFileLists.findTCTFiles.addNew	(		self,
			fileList,
			file )

Definition at line 73 of file getFileLists.py.

    def addNew(self,fileList,file):
        newSplit=file.split("/")
        newStream=newSplit[-3]
        newFile=newSplit[-1]
        for old in fileList:
            oldSplit=old.split("/")
            if oldSplit[-1]==newFile and oldSplit[-3]==newStream:
                return
        #print ("Adding",file)
        fileList+=[file,]
        return
 

checkFileAge()

python.getFileLists.findTCTFiles.checkFileAge	(		self,
			path )

Definition at line 41 of file getFileLists.py.

    def checkFileAge(self,path):
        try:
            fileTime = os.stat(path)[8]
        except Exception:
            return
        age = time() - fileTime
 
        if age > 86400: #More than 24h
            print ("WARNING! File %s is more than %d day(s) old" % (path, int(age/86400.)))
        return
 
 

extractCastorPath()

python.getFileLists.findTCTFiles.extractCastorPath	(		self,
			rttxmlfile,
			pattern )

Definition at line 247 of file getFileLists.py.

    def extractCastorPath(self,rttxmlfile,pattern):
        res=[]
        dom=parse(rttxmlfile)
        archfiles=dom.getElementsByTagName("archivefile")
        for af in archfiles:
            cpEle=af.getElementsByTagName("destination")
            
            if not cpEle:
                s = af.firstChild.nodeValue
                json_acceptable_string = s.replace("'", "\"")
                d = json.loads(json_acceptable_string)
                castorpath=d['src']
            else:
                castorpath=cpEle[0].childNodes[0].data.strip()
            #print (castorpath)
            if len(pattern.findall(castorpath)):
                res+=[castorpath,]
        del dom
        return res
 
 
 

findBetween()

python.getFileLists.findTCTFiles.findBetween	(		self,
			s,
			first,
			last )

Definition at line 85 of file getFileLists.py.

    def findBetween(self, s, first, last ):
        try:
            start = s.index( first ) + len( first )
            end = s.index( last, start )
            return s[start:end]
        except ValueError:
            return ""
 

findFiles()

python.getFileLists.findTCTFiles.findFiles	(		self,
			pattern )

Definition at line 222 of file getFileLists.py.

    def findFiles(self,pattern):
        result=dict()
        if len(self._commonDirs)==0:
            self.getCommonChains()
        #print (self._commonDirs)
        
        for (name,(ref,val)) in self._commonDirs.items():
            reffiles=self.findFilesInDir(ref.directory,pattern)
            valfiles=self.findFilesInDir(val.directory,pattern)
 
            result[name]=()
            #Find pairs:
            for rf in reffiles:
                rfN=rf.split("/")[-1]
                #print ("Checking:",rfN)
                for vf in valfiles:
                    vfN=vf.split("/")[-1]
                    if (vfN == rfN):
                        result[name]+=((rf,vf),)
                        del vf
                        break
        return result
 
 
 

findFilesInDir()

python.getFileLists.findTCTFiles.findFilesInDir	(		self,
			dir,
			filename )

Definition at line 207 of file getFileLists.py.

    def findFilesInDir(self,dir,filename):
        #print ("Searching dir",dir)
        pattern=re.compile(filename)
        result=[]
        ls=os.listdir(dir)
        for f in ls:
            if len(pattern.findall(f)):
                #print ("Found",f)
                self.addNew(result,dir+"/"+f)
        if "rttjobinfo.xml" in ls:
            castorfiles=self.extractCastorPath(dir+"/rttjobinfo.xml",pattern)
            for f in castorfiles:
                self.addNew(result,f)
        return result
            

getChains()

python.getFileLists.findTCTFiles.getChains

(

self

)

Definition at line 124 of file getFileLists.py.

    def getChains(self):
        print ("Input directory:",self._rDir)
        print ("Searching for TCT sub-directories")
        os.path.walk(self._rDir,self.hasLogfile,True)
        print ("Found ",len(self._commonDirs),"directories")
        return self._commonDirs
              
        

getCommonChains()

python.getFileLists.findTCTFiles.getCommonChains

(

self

)

Definition at line 132 of file getFileLists.py.

    def getCommonChains(self):
        print ("Searching for compatible TCT directories ..." )
        allEvents=0
        
        os.path.walk(self._rDir, self.hasLogfile, True)   # Reference directory
        os.path.walk(self._vDir, self.hasLogfile, False)  # Validation directory
        
        names = self._commonDirs.keys()
        for tctname in names:
            if (tctname.startswith("LatestRun") or tctname.endswith("_MP") or tctname.endswith("IDCosmic0") or tctname.endswith("_PHYSVAL") or tctname.endswith("Derived_Outputs")):
                print ("skipping "+tctname)
                self._commonDirs.pop(tctname)
                continue
 
            tcis = self._commonDirs[tctname]
 
            if len(tcis) != 2:
                self._commonDirs.pop(tctname)
                continue
 
            ref = tcis[0]
            val = tcis[1]
 
            formats = ["RDO", "ESD", "AOD", "TAG"] # or anything matching them, though more than one will cause problems! /CO
            
            refEvents = self.getTCTChainInfo(ref)
            if refEvents is None or len(refEvents) == 0:
                print ("No events found in",ref.logfile)
                self._commonDirs.pop(tctname)
                continue
            
            valEvents = self.getTCTChainInfo(val)
            if valEvents is None or len(valEvents) == 0:
                print ("No events found in", val.logfile)
                continue
 
            if (valEvents == refEvents):
                for file in refEvents:
                    allEvents += refEvents[file]
                print ("TCT %s output seems compatible for ref and chk:" % (tctname))
                for format in refEvents:
                    print ("%-70s: ref: %d events, val: %d events" % (format, refEvents[format], valEvents[format]))
            else:
                # workaround for when names of test output files change between two rels 
                print ("The (names of the) output files differ in some way:")
                print (refEvents)
                print (valEvents)
                print ("Will now attempt to match the files by type")
                matchFound = False
                for refFormat in refEvents:
                    if matchFound:
                        break # exit loop if a match was found
                    valFormat = "MOCK"
                    for vFormat in valEvents:
                        #print (vFormat)
                        for f in formats:
                            if matchFound:
                                break
                            if f in refFormat and f in vFormat:
                                valFormat = vFormat
                                print ("Both are %s: %s, %s" % (f, refFormat, valFormat))
                                matchFound = True
                    print ("  %s, ref: %d, val: %d" % (format, refEvents[refFormat], valEvents[valFormat]))
                if not matchFound:
                    # don't compare the files for this then!
                    self._commonDirs.pop(tctname)
                    print ("TCT %s is NOT compatible, outputs different number of events for at least one format:" % tctname)
                    print ()
                
        print ("Found %i compatible TCT chains with at total of %i events" % (len(self._commonDirs), allEvents))
        #rint "Done"
        #sys.exit(0)
        return self._commonDirs
            
 

getTCTChainInfo()

python.getFileLists.findTCTFiles.getTCTChainInfo	(		self,
			tci )

Definition at line 93 of file getFileLists.py.

    def getTCTChainInfo(self,tci):
        validationStartLine = "INFO Validating output files"
        
        try:
            lf = open(tci.logfile,"r")
        except Exception:
            return None
 
        # loop through the log file and find all the output files being validated
        foundFileValidationStart = False
        nextLineHasEventCount = False
        lastOutputFileName = ""
        for l in lf:
            tci.loglines += 1
            if not foundFileValidationStart:
                if validationStartLine in l:
                    foundFileValidationStart = True
            elif "Testing event count..." in l:                
                format = self.findBetween(l, "INFO", "Testing event count...").strip()
                tci.outputDict[format] = -1
                nextLineHasEventCount = True
                lastOutputFileName = format
            elif nextLineHasEventCount:
                tci.outputDict[lastOutputFileName] = int(self.findBetween(l, "Event counting test passed (", " events)."))
                nextLineHasEventCount = False
        #print ("Done parsing %s, found the following output files:" % tci.logfile)
        #for file in tci.outputDict:
        #    print ("  %s : %d events" % (file, tci.outputDict[format]))
        lf.close()
        return tci.outputDict
 

hasLogfile()

python.getFileLists.findTCTFiles.hasLogfile	(		self,
			ref,
			p,
			fl )

Definition at line 53 of file getFileLists.py.

    def hasLogfile(self,ref,p,fl):
        #print ("Checking",p)
        for f in fl:
            logpath=p+'/'+f
            #print ("\t",f)
            if os.path.isfile(logpath) and f.endswith("_log") and not f.endswith("_script_log"):
                #name=f[:-4].rstrip(digits) #Was needed for old rtt version, job index number appended
                name=f[:-4]
                tci=TCTChainInfo(p,logpath)
                if ref: #Reference TCT (the first one)
                    if name in self._commonDirs:
                        print ("ERROR: Duplicate directory name",name)
                    else:
                        self._commonDirs[name]=(tci,)
                else: #validation (the second one)
                    if name in self._commonDirs: #exits also in reference set
                        self._commonDirs[name]+=(tci,)
                return
                  
            

Member Data Documentation

_checkAge

python.getFileLists.findTCTFiles._checkAge = checkAge

protected

Definition at line 34 of file getFileLists.py.

_commonDirs

python.getFileLists.findTCTFiles._commonDirs = dict()

protected

Definition at line 37 of file getFileLists.py.

_rDir

python.getFileLists.findTCTFiles._rDir = rDir

protected

Definition at line 35 of file getFileLists.py.

_rFiles

list python.getFileLists.findTCTFiles._rFiles = []

protected

Definition at line 39 of file getFileLists.py.

_vDir

python.getFileLists.findTCTFiles._vDir = vDir

protected

Definition at line 36 of file getFileLists.py.

_vFiles

list python.getFileLists.findTCTFiles._vFiles = []

protected

Definition at line 38 of file getFileLists.py.

hasLogfile

python.getFileLists.findTCTFiles.hasLogfile

Definition at line 127 of file getFileLists.py.

---

The documentation for this class was generated from the following file:

• getFileLists.py