getPipeDate Namespace Reference

Classes
class	scanDataThread

Functions
def	sensorToNumber (stave, side, module)
def	findFiles (searchFolderPath)
def	appendEntries (fileName, dataType, dataFolder, dateEntries, largestDates, firstDate, lastDate, borkDate)
def	scanData (dataType)
def	main ()

Function Documentation

appendEntries()

def getPipeDate.appendEntries	(		fileName,
			dataType,
			dataFolder,
			dateEntries,
			largestDates,
			firstDate,
			lastDate,
			borkDate
	)

Definition at line 61 of file getPipeDate.py.

def appendEntries(fileName, dataType, dataFolder, dateEntries, largestDates, firstDate, lastDate, borkDate):
 
    try:
        with open(fileName, 'r') as rawFiles:
 
            returnList = [firstDate, largestDates,
                          borkDate, dateEntries, firstDate]
 
            print(firstDate)
 
            index_Jennet = 0
            # read in each line of fileName and do things to it. Does this work with empty files? I'm guessing so.
            for dataLine in rawFiles:
 
                index_Jennet = index_Jennet + 1
                # We actually have two types of data file from DDV, one containing lots of commas, one which doesn't. The len(commasplit) thing is how I
                # differentiate between the two types.
 
                if len(dataLine) > 5:
                    commaSplit = dataLine.split(',')
 
                    if '!!!!' in dataLine:  # Our entry is blank, leg it!
                        return returnList
 
                    if len(commaSplit) < 2:
                        print("Dead")
                    else:
                        # from getDataSafely, format here splits data based on !!! between DCS groups.
 
                        moduleList = dataLine.split('!!!')
 
                        for module in moduleList:  # Hah, look at me using accurate labels for my variables!
 
                            elements = module.split(',')
                            name = elements[0]
                            A = "A"
                            B = "A"
                            if len(name) > 9:
                                A = name[4:6]
                                B = name[7]
 
                            moduleName = 'LI_S' + str(A) + '_' + str(B)
                            try:
                                moduleNumber = int(2 * (int(A)-1)-1)
                                if B == 'C':
                                    moduleNumber += 4
 
                                outName = dataFolder+dataType+'/' + moduleName + '.ssv'
 
                                # check if file exists, make it if not, append it if so.
                                mode = 'a' if os.path.isfile(outName) else 'w'
                                with open(outName, mode) as output:
                                    position = 0
 
                                    # I'm like 90% sure each element is a single time (so, time and sensor reading). This makes it slow.
                                    for element in elements:
 
                                        if position != 0:  # the first element is the DCS group name; not hella useful.
 
                                            tempLines = element.split()
                                            date = tempLines[1].split('-')
                                            time1 = tempLines[2].split(':')
                                            data = tempLines[0]
 
                                            dateTime = datetime.datetime(int(date[2]), int(date[1]), int(
                                                date[0]), int(time1[0]), int(time1[1]), int(time1[2]), int(time1[3]))
                                            # true if data stuff
                                            if (dateTime > lastDate+datetime.timedelta(hours=2)) or (dateTime < firstDate - datetime.timedelta(hours=2)):
                                                returnList[2] = lastDate
                                                print("\nBORK BORK BORK! Error in date of " + fileName +
                                                      " for " + dataType + " borkDate " + str(returnList[2]))
                                                print(
                                                    "First line reads " + dataLine + "\n")
                                                print(
                                                    "Should read " + moduleName+"_"+dataType+"\n")
 
                                                with open(dataFolder+dataType+"BORKBORKBORK.txt", 'w') as borkRepork:
                                                    borkRepork.write("filename " + fileName + "date range " + str(
                                                        firstDate) + "-" + str(lastDate) + " first line " + dataLine + '\n')
                                                return returnList
 
                                            # if time in right range, output.
                                            if dateTime > returnList[1][moduleNumber]:
                                                returnList[3][moduleNumber] += 1
                                                outputLine = moduleName + "_" + dataType + " " + \
                                                    str(dateTime) + " " + str(time.mktime(
                                                        dateTime.timetuple())) + " " + str(data) + "\n"
 
                                                returnList[1][moduleNumber] = dateTime
 
                                                output.write(outputLine)
 
                                        position += 1
 
                                except:  # we failed at something, so print out an error message and run away screaming
                                    returnList[2] = lastDate
                                    print("Something broke :( \n")
                                    print("Could be an error in data values of " + fileName +
                                          " for " + dataType + " borkDate " + str(returnList[2]))
                                    print("First line reads " + name + "\n")
 
                                    with open(dataFolder+dataType+"BORKBORKBORK.txt", 'w') as borkRepork:
                                        borkRepork.write("filename " + fileName + "date range " + str(
                                            firstDate) + "-" + str(lastDate) + " first line " + dataLine + '\n')
                                    print(
                                        "Try running again... often this is an issue with the connection to eos")
                                    return returnList
 
    except IOError:
        print("could not open file " + str(fileName) + " fool")
        # So this probably crashes if filename can't be opened. Try not to let that happen.
        return firstDate
 
    return returnList
 
 

findFiles()

def getPipeDate.findFiles

(

searchFolderPath

)

Definition at line 34 of file getPipeDate.py.

def findFiles(searchFolderPath):  # find all files in searchFolderPath, return as list
 
    try:
        os.chdir(searchFolderPath)
    except IOError:
        print('No entries in ' + searchFolderPath)
        return -1
 
    todaysList = []
 
    # SOMEHOW, this searches the search Folder and returns a list of contained files in well... files.
    for src_dir, dirs, files in os.walk(searchFolderPath):
        # i parse through the files, I think. For each, I check if there's a reference to it in the
        # diary entry for the day. If not, I go ahead and append the figure to the relevent diary entry.
        for file_ in files:
            sortedFile = os.path.join(src_dir, file_)
            todaysList.append(str(sortedFile))
        # now we've got a list containing all files we want to add, sort it alphabetically and add them.
    todaysList.sort()
    return todaysList
 
# Function that actually reads in data from fileName, saves it to... somewhere. Probably dataFolder/datatype/modulename.ssv . It also returns
# stuff like the most recent dates for all DCS groups (endDates), and number of datapoints found for each DCS group (tempDataNumbers).
# Probably says something if it can't read the data as well. Gonna be honest, DDV is a bit of a nightmare and I've spent too long messing
# around with it to remember what my bodges are.
 
 

main()

def getPipeDate.main

(

)

Definition at line 411 of file getPipeDate.py.

def main():
 
    thread1 = scanDataThread(1, 'ENV_TT')
    thread1.start()
 
 

scanData()

def getPipeDate.scanData

(

dataType

)

Definition at line 177 of file getPipeDate.py.

def scanData(dataType):
 
    # home directory definition
    fillerDate = datetime.datetime(2000, 1, 1, 1, 1, 1, 1)
    returnList = [fillerDate, fillerDate, fillerDate, False, "LI_S00_0_M0"]
    # JENNET setsfile paths
    homeDirectory = os.path.expanduser(
        '/eos/atlas/user/j/jdickins/Pixel/LeakageCurrent/')
 
    # define path to folder
    dataFolder = homeDirectory + "/IBLData/processedData/"
    inputFolder = homeDirectory + "/IBLData/rawData/"+dataType+"/"
    entriesFolder = homeDirectory + "/IBLData/rawData/entriesPerDay/" + dataType + "/"
 
    # make directories if they don't exist
    if not os.path.exists(dataFolder+dataType):
        os.mkdir(dataFolder+dataType)
 
    if not os.path.exists(entriesFolder):
        os.mkdir(entriesFolder)
 
    # declare some stuff, so it's in scope when we need it
    startingDates = []
    endDates = []
    filesProcessed = []
    dataPoints = []
    dailyPoints = []
    smallestEndDate = fillerDate
 
    # This section checks to see if there's a file containing the dates we've worked up till.
    if not os.path.exists(dataFolder+dataType+".txt"):
 
        # No dates file found, so create one.
        print("No any file found! at " + dataFolder +
              dataType + " Making default values")
        # set default max and min values for each sensor
 
        with open(dataFolder+dataType+".txt", 'w') as datesFile:
            firstTempDate = datetime.datetime(2015, 5, 1, 0, 0, 0, 0)
            lastTempDate = datetime.datetime(2015, 5, 1, 0, 0, 0, 1)
            smallestEndDate = lastTempDate
 
            for stave in range(1, 15):
                staveString = str(stave)
                if stave < 10:
                    staveString = "0"+str(stave)
                for side in ['A', 'C']:
                    moduleName = 'LI_S' + str(staveString) + '_' + side
                    datesFile.write(
                        moduleName + " " + str(firstTempDate) + " " + str(lastTempDate) + " 0 0\n")
                    startingDates.append(firstTempDate)
                    endDates.append(lastTempDate)
                    filesProcessed.append(0)
                    dataPoints.append(0)
                    dailyPoints.append(0)
 
    else:  # dates file exists, so read dates for each DCS group
        print("Found " + dataFolder+dataType+".txt")
        with open(dataFolder+dataType+".txt", 'r') as datesFile:
 
            holder = 0
 
            for dateLine in datesFile:  # read each line in, each line should correspond to one DCS group
 
                tempDatesLine = dateLine.split()
                filesProcessed.append(int(tempDatesLine[5]))
                dataPoints.append(int(tempDatesLine[6]))
                dailyPoints.append(0)
 
                firstTemp = tempDatesLine[1].split('-')
                lastTemp = tempDatesLine[3].split('-')
 
                firstTempTime = tempDatesLine[2].split(':')
                lastTempTime = tempDatesLine[4].split(':')
 
                firstTempTimes = firstTempTime[2].split('.')
                lastTempTimes = lastTempTime[2].split('.')
 
                if len(firstTempTimes) < 2:
                    firstTempTimes.append(0)
                if len(lastTempTimes) < 2:
                    lastTempTimes.append(0)
 
                firstTempDate = datetime.datetime(int(firstTemp[0]), int(firstTemp[1]), int(firstTemp[2]), int(
                    firstTempTime[0]), int(firstTempTime[1]), int(firstTempTimes[0]), int(firstTempTimes[1]))
                lastTempDate = datetime.datetime(int(lastTemp[0]), int(lastTemp[1]), int(lastTemp[2]), int(
                    lastTempTime[0]), int(lastTempTime[1]), int(lastTempTimes[0]), int(lastTempTimes[1]))
 
                startingDates.append(firstTempDate)
                endDates.append(lastTempDate)
 
                if holder == 0:
                    returnList[0] = firstTempDate
                    returnList[1] = lastTempDate
                    smallestEndDate = lastTempDate
 
                else:
                    if firstTempDate < returnList[0]:
                        returnList[0] = firstTempDate
                    if lastTempDate > returnList[1]:
                        returnList[1] = lastTempDate
                    if lastTempDate < smallestEndDate:
                        smallestEndDate = lastTempDate
 
                holder += 1
 
        print("Investigating " + dataType + " from " + str(smallestEndDate))
 
    holder = 0
 
    # call function to return list of all files in input folder
    fileList = findFiles(inputFolder)
 
    firstTempDate = startingDates[0]
    lastTempDate = endDates[0]
    fileNumber = 0
 
    # iterate through all files from file list, opening them if they're in the time period of interest.
    for fileName in fileList:
 
        # print(fileName)
 
        end = len(fileName)
        endDate1 = fileName[end-4:end]
        endDate2 = fileName[end-7:end-6]
 
        # check file ends with text, and contains underscore in expected place; Could make this more rigorous
        if endDate1 == '.txt' and endDate2 == '_':
 
            startDate = fileName[end-23:end-15]
            endDate = fileName[end-12:end-4]
            endDateSplit = endDate.split('_')
            endDateFile = datetime.datetime(
                2000+int(endDateSplit[0]), int(endDateSplit[1]), int(endDateSplit[2]), 0, 0, 0, 1)
 
            startDateSplit = startDate.split('_')
            startDateFile = datetime.datetime(
                2000+int(startDateSplit[0]), int(startDateSplit[1]), int(startDateSplit[2]), 0, 0, 0, 1)
 
            if endDateFile > smallestEndDate:  # data from region of interest
                # APPEND FILES DOING THINGS HERE
 
                lastTempDate = endDateFile
                [firstTempDate, endDates, fillerDate, tempDataNumbers, smallestEndDate] = appendEntries(
                    fileName, dataType, dataFolder, dailyPoints, endDates, startDateFile, endDateFile, fillerDate)
                # append entries called here. Editing of data files done at that location.
 
                # 112 DCS groups for IBL, if you're doing something else, change that number.#
                for i in range(0, 28):
                    filesProcessed[i] += 1  # number of files
                    # number of data points for each DCS group
                    dataPoints[i] += tempDataNumbers[i]
 
                holderX = 0
 
                # this does something. Probably recording number of data points per day for each DCS group
                for stave in range(1, 15):
                    staveString = str(stave)
                    if stave < 10:
                        staveString = "0"+str(stave)
                    for side in ['A', 'C']:
                        moduleName = 'LI_S' + str(staveString) + '_' + side
                        outName = entriesFolder + moduleName + ".txt"
                        dataLine = str(startDate) + " " + \
                            str(tempDataNumbers[holderX]) + "\n"
                        tempDataNumbers[holderX] = 0
 
                        mode = 'a' if os.path.isfile(outName) else 'w'
                        with open(outName, mode) as output:
                            output.write(dataLine)
                        holderX += 1
 
                # check if the list is bork-a-dorked. If borked, save and break
                if returnList[2] != fillerDate:
                    returnList[2] = fillerDate
                    with open(dataFolder+dataType+".txt", 'w') as datesFile:
                        tempHolder = 0
                        for stave in range(1, 15):
                            staveString = str(stave)
                            if stave < 10:
                                staveString = "0"+str(stave)
                            for side in ['A', 'C']:
                                moduleName = 'LI_S' + \
                                    str(staveString) + '_' + side
                                datesFile.write(moduleName + " " + str(startingDates[tempHolder]) + " " + str(
                                    endDates[tempHolder]) + " " + str(filesProcessed[tempHolder]) + " " + str(dataPoints[tempHolder]) + "\n")
                                tempHolder += 1
 
                    return returnList
 
        sys.stdout.flush()
        fileNumber += 1
 
    # open file, go through lists, if value < smallest, set to that,
 
    # if number bigger, set to that
    # if any value smaller than previous largest value, bork bork!
 
    holder += 1
 
    with open(dataFolder+dataType+".txt", 'w') as datesFile:
        tempHolder = 0
        for staveX in range(1, 15):
            staveStringX = str(staveX)
            if staveX < 10:
                staveStringX = "0"+str(staveX)
            for sideX in ['A', 'C']:
                moduleNameX = 'LI_S' + str(staveStringX) + '_' + sideX
                datesFile.write(moduleNameX + " " + str(startingDates[tempHolder]) + " " + str(
                    endDates[tempHolder]) + " " + str(filesProcessed[tempHolder]) + " " + str(dataPoints[tempHolder]) + "\n")
                tempHolder += 1
 
    returnList[3] = True
 
    return returnList
 
 

sensorToNumber()

def getPipeDate.sensorToNumber	(		stave,
			side,
			module
	)

Definition at line 26 of file getPipeDate.py.

def sensorToNumber(stave, side, module):
 
    number = 8 * (int(stave)-1) + module
    if side == 'C':
        number += 4
    return number-1