25 from os.path
import expanduser
31 number = 8 * (
int(stave)-1) + module
39 os.chdir(searchFolderPath)
41 print(
'No entries in ' + searchFolderPath)
47 for src_dir, dirs, files
in os.walk(searchFolderPath):
51 sortedFile = os.path.join(src_dir, file_)
52 todaysList.append(
str(sortedFile))
61 def appendEntries(fileName,dataType,dataFolder,dateEntries,largestDates,firstDate,lastDate,borkDate):
64 rawFiles =
open(fileName,
'r')
66 print (
"could not open file " +
str(fileName) +
" fool")
69 returnList = [firstDate,largestDates,borkDate,dateEntries,firstDate]
74 for dataLine
in rawFiles:
77 index_Jennet = index_Jennet + 1
82 commaSplit = dataLine.split(
',')
84 if '!!!!' in dataLine:
92 tempLines = dataLine.split()
98 moduleName =
'LI_S' +
str(A) +
'_'+
str(B) +
'_M' +
str(C)
100 moduleNumber =
int(8 * (
int(A)-1) +
int(C)-1)
104 if (name!=moduleName+
"_"+dataType)
or (len(tempLines)<3) :
105 returnList[2] = lastDate
106 print (
"\nBORK BORK BORK! Error in data values of " + fileName +
" for " + dataType +
" borkDate " +
str(returnList[2]) )
107 print (
"First line reads " + dataLine+
"\n")
108 print (
"should read " + moduleName+
"_"+dataType+
"\n")
110 borkRepork =
open(dataFolder+dataType+
"BORKBORKBORK.txt",
'w')
111 borkRepork.write(
"filename " + fileName +
"date range " +
str(firstDate) +
"-" +
str(lastDate) +
" first line " + dataLine +
'\n')
115 outName = dataFolder+dataType+
'/' + moduleName +
'.ssv'
117 date = tempLines[1].
split(
'-')
118 time1 = tempLines[2].
split(
':')
121 if dataType ==
'TModule':
124 data = abs(
float(tempLines[3]))
125 dateTime = datetime.datetime(
int(date[2]),
int(date[1]),
int(date[0]),
int(time1[0]),
int(time1[1]),
int(time1[2]),
int(time1[3]))
128 if (dateTime>lastDate+datetime.timedelta(hours = 2))
or (dateTime<firstDate -datetime.timedelta(hours = 2)) :
129 returnList[2] = lastDate
130 print (
"\nBORK BORK BORK! Error in date of " + fileName +
" for " + dataType +
" borkDate " +
str(returnList[2]) )
131 print (
"First line reads " + dataLine +
"\n")
132 print (
"should read " + moduleName+
"_"+dataType+
"\n")
134 borkRepork =
open(dataFolder+dataType+
"BORKBORKBORK.txt",
'w')
135 borkRepork.write(
"filename " + fileName +
"date range " +
str(firstDate) +
"-" +
str(lastDate) +
" first line " + dataLine +
'\n')
140 if dateTime>returnList[1][moduleNumber]:
141 returnList[3][moduleNumber]+=1
144 outputLine = moduleName+
'_'+dataType +
" " +
str(dateTime) +
" " +
str( time.mktime(dateTime.timetuple()) ) +
" " +
str(data) +
"\n"
146 returnList[1][moduleNumber] = dateTime
149 if os.path.isfile(outName) ==
False:
151 output =
open(outName,
'w')
152 output.write(outputLine)
155 output =
open(outName,
'a')
156 output.write(outputLine)
162 moduleList = dataLine.split(
'!!!')
164 for module
in moduleList:
166 elements = module.split(
',')
177 moduleName =
'LI_S' +
str(A) +
'_'+
str(B) +
'_M' +
str(C)
181 moduleNumber =
int(8 * (
int(A)-1) +
int(C)-1)
185 outName = dataFolder+dataType+
'/' + moduleName +
'.ssv'
188 if os.path.isfile(outName) ==
False:
189 output =
open(outName,
'w')
191 output =
open(outName,
'a')
196 for element
in elements:
200 tempLines = element.split()
201 date= tempLines[1].
split(
'-')
202 time1 = tempLines[2].
split(
':')
206 if dataType ==
'TModule':
209 data = abs(
float(tempLines[0]))
211 dateTime = datetime.datetime(
int(date[2]),
int(date[1]),
int(date[0]),
int(time1[0]),
int(time1[1]),
int(time1[2]),
int(time1[3]))
214 if (dateTime>lastDate+datetime.timedelta(hours = 2))
or (dateTime<firstDate -datetime.timedelta(hours = 2)) :
215 returnList[2] = lastDate
216 print (
"\nBORK BORK BORK! Error in date of " + fileName +
" for " + dataType +
" borkDate " +
str(returnList[2]) )
217 print (
"First line reads " + dataLine +
"\n")
218 print (
"Should read " + moduleName+
"_"+dataType+
"\n")
220 borkRepork =
open(dataFolder+dataType+
"BORKBORKBORK.txt",
'w')
221 borkRepork.write(
"filename " + fileName +
"date range " +
str(firstDate) +
"-" +
str(lastDate) +
" first line " + dataLine +
'\n')
226 if dateTime>returnList[1][moduleNumber]:
227 returnList[3][moduleNumber]+=1
228 outputLine = moduleName +
"_" + dataType +
" " +
str(dateTime) +
" " +
str( time.mktime(dateTime.timetuple()) ) +
" " +
str(data) +
"\n"
230 returnList[1][moduleNumber] = dateTime
232 output.write(outputLine)
241 returnList[2] = lastDate
242 print(
"Something broke :( \n")
243 print(
"Could be an error in data values of " + fileName +
" for " + dataType +
" borkDate " +
str(returnList[2]) )
244 print(
"First line reads " + name +
"\n")
246 borkRepork =
open(dataFolder+dataType+
"BORKBORKBORK.txt",
'w')
247 borkRepork.write(
"filename " + fileName +
"date range " +
str(firstDate) +
"-" +
str(lastDate) +
" first line " + dataLine +
'\n')
249 print(
"Try running again... often this is an issue with the connection to eos")
258 fillerDate = datetime.datetime(2000,1,1,1,1,1,1)
259 tempDate = datetime.datetime(2000,1,1,1,1,1,1)
260 returnList =[fillerDate,fillerDate,fillerDate,
False,
"LI_S00_0_M0"]
262 homeDirectory = os.path.expanduser(
'/eos/atlas/user/j/jdickins/Pixel/LeakageCurrent/')
265 dataFolder = homeDirectory +
"/IBLData/processedData/"
266 inputFolder = homeDirectory +
"/IBLData/rawData/"+dataType+
"/"
267 entriesFolder = homeDirectory +
"/IBLData/rawData/entriesPerDay/" + dataType +
"/"
270 if not os.path.exists(dataFolder+dataType):
271 os.mkdir(dataFolder+dataType)
273 if not os.path.exists(entriesFolder):
274 os.mkdir(entriesFolder)
282 smallestEndDate=fillerDate
286 if not os.path.exists(dataFolder+dataType+
".txt"):
289 print(
"No any file found! at " + dataFolder+dataType +
" Making default values")
292 datesFile =
open(dataFolder+dataType+
".txt",
'w')
293 firstTempDate = datetime.datetime(2015,5,1,0,0,0,0)
294 lastTempDate = datetime.datetime(2015,5,1,0,0,0,1)
295 smallestEndDate = lastTempDate
297 for stave
in range(1,15):
298 staveString =
str(stave)
300 staveString=
"0"+
str(stave)
301 for side
in [
'A',
'C']:
302 for DCSGroup
in range(1,5):
303 moduleName =
'LI_S' +
str(staveString) +
'_' + side +
'_M' +
str(DCSGroup)
304 datesFile.write(moduleName +
" " +
str(firstTempDate) +
" " +
str(lastTempDate) +
" 0 0\n")
305 startingDates.append(firstTempDate)
306 endDates.append(lastTempDate)
307 filesProcessed.append(0)
309 dailyPoints.append(0)
314 print(
"Found " + dataFolder+dataType+
".txt")
315 datesFile =
open(dataFolder+dataType+
".txt",
'r')
319 for dateLine
in datesFile:
321 tempDatesLine = dateLine.split()
322 filesProcessed.append(
int(tempDatesLine[5]) )
323 dataPoints.append(
int( tempDatesLine[6]) )
324 dailyPoints.append(0)
326 firstTemp = tempDatesLine[1].
split(
'-')
327 lastTemp = tempDatesLine[3].
split(
'-')
329 firstTempTime = tempDatesLine[2].
split(
':')
330 lastTempTime = tempDatesLine[4].
split(
':')
332 firstTempTimes = firstTempTime[2].
split(
'.')
333 lastTempTimes = lastTempTime[2].
split(
'.')
335 if len(firstTempTimes)<2:
336 firstTempTimes.append(0)
337 if len(lastTempTimes)<2:
338 lastTempTimes.append(0)
340 firstTempDate = datetime.datetime(
int(firstTemp[0]),
int(firstTemp[1]),
int(firstTemp[2]),
int(firstTempTime[0]),
int(firstTempTime[1]),
int(firstTempTimes[0]),
int(firstTempTimes[1]))
341 lastTempDate = datetime.datetime(
int(lastTemp[0]),
int(lastTemp[1]),
int(lastTemp[2]),
int(lastTempTime[0]),
int(lastTempTime[1]),
int(lastTempTimes[0]),
int(lastTempTimes[1]))
343 startingDates.append(firstTempDate)
344 endDates.append(lastTempDate)
347 returnList[0] = firstTempDate
348 returnList[1] = lastTempDate
349 smallestEndDate=lastTempDate
353 if firstTempDate<returnList[0]:
354 returnList[0] = firstTempDate
355 if lastTempDate>returnList[1]:
356 returnList[1] = lastTempDate
357 if lastTempDate < smallestEndDate:
358 smallestEndDate = lastTempDate
364 print (
"Investigating " + dataType +
" from " +
str(smallestEndDate))
372 firstTempDate = startingDates[0]
373 lastTempDate = endDates[0]
374 numberFiles = len(fileList)
378 for fileName
in fileList:
383 endDate1 = fileName[end-4:end]
384 endDate2 = fileName[end-7:end-6]
386 if endDate1==
'.txt' and endDate2==
'_':
389 startDate = fileName[end-23:end-15]
390 endDate = fileName[end-12:end-4]
391 endDateSplit = endDate.split(
'_')
392 endDateFile = datetime.datetime(2000+
int(endDateSplit[0]),
int(endDateSplit[1]),
int(endDateSplit[2]),0,0,0,1)
394 startDateSplit = startDate.split(
'_')
395 startDateFile = datetime.datetime(2000+
int(startDateSplit[0]),
int(startDateSplit[1]),
int(startDateSplit[2]),0,0,0,1)
397 if endDateFile > smallestEndDate:
400 lastTempDate = endDateFile
401 [firstTempDate,endDates,fillerDate,tempDataNumbers,smallestEndDate] =
appendEntries(fileName,dataType,dataFolder,dailyPoints,endDates,startDateFile,endDateFile,fillerDate)
404 for i
in range(0,112):
406 filesProcessed[i] +=1
407 dataPoints[i]+=tempDataNumbers[i]
412 for stave
in range(1,15):
413 staveString =
str(stave)
415 staveString=
"0"+
str(stave)
416 for side
in [
'A',
'C']:
417 for DCSGroup
in range(1,5):
418 moduleName =
'LI_S' +
str(staveString) +
'_' + side +
'_M' +
str(DCSGroup)
419 outName = entriesFolder + moduleName +
".txt"
420 dataLine =
str(startDate) +
" " +
str(tempDataNumbers[holderX]) +
"\n"
421 tempDataNumbers[holderX]=0
423 if os.path.isfile(outName) ==
False:
424 output =
open(outName,
'w')
425 output.write(dataLine)
429 output =
open(outName,
'a')
430 output.write(dataLine)
435 if returnList[2]!=fillerDate:
436 returnList[2] = fillerDate
437 datesFile =
open(dataFolder+dataType+
".txt",
'w')
439 for stave
in range(1,15):
440 staveString =
str(stave)
442 staveString=
"0"+
str(stave)
443 for side
in [
'A',
'C']:
444 for DCSGroup
in range(1,5):
445 moduleName =
'LI_S' +
str(staveString) +
'_' + side +
'_M' +
str(DCSGroup)
446 datesFile.write(moduleName +
" " +
str(startingDates[tempHolder]) +
" " +
str(endDates[tempHolder]) +
" " +
str(filesProcessed[tempHolder]) +
" " + str ( dataPoints[tempHolder] ) +
"\n")
464 datesFile =
open(dataFolder+dataType+
".txt",
'w')
466 for staveX
in range(1,15):
467 staveStringX =
str(staveX)
469 staveStringX=
"0"+
str(staveX)
470 for sideX
in [
'A',
'C']:
471 for DCSGroupX
in range(1,5):
473 moduleNameX =
'LI_S' +
str(staveStringX) +
'_' + sideX +
'_M' +
str(DCSGroupX)
475 datesFile.write(moduleNameX +
" " +
str(startingDates[tempHolder]) +
" " +
str(endDates[tempHolder]) +
" " +
str(filesProcessed[tempHolder]) +
" " + str ( dataPoints[tempHolder] ) +
"\n")
486 threading.Thread.__init__(self)
491 print (
"Starting getIBLDate.py for " + self.
name)
493 print (
"Exiting getIBLDate.py for " + self.
name +
" with no issues, data range from " +
str(returnList[0]) +
" to " +
str(returnList[1]) )
500 thread1 = scanDataThread (1,
'HV_VMeas' )
503 thread2 = scanDataThread (2,
'PP4LV' )
506 thread3 = scanDataThread (3,
'HV_IMeas' )
509 thread4 = scanDataThread (4,
'TModule' )
512 if __name__ ==
"__main__":