ATLAS Offline Software
Loading...
Searching...
No Matches
grabAllAutoStave.py
Go to the documentation of this file.
1# Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
2
3# Python script from Nick Dann
4# Clean-ups for clarity and additional comments from Jennet Dickinson
5# Search for JENNET to find where to replace file paths etc
6
7# Comments from Nick:
8# Python script to grab IBL DCS data from the ATLAS DCS data viewer.
9# DO NOT USE THE MULTITHREADING FUNCTION!
10# grabData('dataType') is the function you should call to download DCS data. Program works from inside CERN network
11# There are some strings you'll have to change to point to locations on your file system. BE CAREFUL; no idea what this will do if it can't find the right file structure.
12# Timezones might be an issue; I used a PC running on UTC to get around this; there's probably a better solution, but that's what I did
13
14#!/usr/bin/env python3
15import os
16import time
17import shutil
18import subprocess
19import datetime
20import fileinput
21import sys
22import random
23from os.path import expanduser
24import threading
25from datetime import date, timedelta
26
27# Function to iterate through dates between start and end in steps of delta. ~ish
28
29
30def perdelta(start, end, delta):
31 curr = start
32 while curr < end:
33 yield curr, min(curr + delta, end)
34 curr += delta
35
36# Function to iterate through dates between end and start in steps of delta. ~ish
37
38
39def perdeltadown(start, end, delta):
40 curr = end
41 while curr > start:
42 yield max(curr - delta, start), curr
43 curr -= delta
44
45# Function to return list of all files in search folder path, sorted alphabetically
46
47
48def findFiles(searchFolderPath):
49
50 try:
51 os.chdir(searchFolderPath)
52 except IOError:
53 print('No entries in ' + searchFolderPath)
54 return -1
55
56 todaysList = []
57
58 # SOMEHOW, this searches the searchFolderPath and returns a list of contained files in well... files.
59 for src_dir, dirs, files in os.walk(searchFolderPath):
60
61 # Parse through the files, appending the file name to a list of files.
62 for file_ in files:
63
64 sortedFile = os.path.join(src_dir, file_)
65 todaysList.append(str(sortedFile))
66
67 # Now we've got a list containing all files we want to add, sort it alphabetically and return them.
68 todaysList.sort()
69 return todaysList
70
71# Function to return required dates
72
73
74def checkDates(dataType, dataFolder):
75
76 # Placeholder declarations
77 fillerDate = datetime.datetime(2000, 1, 1, 1, 1, 1, 1)
78 tempDate = datetime.datetime(2000, 1, 1, 1, 1, 1, 1)
79 returnList = [fillerDate, fillerDate, fillerDate]
80
81 startingDates = []
82 endDates = []
83
84 # Check: what dates have I already downloaded?
85 # If no file containing previously used dates, create one with default values.
86 # "No any file found!" is what my motherboard says whilst booting my home PC, I found it funny and copied it over here
87 if not os.path.exists(dataFolder+dataType+".txt"):
88 print("No any file found! at " + dataFolder +
89 dataType + " Making default values")
90
91 with open(dataFolder+dataType+".txt", 'w') as datesFile:
92 firstTempDate = datetime.datetime(2018, 1, 10, 1, 1, 1, 1)
93 lastTempDate = datetime.datetime(2017, 12, 21, 1, 1, 1)
94
95 datesFile.write(dataType + " " + str(firstTempDate) + " " +
96 str(lastTempDate) + "\n")
97 startingDates.append(firstTempDate)
98 endDates.append(lastTempDate)
99
100 # If dates file exists, read from it
101 else:
102 print("Found " + dataFolder+dataType+".txt")
103 with open(dataFolder+dataType+".txt", 'r') as datesFile:
104
105 for dateLine in datesFile:
106 tempDatesLine = dateLine.split()
107 firstTemp = tempDatesLine[1].split('-')
108 lastTemp = tempDatesLine[3].split('-')
109
110 firstTempTime = tempDatesLine[2].split(':')
111 lastTempTime = tempDatesLine[4].split(':')
112
113 firstTempTimes = firstTempTime[2].split('.')
114 lastTempTimes = lastTempTime[2].split('.')
115
116 if len(firstTempTimes) < 2:
117 firstTempTimes.append(0)
118 if len(lastTempTimes) < 2:
119 lastTempTimes.append(0)
120
121 firstTempDate = datetime.datetime(
122 int(firstTemp[0]), int(firstTemp[1]), int(firstTemp[2]),
123 int(firstTempTime[0]), int(firstTempTime[1]),
124 int(firstTempTimes[0]), int(firstTempTimes[1]))
125 lastTempDate = datetime.datetime(
126 int(lastTemp[0]), int(lastTemp[1]), int(lastTemp[2]),
127 int(lastTempTime[0]), int(lastTempTimes[0]),
128 int(lastTempTimes[0]), int(lastTempTimes[1]))
129
130 startingDates.append(firstTempDate)
131 endDates.append(lastTempDate)
132
133 datesList = [startingDates, endDates]
134 # Return start and end dates for each sensor
135 return datesList
136
137# Function to save data of type dataType in dataFolder for all sensors, for specified dates.
138# Currently used fof HV_VMease, PP4LV and TModules
139
140
141def grabData(dataType):
142
143 url = 'http://atlas-ddv.cern.ch:8089/multidata/getDataSafely'
144 url2 = 'http://atlas-ddv.cern.ch:8089/multidata/downloadTxtData'
145
146 # JENNET changed the output file path
147 defaultPath = os.path.expanduser(
148 '/eos/atlas/user/j/jdickins/Pixel/LeakageCurrent/')
149 # The data is saved to dataFolder / dataType / moduleGroup / startDate-endDate.txt
150 # This script will autogenerate folders for this.
151 dataFolder = defaultPath + "/IBLData/rawData/"
152
153 # Check top level directory exists, then make folder for HV I, T, HV V, LV I and.. actually should get LV V
154 if not os.path.exists(dataFolder):
155 os.mkdir(dataFolder)
156
157 if not os.path.exists(dataFolder+dataType):
158 os.mkdir(dataFolder+dataType)
159
160 [startDates, endDates] = checkDates(dataType, dataFolder)
161 currentDay = date.today()
162 firstDay = endDates[0].date()-timedelta(days=1)
163 # Iterate through all data in below dates, with a time step of time delta. (So from 1/1/2016 - 15/5/2016, with one file per day). You could probably increase the timedelta, but it'll increase the chance of the program dying, so I didn't. You could also create a bunch of folders to hold everything, or something similar.... I should do that. I did that!
164 # Jennet has issues for TModule with time delta = 1 day. Had to switch to grabDataStave
165
166 sensorNumber = 0
167 if firstDay+timedelta(days=1) < currentDay:
168 for s, e in perdelta(firstDay, currentDay, timedelta(days=1)):
169
170 # Check directory exists, make it if not
171 if not os.path.exists(dataFolder+dataType+'/'):
172 os.mkdir(dataFolder+dataType+'/')
173
174 # Generate data name
175 # Generate save file name. I save in format YYYY/MM/DD, so it is in alphabetical order.
176 saveFileName = dataFolder+dataType+'/' + \
177 s.strftime("%Y_%m_%d") + '-' + e.strftime("%Y_%m_%d") + '.txt'
178 if os.path.exists(saveFileName):
179 os.remove(saveFileName)
180
181 # Nick's magic: Jennet is baffled. But it works
182 # Create wget command
183 cmd = 'wget --post-data "queryInfo=atlas_pvssPIX, alias, LI_S01_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime(
184 "%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!" ' + url + ' --output-document=' + saveFileName
185
186 # Execute wget command
187 os.system(cmd)
188
189 firstTempDate = min(datetime.datetime(
190 s.year, s.month, s.day, 0, 0, 0, 0), startDates[0])
191 startDates[0] = firstTempDate
192
193 lastTempDate = max(datetime.datetime(
194 e.year, e.month, e.day, 0, 0, 0, 0), endDates[0])
195 endDates[0] = lastTempDate
196
197 with open(dataFolder+dataType+".txt", 'w') as datesFile:
198 datesFile.write(dataType + " " + str(firstTempDate) + " " +
199 str(lastTempDate) + "\n")
200
201 sleepTime = random.randint(60, 120)
202 time.sleep(sleepTime)
203
204# Function to save data of type dataType in dataFolder for all sensors, for specified dates. Only used to HV_IMeas
205
206
207def grabDataStave(dataType):
208
209 # url is the line you're supposed to use. url2 is the line that works better.
210 url = 'http://atlas-ddv.cern.ch:8089/multidata/getDataSafely'
211 url2 = 'http://atlas-ddv.cern.ch:8089/multidata/downloadTxtData'
212
213 # JENNET changed the output file path
214 defaultPath = os.path.expanduser(
215 '/eos/atlas/user/j/jdickins/Pixel/LeakageCurrent/')
216 # The data is saved to dataFolder / dataType / moduleGroup / startDate-endDate.txt
217 # This script will autogenerate folders for this.
218 dataFolder = defaultPath + "/IBLData/rawData/"
219
220 # Check top level directory exists, then make folder for HV I, T, HV V, LV I and.. actually should get LV V
221 if not os.path.exists(dataFolder):
222 os.mkdir(dataFolder)
223 # For dataType in ['HV_IMeas', 'TModule','HV_VMeas','PP4LV']:
224 if not os.path.exists(dataFolder+dataType):
225 os.mkdir(dataFolder+dataType)
226
227 [startDates, endDates] = checkDates(dataType, dataFolder)
228 currentDay = date.today()
229 firstDay = endDates[0].date()-timedelta(days=1)
230
231# firstDay = datetime.date(2018,3,7)
232# currentDay = datetime.date(2018,3,10)
233 # Iterate through all data in below dates, with a time step of time delta. (So from 1/1/2016 - 15/5/2016, with one file per day).
234
235 # Iterate through staves, modules and sides
236 sensorNumber = 0
237
238 if firstDay+timedelta(days=1) < currentDay:
239 for s, e in perdelta(firstDay, currentDay, timedelta(days=1)):
240 if not os.path.exists(dataFolder+dataType+'/'):
241 os.mkdir(dataFolder+dataType+'/')
242
243 if not os.path.exists(dataFolder+dataType+'Stave/'):
244 os.mkdir(dataFolder+dataType+'Stave/')
245
246 saveFileName2 = dataFolder+dataType+'/' + \
247 s.strftime("%Y_%m_%d") + '-' + e.strftime("%Y_%m_%d") + '.txt'
248 if os.path.exists(saveFileName2):
249 os.remove(saveFileName2)
250
251 with open(saveFileName2, 'w') as saveFile:
252 introubleCount = []
253 introubleMean = 0
254
255 staveString = "stave"
256 for staveNumber in range(1, 15):
257 if staveNumber < 10:
258 staveString = "0" + str(staveNumber)
259 else:
260 staveString = str(staveNumber)
261
262 # Generate save file name. I save in format YYYY/MM/DD, so it is in alphabetical order.
263 saveFileName = dataFolder+dataType+'Stave/' + \
264 s.strftime(
265 "%Y_%m_%d") + '-' + e.strftime("%Y_%m_%d") + 'Stave' + staveString + '.txt'
266 if os.path.exists(saveFileName):
267 os.remove(saveFileName)
268
269 # Create wget command
270 cmd = 'wget --post-data "queryInfo=atlas_pvssPIX, alias, LI_S' + staveString + '_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime(
271 "%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!" ' + url + ' --output-document=' + saveFileName
272
273 # Execute wget command
274 os.system(cmd)
275
276 if (os.path.getsize(saveFileName) < 300):
277 # i.e. if the file is small, download probably failed, try a different command
278
279 sleepTime = random.randint(30, 62)
280 time.sleep(sleepTime)
281 os.remove(saveFileName)
282
283 cmd = 'wget --post-data "queryInfo=atlas_pvssPIX, alias, LI_S' + staveString + '_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime(
284 "%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!!" ' + url2 + ' --output-document=' + saveFileName
285
286 # Execute wget command
287 os.system(cmd)
288
289 if (0 < os.path.getsize(saveFileName) < 300):
290 # If this happens again, even url2 couldn't save us. You're gonna have to do stuff yourself
291
292 os.remove(saveFileName)
293 print("bork bork at " + s.strftime("%Y_%m_%d") +
294 " please sign off yourself!")
295 break
296
297 with open(saveFileName, 'r') as bloop:
298
299 if (staveNumber != 1):
300 saveFile.write("!!!")
301
302 introuble = 0
303
304 for bloopLine in bloop:
305 saveFile.write(bloopLine)
306 introuble += 1
307
308 introubleMean += introuble
309
310 introubleCount.append(introuble)
311 sleepTime = random.randint(30, 62)
312 # Go to sleep so the DCS people don't yell at you
313 time.sleep(sleepTime)
314
315 breakThatStick = False
316
317 if introubleMean > 30:
318 for checker in range(0, 14):
319 if introubleCount[checker] < 2:
320 # If a sensor's data is less than 2 entries, whilst there are more than 30 entries for the day.
321 print("borkalork in stave " + str(checker+1))
322 breakThatStick = True
323
324 if breakThatStick:
325 print("breaking now at " + s.strftime("%d-%m-%Y"))
326 break
327
328 firstTempDate = min(datetime.datetime(
329 s.year, s.month, s.day, 0, 0, 0, 0), startDates[0])
330 startDates[0] = firstTempDate
331
332 lastTempDate = max(datetime.datetime(
333 e.year, e.month, e.day, 0, 0, 0, 0), endDates[0])
334 endDates[0] = lastTempDate
335
336 with open(dataFolder+dataType+".txt", 'w') as datesFile:
337 datesFile.write(dataType + " " + str(firstTempDate) + " " +
338 str(lastTempDate) + "\n")
339
340 sleepTime = random.randint(10, 30)
341 time.sleep(sleepTime)
342
343
346
347
348def main():
349
350 grabData('HV_VMeas')
351 grabData('PP4LV')
352
353 # grabData no longer works reliably for HV_IMeas, use grabStaveData instead
354 # Use this if grabData returns an error about very big DB request
355 grabDataStave('HV_IMeas')
356 grabDataStave('TModule')
357
358
359if __name__ == "__main__":
360 main()
print
void print(char *figname, TCanvas *c1)
Definition TRTCalib_StrawStatusPlots.cxx:26
min
#define min(a, b)
Definition cfImp.cxx:40
max
#define max(a, b)
Definition cfImp.cxx:41
date
std::string date()
sadly, includes a return at the end
Definition hcg.cxx:58
split
std::vector< std::string > split(const std::string &s, const std::string &t=":")
Definition hcg.cxx:177
grabAllAutoStave.findFiles
findFiles(searchFolderPath)
Definition grabAllAutoStave.py:48
grabAllAutoStave.checkDates
checkDates(dataType, dataFolder)
Definition grabAllAutoStave.py:74
grabAllAutoStave.main
main()
PROGRAM STARTS HERE #.
Definition grabAllAutoStave.py:348
grabAllAutoStave.perdelta
perdelta(start, end, delta)
Definition grabAllAutoStave.py:30
grabAllAutoStave.perdeltadown
perdeltadown(start, end, delta)
Definition grabAllAutoStave.py:39
Generated on for ATLAS Offline Software by doxygen 1.14.0