ATLAS Offline Software
grabAllAutoStave.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
2 
3 # Python script from Nick Dann
4 # Clean-ups for clarity and additional comments from Jennet Dickinson
5 # Search for JENNET to find where to replace file paths etc
6 
7 # Comments from Nick:
8 # Python script to grab IBL DCS data from the ATLAS DCS data viewer.
9 # DO NOT USE THE MULTITHREADING FUNCTION!
10 # grabData('dataType') is the function you should call to download DCS data. Program works from inside CERN network
11 # There are some strings you'll have to change to point to locations on your file system. BE CAREFUL; no idea what this will do if it can't find the right file structure.
12 # Timezones might be an issue; I used a PC running on UTC to get around this; there's probably a better solution, but that's what I did
13 
14 #!/usr/bin/env python3
15 import os
16 import time
17 import shutil
18 import subprocess
19 import datetime
20 import fileinput
21 import sys
22 import random
23 from os.path import expanduser
24 import threading
25 from datetime import date, timedelta
26 
27 # Function to iterate through dates between start and end in steps of delta. ~ish
28 
29 
30 def perdelta(start, end, delta):
31  curr = start
32  while curr < end:
33  yield curr, min(curr + delta, end)
34  curr += delta
35 
36 # Function to iterate through dates between end and start in steps of delta. ~ish
37 
38 
39 def perdeltadown(start, end, delta):
40  curr = end
41  while curr > start:
42  yield max(curr - delta, start), curr
43  curr -= delta
44 
45 # Function to return list of all files in search folder path, sorted alphabetically
46 
47 
48 def findFiles(searchFolderPath):
49 
50  try:
51  os.chdir(searchFolderPath)
52  except IOError:
53  print('No entries in ' + searchFolderPath)
54  return -1
55 
56  todaysList = []
57 
58  # SOMEHOW, this searches the searchFolderPath and returns a list of contained files in well... files.
59  for src_dir, dirs, files in os.walk(searchFolderPath):
60 
61  # Parse through the files, appending the file name to a list of files.
62  for file_ in files:
63 
64  sortedFile = os.path.join(src_dir, file_)
65  todaysList.append(str(sortedFile))
66 
67  # Now we've got a list containing all files we want to add, sort it alphabetically and return them.
68  todaysList.sort()
69  return todaysList
70 
71 # Function to return required dates
72 
73 
74 def checkDates(dataType, dataFolder):
75 
76  # Placeholder declarations
77  fillerDate = datetime.datetime(2000, 1, 1, 1, 1, 1, 1)
78  tempDate = datetime.datetime(2000, 1, 1, 1, 1, 1, 1)
79  returnList = [fillerDate, fillerDate, fillerDate]
80 
81  startingDates = []
82  endDates = []
83 
84  # Check: what dates have I already downloaded?
85  # If no file containing previously used dates, create one with default values.
86  # "No any file found!" is what my motherboard says whilst booting my home PC, I found it funny and copied it over here
87  if not os.path.exists(dataFolder+dataType+".txt"):
88  print("No any file found! at " + dataFolder +
89  dataType + " Making default values")
90 
91  with open(dataFolder+dataType+".txt", 'w') as datesFile:
92  firstTempDate = datetime.datetime(2018, 1, 10, 1, 1, 1, 1)
93  lastTempDate = datetime.datetime(2017, 12, 21, 1, 1, 1)
94 
95  datesFile.write(dataType + " " + str(firstTempDate) + " " +
96  str(lastTempDate) + "\n")
97  startingDates.append(firstTempDate)
98  endDates.append(lastTempDate)
99 
100  # If dates file exists, read from it
101  else:
102  print("Found " + dataFolder+dataType+".txt")
103  with open(dataFolder+dataType+".txt", 'r') as datesFile:
104 
105  for dateLine in datesFile:
106  tempDatesLine = dateLine.split()
107  firstTemp = tempDatesLine[1].split('-')
108  lastTemp = tempDatesLine[3].split('-')
109 
110  firstTempTime = tempDatesLine[2].split(':')
111  lastTempTime = tempDatesLine[4].split(':')
112 
113  firstTempTimes = firstTempTime[2].split('.')
114  lastTempTimes = lastTempTime[2].split('.')
115 
116  if len(firstTempTimes) < 2:
117  firstTempTimes.append(0)
118  if len(lastTempTimes) < 2:
119  lastTempTimes.append(0)
120 
121  firstTempDate = datetime.datetime(
122  int(firstTemp[0]), int(firstTemp[1]), int(firstTemp[2]),
123  int(firstTempTime[0]), int(firstTempTime[1]),
124  int(firstTempTimes[0]), int(firstTempTimes[1]))
125  lastTempDate = datetime.datetime(
126  int(lastTemp[0]), int(lastTemp[1]), int(lastTemp[2]),
127  int(lastTempTime[0]), int(lastTempTimes[0]),
128  int(lastTempTimes[0]), int(lastTempTimes[1]))
129 
130  startingDates.append(firstTempDate)
131  endDates.append(lastTempDate)
132 
133  datesList = [startingDates, endDates]
134  # Return start and end dates for each sensor
135  return datesList
136 
137 # Function to save data of type dataType in dataFolder for all sensors, for specified dates.
138 # Currently used fof HV_VMease, PP4LV and TModules
139 
140 
141 def grabData(dataType):
142 
143  url = 'http://atlas-ddv.cern.ch:8089/multidata/getDataSafely'
144  url2 = 'http://atlas-ddv.cern.ch:8089/multidata/downloadTxtData'
145 
146  # JENNET changed the output file path
147  defaultPath = os.path.expanduser(
148  '/eos/atlas/user/j/jdickins/Pixel/LeakageCurrent/')
149  # The data is saved to dataFolder / dataType / moduleGroup / startDate-endDate.txt
150  # This script will autogenerate folders for this.
151  dataFolder = defaultPath + "/IBLData/rawData/"
152 
153  # Check top level directory exists, then make folder for HV I, T, HV V, LV I and.. actually should get LV V
154  if not os.path.exists(dataFolder):
155  os.mkdir(dataFolder)
156 
157  if not os.path.exists(dataFolder+dataType):
158  os.mkdir(dataFolder+dataType)
159 
160  [startDates, endDates] = checkDates(dataType, dataFolder)
161  currentDay = date.today()
162  firstDay = endDates[0].date()-timedelta(days=1)
163  # Iterate through all data in below dates, with a time step of time delta. (So from 1/1/2016 - 15/5/2016, with one file per day). You could probably increase the timedelta, but it'll increase the chance of the program dying, so I didn't. You could also create a bunch of folders to hold everything, or something similar.... I should do that. I did that!
164  # Jennet has issues for TModule with time delta = 1 day. Had to switch to grabDataStave
165 
166  sensorNumber = 0
167  if firstDay+timedelta(days=1) < currentDay:
168  for s, e in perdelta(firstDay, currentDay, timedelta(days=1)):
169 
170  # Check directory exists, make it if not
171  if not os.path.exists(dataFolder+dataType+'/'):
172  os.mkdir(dataFolder+dataType+'/')
173 
174  # Generate data name
175  # Generate save file name. I save in format YYYY/MM/DD, so it is in alphabetical order.
176  saveFileName = dataFolder+dataType+'/' + \
177  s.strftime("%Y_%m_%d") + '-' + e.strftime("%Y_%m_%d") + '.txt'
178  if os.path.exists(saveFileName):
179  os.remove(saveFileName)
180 
181  # Nick's magic: Jennet is baffled. But it works
182  # Create wget command
183  cmd = 'wget --post-data "queryInfo=atlas_pvssPIX, alias, LI_S01_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S01_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S02_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S03_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S04_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S05_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S06_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S07_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime(
184  "%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S08_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S09_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S10_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S11_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S12_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S13_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S14_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!" ' + url + ' --output-document=' + saveFileName
185 
186  # Execute wget command
187  os.system(cmd)
188 
189  firstTempDate = min(datetime.datetime(
190  s.year, s.month, s.day, 0, 0, 0, 0), startDates[0])
191  startDates[0] = firstTempDate
192 
193  lastTempDate = max(datetime.datetime(
194  e.year, e.month, e.day, 0, 0, 0, 0), endDates[0])
195  endDates[0] = lastTempDate
196 
197  with open(dataFolder+dataType+".txt", 'w') as datesFile:
198  datesFile.write(dataType + " " + str(firstTempDate) + " " +
199  str(lastTempDate) + "\n")
200 
201  sleepTime = random.randint(60, 120)
202  time.sleep(sleepTime)
203 
204 # Function to save data of type dataType in dataFolder for all sensors, for specified dates. Only used to HV_IMeas
205 
206 
207 def grabDataStave(dataType):
208 
209  # url is the line you're supposed to use. url2 is the line that works better.
210  url = 'http://atlas-ddv.cern.ch:8089/multidata/getDataSafely'
211  url2 = 'http://atlas-ddv.cern.ch:8089/multidata/downloadTxtData'
212 
213  # JENNET changed the output file path
214  defaultPath = os.path.expanduser(
215  '/eos/atlas/user/j/jdickins/Pixel/LeakageCurrent/')
216  # The data is saved to dataFolder / dataType / moduleGroup / startDate-endDate.txt
217  # This script will autogenerate folders for this.
218  dataFolder = defaultPath + "/IBLData/rawData/"
219 
220  # Check top level directory exists, then make folder for HV I, T, HV V, LV I and.. actually should get LV V
221  if not os.path.exists(dataFolder):
222  os.mkdir(dataFolder)
223  # For dataType in ['HV_IMeas', 'TModule','HV_VMeas','PP4LV']:
224  if not os.path.exists(dataFolder+dataType):
225  os.mkdir(dataFolder+dataType)
226 
227  [startDates, endDates] = checkDates(dataType, dataFolder)
228  currentDay = date.today()
229  firstDay = endDates[0].date()-timedelta(days=1)
230 
231 # firstDay = datetime.date(2018,3,7)
232 # currentDay = datetime.date(2018,3,10)
233  # Iterate through all data in below dates, with a time step of time delta. (So from 1/1/2016 - 15/5/2016, with one file per day).
234 
235  # Iterate through staves, modules and sides
236  sensorNumber = 0
237 
238  if firstDay+timedelta(days=1) < currentDay:
239  for s, e in perdelta(firstDay, currentDay, timedelta(days=1)):
240  if not os.path.exists(dataFolder+dataType+'/'):
241  os.mkdir(dataFolder+dataType+'/')
242 
243  if not os.path.exists(dataFolder+dataType+'Stave/'):
244  os.mkdir(dataFolder+dataType+'Stave/')
245 
246  saveFileName2 = dataFolder+dataType+'/' + \
247  s.strftime("%Y_%m_%d") + '-' + e.strftime("%Y_%m_%d") + '.txt'
248  if os.path.exists(saveFileName2):
249  os.remove(saveFileName2)
250 
251  with open(saveFileName2, 'w') as saveFile:
252  introubleCount = []
253  introubleMean = 0
254 
255  staveString = "stave"
256  for staveNumber in range(1, 15):
257  if staveNumber < 10:
258  staveString = "0" + str(staveNumber)
259  else:
260  staveString = str(staveNumber)
261 
262  # Generate save file name. I save in format YYYY/MM/DD, so it is in alphabetical order.
263  saveFileName = dataFolder+dataType+'Stave/' + \
264  s.strftime(
265  "%Y_%m_%d") + '-' + e.strftime("%Y_%m_%d") + 'Stave' + staveString + '.txt'
266  if os.path.exists(saveFileName):
267  os.remove(saveFileName)
268 
269  # Create wget command
270  cmd = 'wget --post-data "queryInfo=atlas_pvssPIX, alias, LI_S' + staveString + '_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime(
271  "%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!" ' + url + ' --output-document=' + saveFileName
272 
273  # Execute wget command
274  os.system(cmd)
275 
276  if (os.path.getsize(saveFileName) < 300):
277  # i.e. if the file is small, download probably failed, try a different command
278 
279  sleepTime = random.randint(30, 62)
280  time.sleep(sleepTime)
281  os.remove(saveFileName)
282 
283  cmd = 'wget --post-data "queryInfo=atlas_pvssPIX, alias, LI_S' + staveString + '_A_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_A_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime(
284  "%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M1_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M2_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M3_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!atlas_pvssPIX, alias, LI_S' + staveString + '_C_M4_' + dataType + ', ' + s.strftime("%d-%m-%Y") + ' 00:00, ' + e.strftime("%d-%m-%Y") + ' 00:01, , , , , ,no, , +2!!" ' + url2 + ' --output-document=' + saveFileName
285 
286  # Execute wget command
287  os.system(cmd)
288 
289  if (0 < os.path.getsize(saveFileName) < 300):
290  # If this happens again, even url2 couldn't save us. You're gonna have to do stuff yourself
291 
292  os.remove(saveFileName)
293  print("bork bork at " + s.strftime("%Y_%m_%d") +
294  " please sign off yourself!")
295  break
296 
297  with open(saveFileName, 'r') as bloop:
298 
299  if (staveNumber != 1):
300  saveFile.write("!!!")
301 
302  introuble = 0
303 
304  for bloopLine in bloop:
305  saveFile.write(bloopLine)
306  introuble += 1
307 
308  introubleMean += introuble
309 
310  introubleCount.append(introuble)
311  sleepTime = random.randint(30, 62)
312  # Go to sleep so the DCS people don't yell at you
313  time.sleep(sleepTime)
314 
315  breakThatStick = False
316 
317  if introubleMean > 30:
318  for checker in range(0, 14):
319  if introubleCount[checker] < 2:
320  # If a sensor's data is less than 2 entries, whilst there are more than 30 entries for the day.
321  print("borkalork in stave " + str(checker+1))
322  breakThatStick = True
323 
324  if breakThatStick:
325  print("breaking now at " + s.strftime("%d-%m-%Y"))
326  break
327 
328  firstTempDate = min(datetime.datetime(
329  s.year, s.month, s.day, 0, 0, 0, 0), startDates[0])
330  startDates[0] = firstTempDate
331 
332  lastTempDate = max(datetime.datetime(
333  e.year, e.month, e.day, 0, 0, 0, 0), endDates[0])
334  endDates[0] = lastTempDate
335 
336  with open(dataFolder+dataType+".txt", 'w') as datesFile:
337  datesFile.write(dataType + " " + str(firstTempDate) + " " +
338  str(lastTempDate) + "\n")
339 
340  sleepTime = random.randint(10, 30)
341  time.sleep(sleepTime)
342 
343 
346 
347 
348 def main():
349 
350  grabData('HV_VMeas')
351  grabData('PP4LV')
352 
353  # grabData no longer works reliably for HV_IMeas, use grabStaveData instead
354  # Use this if grabData returns an error about very big DB request
355  grabDataStave('HV_IMeas')
356  grabDataStave('TModule')
357 
358 
359 if __name__ == "__main__":
360  main()
grabAllAutoStave.grabDataStave
def grabDataStave(dataType)
Definition: grabAllAutoStave.py:207
CaloCellPos2Ntuple.int
int
Definition: CaloCellPos2Ntuple.py:24
max
constexpr double max()
Definition: ap_fixedTest.cxx:33
min
constexpr double min()
Definition: ap_fixedTest.cxx:26
grabAllAutoStave.checkDates
def checkDates(dataType, dataFolder)
Definition: grabAllAutoStave.py:74
grabAllAutoStave.main
def main()
PROGRAM STARTS HERE #.
Definition: grabAllAutoStave.py:348
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:195
grabAllAutoStave.findFiles
def findFiles(searchFolderPath)
Definition: grabAllAutoStave.py:48
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:25
fillHVMap_fromASCII.date
string date
Definition: fillHVMap_fromASCII.py:8
Trk::open
@ open
Definition: BinningType.h:40
grabAllAutoStave.perdelta
def perdelta(start, end, delta)
Definition: grabAllAutoStave.py:30
grabAllAutoStave.perdeltadown
def perdeltadown(start, end, delta)
Definition: grabAllAutoStave.py:39
str
Definition: BTagTrackIpAccessor.cxx:11
Trk::split
@ split
Definition: LayerMaterialProperties.h:38
grabAllAutoStave.grabData
def grabData(dataType)
Definition: grabAllAutoStave.py:141
Generated on Wed Jan 8 2025 21:10:28 for ATLAS Offline Software by   doxygen 1.8.18