ATLAS Offline Software
TileInputFiles.py
Go to the documentation of this file.
1 #
2 # Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
3 #
4 
5 '''
6 @file TileInputFiles.py
7 @brief Python configuration of Tile input files
8 '''
9 
10 from AthenaCommon.Logging import logging
11 from subprocess import check_output
12 from subprocess import CalledProcessError
13 import six
14 
15 def getInputDirectory(run, stream=None, project=None, suffix=None, year=None):
16  """
17  Function to find input directory with Tile Calorimeter input data files: calibrations, ...
18  Arguments:
19  run -- run number
20  stream -- run stream
21  project -- data project
22  suffix -- directory suffix
23  skipBadFiles -- skip known bad files
24  year -- year of data
25  """
26 
27  log = logging.getLogger( 'TileInputFiles.getInputDirectory' )
28 
29  if run < 10:
30  directory = '.'
31  else:
32  if not year:
33  yr={ 2023:441536, 2022:408681, 2021:387034, 2020:374260,
34  2019:367983, 2018:342531, 2017:314451, 2016:288032, 2015:248505,
35  2014:224307, 2013:216705, 2012:194688, 2011:171194, 2010:142682,
36  2009:99717, 2008:35430, 2007:0}
37  for year,beg in yr.items():
38  if run>=beg:
39  break
40 
41  if stream or project or suffix:
42  if not stream:
43  stream = 'physics_Main'
44  log.warning('%s is not set up and will be used: %s' , 'Run stream', stream)
45  elif stream == 'Tile':
46  stream = 'calibration_Tile'
47  if not project:
48  if 'calibration' in stream and 'Tile' not in stream:
49  project = f'data{year%100}_calib'
50  else:
51  project = f'data{year%100}_13p6TeV'
52  log.warning('%s is not set up and will be used: %s' , 'Data project', project)
53  elif 'data' not in project:
54  project = f'data{year%100}_{project}'
55  if not suffix:
56  if stream == 'physics_Main' or stream == 'physics_MinBias' or stream.startswith('calibration'):
57  suffix = 'daq.RAW'
58  else:
59  suffix = 'merge.RAW'
60  log.warning('%s is not set up and will be used: %s' , 'Directory suffix', suffix)
61 
62  run=str(run).zfill(8)
63  directory = f'/eos/atlas/atlastier0/rucio/{project}/{stream}/{run}/{project}.{run}.{stream}.{suffix}'
64 
65  else:
66  directory = f'/eos/atlas/atlascerngroupdisk/det-tile/online/{year}/daq'
67 
68  return directory
69 
70 
71 def findFiles(run, path=None, filter='.', stream=None, project=None, suffix=None, year=None, skipBadFiles=True):
72  """
73  Function to find Tile Calorimeter input data files: calibrations, ...
74  Arguments:
75  run -- run number
76  path -- input directory
77  filter -- data file filter
78  stream -- run stream
79  project -- data project
80  suffix -- directory suffix
81  year -- year, data taken in
82  skipBadFiles - skip known bad files
83  """
84 
85  log = logging.getLogger( 'TileInputFiles.findFiles' )
86 
87  if not path:
88  path = getInputDirectory(run, stream, project, suffix, year)
89 
90  if not path:
91  log.warning('There is no input directory')
92  return []
93 
94  log.info('Input directory: %s', path)
95 
96  run=str(run).zfill(7) if int(run) > 0 else str(run)
97  if (path.startswith('/eos/')):
98  listRunFiles = f'xrdfs eosatlas ls -l {path} | grep -e {run} | grep -v "#" '
99  listRunFiles += f'| grep -v -e " [ 0-9][ 0-9][0-9] " | grep {filter} | sed "s|^.*/||" '
100 
101  else:
102  listRunFiles = f'ls {path} | grep -e {run} | grep {filter}'
103 
104  files = []
105  try:
106  files = check_output(listRunFiles, shell = True).splitlines()
107  except CalledProcessError:
108  log.warning('It seems that there are no such directory: %s', path)
109 
110  badFiles = ""
111  if skipBadFiles:
112  for badDataFiles in ['/afs/cern.ch/user/t/tilebeam/ARR/bad_data_files', '/afs/cern.ch/user/t/tiledaq/public/bad_data_files']:
113  try:
114  badFiles += open(badDataFiles).read()
115  except Exception:
116  log.warning('Can not read file with bad data files: %s => It is ignored', badDataFiles)
117 
118  fullNames = []
119  files = [six.ensure_str(f) for f in files]
120  for file_name in (files):
121  good = (file_name not in badFiles)
122  if good:
123  if (path.startswith('/eos/')):
124  fullNames.append(f'root://eosatlas.cern.ch/{path}/{file_name}')
125  else:
126  fullNames.append(f'{path}/{file_name}')
127  else:
128  log.warning('Excluding known bad data file: %s', file_name)
129 
130  return fullNames
131 
132 
134  """
135  Function to find Tile Calorimeter input data files (calibrations, ...) from arguments
136  Arguments:
137  args -- arguments prepared by argument parser
138  """
139  files = findFiles(run=args.run, path=args.inputDirectory, filter=args.filter, stream=args.stream,
140  project=args.project, suffix=args.suffix, year=args.year, skipBadFiles=args.skipBadFiles)
141  return files
142 
143 
144 def getArgumentParser(**kwargs):
145  """
146  Function to construct and return argument parser
147  """
148 
149  import argparse
150  parser= argparse.ArgumentParser('Script to find Tile Calorimeter input data files: calibrations, ...', **kwargs)
151  files = parser.add_argument_group('Tile find input files')
152  files.add_argument("-r", "--run", type=int, default=None, help="Run number")
153  files.add_argument("--inputDirectory", type=str, default=None, help="Input directory")
154  files.add_argument("-s", "--stream", type=str, default=None, help="Run stream")
155  files.add_argument("-p", "--project", type=str, default=None, help="Data project")
156  files.add_argument("-f", "--filter", type=str, default=".", help="Data file filter")
157  files.add_argument("--suffix", type=str, default=None, help="Directory suffix")
158  files.add_argument("--skipBadFiles", type=bool, default=True, help="Skip bad data files?")
159  files.add_argument("-y", "--year", type=int, default=None, help="Year, data taken in")
160 
161  return parser
162 
163 
164 if __name__=='__main__':
165 
166  log = logging.getLogger( 'TileInputFiles' )
167 
168  parser = getArgumentParser()
169  args = parser.parse_args()
170 
172 
173  log.info('Input files: %s', files)
174  if not files:
175  log.warning("No run data files are found")
TileInputFiles.getArgumentParser
def getArgumentParser(**kwargs)
Definition: TileInputFiles.py:144
read
IovVectorMap_t read(const Folder &theFolder, const SelectionCriterion &choice, const unsigned int limit=10)
Definition: openCoraCool.cxx:569
CaloCellPos2Ntuple.int
int
Definition: CaloCellPos2Ntuple.py:24
TileInputFiles.findFiles
def findFiles(run, path=None, filter='.', stream=None, project=None, suffix=None, year=None, skipBadFiles=True)
Definition: TileInputFiles.py:71
TileInputFiles.getInputDirectory
def getInputDirectory(run, stream=None, project=None, suffix=None, year=None)
Definition: TileInputFiles.py:15
TileInputFiles.findFilesFromAgruments
def findFilesFromAgruments(args)
Definition: TileInputFiles.py:133
Trk::open
@ open
Definition: BinningType.h:40
str
Definition: BTagTrackIpAccessor.cxx:11