ATLAS Offline Software
Classes | Functions | Variables
python.update_ci_reference_files Namespace Reference

Classes

class  CITest
 

Functions

def process_log_file (url, branch, test_name)
 
def process_diffpool_change (text, ami_tag, mr_number, human_readable_date, test_name)
 
def process_digest_change (text, ami_tag, mr_number, human_readable_date, test_name)
 
def update_reference_files (actually_update=True, update_local_files=False)
 
def create_dir_and_copy_refs (test, actually_update=False)
 
def process_CI_Tests_json (data)
 
def strip_url (href)
 
def strip_href (href)
 
def process_CI_Builds_Summary (project)
 
def extract_links_from_json (url)
 
def summarise_failing_tests (check_for_duplicates=True)
 

Variables

 failing_tests
 
 dirs_created
 
 debug
 
 parser
 
 description
 
 formatter_class
 
 help
 
 action
 
 args
 
 mr_url
 
 gl_project
 
 mr
 
 author
 
 remote
 
 local_branch
 
 msg
 
 update_local_files
 
 not_in_athena_dir
 
 shell
 
 commands
 

Function Documentation

◆ create_dir_and_copy_refs()

def python.update_ci_reference_files.create_dir_and_copy_refs (   test,
  actually_update = False 
)
If called with actually_update=False, this function will return a list of commands which would have been executed.

Definition at line 280 of file update_ci_reference_files.py.

280 def create_dir_and_copy_refs(test, actually_update=False):
281  """
282  If called with actually_update=False, this function will return a list of commands which would have been executed.
283  """
284  commands = []
285  if test.new_version_directory not in dirs_created:
286  commands.append("mkdir " + test.new_version_directory)
287  dirs_created.append(test.new_version_directory)
288 
289  # Copy new directory first, then copy old (in case the new MR did not touch all files)
290  # Important! Use no-clobber for second copy or we will overwrite the new data with old!
291  commands.append("cp " + test.copied_file_path + "* "+ test.new_version_directory+"/")
292  commands.append("cp -n " + test.existing_ref + "/* "+ test.new_version_directory+"/")
293  if actually_update:
294  print(' -> Copying files from {} to {}'.format(test.copied_file_path, test.new_version_directory))
295  try:
296  for command in commands:
297  try:
298  subprocess.call( command, shell=True)
299  except Exception as e:
300  print('Command failed due to:', e)
301  print('Do you have EOS available on this machine?')
302  except Exception as e:
303  print('FATAL: Unable to copy files due to:', e)
304  sys.exit(1)
305 
306  f = open(test.new_version_directory+'/info.txt', 'w')
307  f.write('Merge URL: https://gitlab.cern.ch/atlas/athena/-/merge_requests/{}\n'.format(test.mr))
308  f.write('Date: {}\n'.format(test.date))
309  f.write('AMI: {}\n'.format(test.tag))
310  f.write('Test name: {}\n'.format(test.name))
311  f.write('Files copied from: {}\n'.format(test.copied_file_path))
312  f.close()
313 
314  return commands
315 

◆ extract_links_from_json()

def python.update_ci_reference_files.extract_links_from_json (   url)

Definition at line 347 of file update_ci_reference_files.py.

347 def extract_links_from_json(url):
348  headers = {'Accept': 'application/json'}
349  r = requests.get(url+'&json', headers=headers)
350  data = r.json()["rows_s"]
351  # First row is header.
352  # Currently this is: 'Release', 'Platform', 'Project', 'git branch<BR>(link to MR)', 'Job time stamp', 'git clone', 'Externals build', 'CMake config', 'Build time', 'Comp. Errors (w/warnings)', 'Test time', 'CI tests errors (w/warnings)', 'Host'
353  for project in data[1:]:
355 

◆ process_CI_Builds_Summary()

def python.update_ci_reference_files.process_CI_Builds_Summary (   project)

Definition at line 332 of file update_ci_reference_files.py.

332 def process_CI_Builds_Summary(project):
333  # Each entry is one column in the table. 11th is the tests column.
334  # URL to tests page is in form:
335  # <a href="/testsview/?nightly=MR-CI-builds&rel=MR-66303-2023-10-10-19-08&ar=x86_64-centos7-gcc112-opt&proj=AthGeneration">0 (0)</a>
336  test_counts = strip_href(project[11])
337  # This is e.g. '0 (0)'
338  test_error_counts = int(test_counts.split(' ')[0])
339  if test_error_counts > 0:
340  # Okay, we have an error!
341  project_url = 'https://bigpanda.cern.ch'+strip_url(project[11])
342  headers = {'Accept': 'application/json'}
343  r = requests.get(project_url+'&json', headers=headers)
344  data = r.json()["rows_s"]
345  process_CI_Tests_json(data[1:])
346 

◆ process_CI_Tests_json()

def python.update_ci_reference_files.process_CI_Tests_json (   data)

Definition at line 316 of file update_ci_reference_files.py.

316 def process_CI_Tests_json(data):
317  # Each list entry is one column in the table.
318  for row in data:
319  if ('ERROR' in row[0]):
320  process_log_file(strip_url(row[2]), branch = row[1], test_name=strip_href(row[2]))
321 

◆ process_diffpool_change()

def python.update_ci_reference_files.process_diffpool_change (   text,
  ami_tag,
  mr_number,
  human_readable_date,
  test_name 
)

Definition at line 111 of file update_ci_reference_files.py.

111 def process_diffpool_change(text, ami_tag, mr_number, human_readable_date, test_name):
112  eos_path_root = '/eos/atlas/atlascerngroupdisk/data-art/grid-input/WorkflowReferences/'
113 
114  # Copied file path
115  # e.g. from ERROR Copied '../SimulationRun3FullSim/run_s4006/myHITS.pool.root' to '/eos/atlas/atlascerngroupdisk/proj-sit/gitlabci/MR63410_a84345c776e93f0d7f25d00c9e91e35bcb965d09/SimulationRun3FullSimChecks'
116  copied_file_match = re.search(r'^ERROR Copied.*', text, flags=re.MULTILINE)
117  if not copied_file_match:
118  print("FATAL: Could not find matching copied file")
119  sys.exit(1)
120  copied_file_path = copied_file_match.group().split('to')[1].strip().strip("'")+'/'
121 
122  # Reference file paths
123  ref_file_match = re.search(r'INFO Reading the reference file from location.*', text)
124  if not ref_file_match:
125  print("FATAL: Could not find matching reference file")
126  sys.exit(1)
127 
128  ref_file_path = ref_file_match.group().split('location')[1].strip()
129  existing_version_number= ref_file_path.split('/')[-2]
130  branch = ref_file_path.split('/')[-4]
131  new_version_number = 'v'+str(int(existing_version_number[1:])+1)
132  new_version_directory = eos_path_root+branch+'/'+ami_tag+'/'+new_version_number
133  old_version_directory = eos_path_root+branch+'/'+ami_tag+'/'+existing_version_number
134  # Copied file path
135  # e.g. from ERROR Copied '../SimulationRun3FullSim/run_s4006/myHITS.pool.root' to '/eos/atlas/atlascerngroupdisk/proj-sit/gitlabci/MR63410_a84345c776e93f0d7f25d00c9e91e35bcb965d09/SimulationRun3FullSimChecks'
136  copied_file_match = re.search(r'^ERROR Copied.*', text, flags=re.MULTILINE)
137  if not copied_file_match:
138  print("FATAL: Could not find matching copied file")
139  sys.exit(1)
140 
141  # Sanity checks
142  ami_tag_check = ref_file_path.split('/')[-3].strip()
143  if ami_tag_check!=ami_tag:
144  print('FATAL: Sanity check: "{}" from reference file path "{}" does not match ami tag "{}" extracted previously.'.format(ami_tag_check, ref_file_path, ami_tag))
145  sys.exit(1)
146 
147 
148  test = CITest(name=test_name, tag=ami_tag, mr=mr_number, date=human_readable_date, existing_ref = old_version_directory, existing_version = existing_version_number, new_version = new_version_number, new_version_directory = new_version_directory, copied_file_path = copied_file_path, digest_old=None, digest_new=None, type='DiffPool')
149  return test
150 

◆ process_digest_change()

def python.update_ci_reference_files.process_digest_change (   text,
  ami_tag,
  mr_number,
  human_readable_date,
  test_name 
)

Definition at line 151 of file update_ci_reference_files.py.

151 def process_digest_change(text, ami_tag, mr_number, human_readable_date, test_name):
152  # Some things aren't so relevant for digest changes
153  existing_version_number = None
154  new_version_directory = None
155  copied_file_path = None
156  new_version_number=None
157 
158  # differs from the reference 'q447_AOD_digest.ref' (<):
159  ref_file_match = re.search(r'(.*differs from the reference \')(.*)(\')', text)
160  if not ref_file_match:
161  print("FATAL: Could not find matching reference file")
162  sys.exit(1)
163  ref_file_path = ref_file_match.groups()[1]
164 
165  old_diff_lines = []
166  new_diff_lines = []
167  diff_started = False # Once we hit the beginning of the diff, we start recording
168  # Diff starts with e.g.
169  # ERROR The output 'q449_AOD_digest.txt' (>) differs from the reference 'q449_AOD_digest.ref' (<):
170  # and ends with next INFO line
171 
172  for line in text.split('\n'):
173  if 'differs from the reference' in line:
174  # Start of the diff
175  diff_started = True
176  elif diff_started:
177  if line.startswith('&lt;'):
178  old_diff_lines.append(line)
179  elif line.startswith('&gt;'):
180  new_diff_lines.append(line)
181  elif 'INFO' in line:
182  # End of the diff
183  break
184 
185  test = CITest(name=test_name, tag=ami_tag, mr=mr_number, date=human_readable_date, existing_ref = ref_file_path, existing_version = existing_version_number, new_version = new_version_number, new_version_directory = new_version_directory, copied_file_path = copied_file_path, digest_old=old_diff_lines, digest_new=new_diff_lines, type='Digest')
186  return test
187 

◆ process_log_file()

def python.update_ci_reference_files.process_log_file (   url,
  branch,
  test_name 
)
So now we have a URL to a failing test.
We need to check that the test is failing for the correct reason - namely a reference file which needs updating
The information we need to collect is:
- the AMI tag of the failing tests
- the merge request number
- the location of the reference file
- the location of the copied file
- the name of the test
- the new version number
- the new version directory

Definition at line 60 of file update_ci_reference_files.py.

60 def process_log_file(url, branch, test_name):
61  """So now we have a URL to a failing test.
62  We need to check that the test is failing for the correct reason - namely a reference file which needs updating
63  The information we need to collect is:
64  - the AMI tag of the failing tests
65  - the merge request number
66  - the location of the reference file
67  - the location of the copied file
68  - the name of the test
69  - the new version number
70  - the new version directory
71  """
72  page = requests.get(url)
73  text = page.text
74 
75  # First check that this looks like a test whose ref files need updating, bail otherwise
76  # INFO All q442 athena steps completed successfully
77  test_match = re.search(r'All (?P<ami_tag>\w+) athena steps completed successfully', text)
78  ami_tag = test_match.group('ami_tag') if test_match else None
79 
80  # We have two types of tests, but lets try to extract some common information
81  if not ami_tag:
82  # Okay, maybe it was truncated? Try again.
83  match_attempt_2 = re.search(r'AMIConfig (?P<ami_tag>\w+)', text)
84  if match_attempt_2:
85  ami_tag = match_attempt_2.group('ami_tag')
86 
87  if not ami_tag:
88  print('WARNING: Did not find an AMI tag in the test "{}". Ignoring.'.format(test_name))
89  return
90 
91  mr_match = re.search(r'NICOS_TestLog_MR-(?P<mr_number>\d+)-(?P<date>\d{4}-\d{2}-\d{2}-\d{2}-\d{2})', url)
92  if not mr_match:
93  print('FATAL: Could not process the URL as expected. Aborting.')
94  print(url)
95  sys.exit(1)
96 
97  mr_number = mr_match.group('mr_number')
98  date = mr_match.group('date')
99  human_readable_date = ':'.join(date.split('-')[0:3]) + " at " + ':'.join(date.split('-')[3:])
100 
101  if "Your change breaks the digest in test" in text:
102  # Okay, we have a digest change
103  failing_tests[branch].append(process_digest_change(text, ami_tag, mr_number, human_readable_date, test_name))
104 
105  if 'ERROR Your change breaks the frozen tier0 policy in test' in text or 'ERROR Your change breaks the frozen derivation policy in test' in text:
106  # DiffPool change
107  failing_tests[branch].append(process_diffpool_change(text, ami_tag, mr_number, human_readable_date, test_name))
108 
109  return
110 

◆ strip_href()

def python.update_ci_reference_files.strip_href (   href)

Definition at line 327 of file update_ci_reference_files.py.

327 def strip_href(href):
328  value = href[href.find('>')+1:] # Strip everything up to first >
329  value = value[:value.find('<')]
330  return value
331 

◆ strip_url()

def python.update_ci_reference_files.strip_url (   href)

Definition at line 322 of file update_ci_reference_files.py.

322 def strip_url(href):
323  url = href[href.find('"')+1:] # Strip everything up to first quotation mark
324  url = url[:url.find('"')]
325  return url
326 

◆ summarise_failing_tests()

def python.update_ci_reference_files.summarise_failing_tests (   check_for_duplicates = True)

Definition at line 356 of file update_ci_reference_files.py.

356 def summarise_failing_tests(check_for_duplicates = True):
357  print('Summary of tests which need work:')
358 
359  if not failing_tests:
360  print(" -> None found. Aborting.")
361  return None
362 
363  mr = None
364  reference_folders = []
365  for branch,tests in failing_tests.items():
366  print (' * Branch: {}'.format(branch))
367  for test in tests:
368  print(' - ', test)
369  if test.type == 'DiffPool':
370  if not test.new_version_directory:
371  print('FATAL: No path to "new version" for test {} of type DiffPool.'.format(test.name))
372  sys.exit(1)
373 
374  if os.path.exists(test.new_version_directory):
375  msg = f'WARNING: The directory {test.new_version_directory} already exists. Are you sure you want to overwrite the existing references?'
376  if input("%s (y/N) " % msg).lower() != 'y':
377  sys.exit(1)
378 
379  if (test.existing_ref not in reference_folders):
380  reference_folders.append(test.existing_ref)
381  elif check_for_duplicates:
382  print('FATAL: Found two tests which both change the same reference file: {}, which is not supported.'.format(test.existing_ref))
383  print('Consider running again in --test-run mode, to get a copy of the copy commands that could be run.')
384  print('The general advice is to take the largest file (since it will have the most events), and/or take the non-legacy one.')
385  sys.exit(1)
386  mr = test.mr
387  return 'https://gitlab.cern.ch/atlas/athena/-/merge_requests/'+mr
388 

◆ update_reference_files()

def python.update_ci_reference_files.update_reference_files (   actually_update = True,
  update_local_files = False 
)

Definition at line 188 of file update_ci_reference_files.py.

188 def update_reference_files(actually_update=True, update_local_files=False):
189  print
190  print('Updating reference files')
191  print('========================')
192  commands = []
193  for branch, tests in failing_tests.items():
194  for test in tests:
195  print('Processing test: {} on branch {}'.format(test.name, branch))
196  if test.type == 'DiffPool':
197  print(' * This is a DiffPool test, and currently has version {} of {}. Will update References.py with new version.'.format(test.existing_version, test.tag))
198  if actually_update:
199  print(' -> The new version is: {}. Creating directory and copying files on EOS now.'.format(test.new_version))
200  create_dir_and_copy_refs(test, True)
201  else:
202  # We will print these later, so we can sanity check them when in test mode
203  commands.extend(create_dir_and_copy_refs(test, False))
204  # Remove any duplicates, whilst preserving the order
205  commands = list(dict.fromkeys(commands))
206 
207  # Now, update local References.py file
208  if update_local_files:
209  data = []
210  if debug:
211  print ('Updating local References.py file with new version {} for tag {}'.format(test.new_version, test.tag))
212  line_found = False
213  with open('Tools/WorkflowTestRunner/python/References.py', 'r') as f:
214  lines = f.readlines()
215  for line in lines:
216  if test.tag in line:
217  if test.existing_version in line:
218  line = line.replace(test.existing_version, test.new_version)
219  else:
220  print('')
221  print('** WARNING: For tag {} we were looking for existing version {}, but the line in the file is: {}'.format(test.tag, test.existing_version, line), end='')
222  print('** Are you sure your branch is up-to-date with main? We cannot update an older version of References.py!')
223  line_found = True
224  data.append(line)
225 
226  if not line_found:
227  print('** WARNING - no matching line was found for the AMI tag {} in References.py. Are you sure your branch is up-to-date with main? We cannot update an older version of References.py!'.format(test.tag))
228 
229  with open('Tools/WorkflowTestRunner/python/References.py', 'w') as f:
230  f.writelines(data)
231  elif test.type == 'Digest' and update_local_files:
232  print(' * This is a Digest test. Need to update reference file {}.'.format(test.existing_ref))
233  data = []
234 
235  diff_line=0 # We will use this to keep track of which line in the diff we are on
236  with open('Tools/PROCTools/data/'+test.existing_ref, 'r') as f:
237  lines = f.readlines()
238  for current_line, line in enumerate(lines):
239  split_curr_line = line.split()
240  if (split_curr_line[0] == 'run'): # Skip header line
241  data.append(line)
242  continue
243 
244  # So, we expect first two numbers to be run/event respectively
245  if (not split_curr_line[0].isnumeric()) or (not split_curr_line[1].isnumeric()):
246  print('FATAL: Found a line in current digest which does not start with run/event numbers: {}'.format(line))
247  sys.exit(1)
248 
249  split_old_diff_line = test.digest_old[diff_line].split()
250  split_old_diff_line.pop(0) # Remove the < character
251  split_new_diff_line = test.digest_new[diff_line].split()
252  split_new_diff_line.pop(0) # Remove the > character
253 
254  # Let's check to see if the run/event numbers match
255  if split_curr_line[0] == split_old_diff_line[0] and split_curr_line[1] == split_old_diff_line[1]:
256  # Okay so run/event numbers match. Let's just double-check it wasn't already updated
257  if split_curr_line!=split_old_diff_line:
258  print('FATAL: It seems like this line was already changed.')
259  print('Line we expected: {}'.format(test.old_diff_lines[diff_line]))
260  print('Line we got : {}'.format(line))
261  sys.exit(1)
262 
263  # Check if the new run/event numbers match
264  if split_curr_line[0] == split_new_diff_line[0] and split_curr_line[1] == split_new_diff_line[1]:
265  #Replace the existing line with the new one, making sure we right align within 12 characters
266  data.append("".join(["{:>12}".format(x) for x in split_new_diff_line])+ '\n')
267  if ((diff_line+1)<len(test.digest_old)):
268  diff_line+=1
269  continue
270 
271  # Otherwise, we just keep the existing line
272  data.append(line)
273 
274  print(' -> Updating PROCTools digest file {}'.format(test.existing_ref))
275  with open('Tools/PROCTools/data/'+test.existing_ref, 'w') as f:
276  f.writelines(data)
277  return commands
278 
279 

Variable Documentation

◆ action

python.update_ci_reference_files.action

Definition at line 393 of file update_ci_reference_files.py.

◆ args

python.update_ci_reference_files.args

Definition at line 394 of file update_ci_reference_files.py.

◆ author

python.update_ci_reference_files.author

Definition at line 416 of file update_ci_reference_files.py.

◆ commands

python.update_ci_reference_files.commands

Definition at line 436 of file update_ci_reference_files.py.

◆ debug

python.update_ci_reference_files.debug

Definition at line 58 of file update_ci_reference_files.py.

◆ description

python.update_ci_reference_files.description

Definition at line 390 of file update_ci_reference_files.py.

◆ dirs_created

python.update_ci_reference_files.dirs_created

Definition at line 57 of file update_ci_reference_files.py.

◆ failing_tests

python.update_ci_reference_files.failing_tests

Definition at line 56 of file update_ci_reference_files.py.

◆ formatter_class

python.update_ci_reference_files.formatter_class

Definition at line 391 of file update_ci_reference_files.py.

◆ gl_project

python.update_ci_reference_files.gl_project

Definition at line 414 of file update_ci_reference_files.py.

◆ help

python.update_ci_reference_files.help

Definition at line 392 of file update_ci_reference_files.py.

◆ local_branch

python.update_ci_reference_files.local_branch

Definition at line 418 of file update_ci_reference_files.py.

◆ mr

python.update_ci_reference_files.mr

Definition at line 415 of file update_ci_reference_files.py.

◆ mr_url

python.update_ci_reference_files.mr_url

Definition at line 408 of file update_ci_reference_files.py.

◆ msg

python.update_ci_reference_files.msg

Definition at line 427 of file update_ci_reference_files.py.

◆ not_in_athena_dir

python.update_ci_reference_files.not_in_athena_dir

Definition at line 430 of file update_ci_reference_files.py.

◆ parser

python.update_ci_reference_files.parser

Definition at line 390 of file update_ci_reference_files.py.

◆ remote

python.update_ci_reference_files.remote

Definition at line 417 of file update_ci_reference_files.py.

◆ shell

python.update_ci_reference_files.shell

Definition at line 430 of file update_ci_reference_files.py.

◆ update_local_files

python.update_ci_reference_files.update_local_files

Definition at line 428 of file update_ci_reference_files.py.

python.update_ci_reference_files.process_CI_Builds_Summary
def process_CI_Builds_Summary(project)
Definition: update_ci_reference_files.py:332
vtune_athena.format
format
Definition: vtune_athena.py:14
python.update_ci_reference_files.process_digest_change
def process_digest_change(text, ami_tag, mr_number, human_readable_date, test_name)
Definition: update_ci_reference_files.py:151
CaloCellPos2Ntuple.int
int
Definition: CaloCellPos2Ntuple.py:24
python.update_ci_reference_files.strip_href
def strip_href(href)
Definition: update_ci_reference_files.py:327
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
python.update_ci_reference_files.update_reference_files
def update_reference_files(actually_update=True, update_local_files=False)
Definition: update_ci_reference_files.py:188
python.update_ci_reference_files.strip_url
def strip_url(href)
Definition: update_ci_reference_files.py:322
PlotPulseshapeFromCool.input
input
Definition: PlotPulseshapeFromCool.py:106
python.update_ci_reference_files.process_CI_Tests_json
def process_CI_Tests_json(data)
Definition: update_ci_reference_files.py:316
histSizes.list
def list(name, path='/')
Definition: histSizes.py:38
python.update_ci_reference_files.extract_links_from_json
def extract_links_from_json(url)
Definition: update_ci_reference_files.py:347
python.update_ci_reference_files.create_dir_and_copy_refs
def create_dir_and_copy_refs(test, actually_update=False)
Definition: update_ci_reference_files.py:280
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:25
python.update_ci_reference_files.process_log_file
def process_log_file(url, branch, test_name)
Definition: update_ci_reference_files.py:60
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
Trk::open
@ open
Definition: BinningType.h:40
python.update_ci_reference_files.process_diffpool_change
def process_diffpool_change(text, ami_tag, mr_number, human_readable_date, test_name)
Definition: update_ci_reference_files.py:111
str
Definition: BTagTrackIpAccessor.cxx:11
python.update_ci_reference_files.summarise_failing_tests
def summarise_failing_tests(check_for_duplicates=True)
Definition: update_ci_reference_files.py:356
Trk::split
@ split
Definition: LayerMaterialProperties.h:38