ATLAS Offline Software
diff-jobo-cfg.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
4 
5 # @file: diff-jobo-cfg.py
6 # @purpose: check that 2 jobosvc.ascii files (produced with find_cfg_dups.py)
7 # have same content (both in configurables and properties)
8 # @author: Sebastien Binet <binet@cern.ch>
9 # Adrien Renaud <renaud@lal.in2p3.fr>
10 # @date: May 2010
11 #
12 # @example:
13 #
14 # diff-jobo-cfg ref.josvc.ascii chk.josvc.ascii
15 #
16 
17 __author__ = "Sebastien Binet, Adrien Renaud"
18 
19 import sys
20 import os
21 
22 from optparse import OptionParser
23 
24 def dump_seq(seq):
25  for i in seq:
26  print (i)
27  pass
28 
29 def cxx_sort(dpp):
30  from collections import defaultdict
31  cxx_dpp = defaultdict(list)
32 
33  for k,v in dpp.iteritems():
34  #print (v['cxx_type'])
35  cxx_dpp[v['cxx_type']].append({k:v})
36 
37  for k,v in cxx_dpp.iteritems():
38  print ('---',k)
39  for vv in v:
40  print ('------',vv.keys())
41  print ('---------',vv.values())#['comp_type']
42  return cxx_dpp
43 
44 def load_cfg_file(fname):
45  """return the dictionary of components and their properties
46  """
47  comps_db = {}
48  try:
49  import shelve
50  comps_db = shelve.open(fname, 'r')
51  return comps_db['all-cfgs']
52  except Exception:
53  exec (open(fname).read(), comps_db)
54  return comps_db['d']
55 
56 def dict_diff(ref, chk):
57  """ Return a dict of keys that differ with another config object. If a value is
58  not found in one fo the configs, it will be represented by KEYNOTFOUND.
59  @param ref: First dictionary to diff.
60  @param chk: Second dicationary to diff.
61  @return diff: Dict of Key => (ref.val, chk.val)
62  """
63  diff = {}
64  # Check all keys in ref dict
65  for k in ref.iterkeys():
66  if not (k in chk):
67  diff[k] = (ref[k], '<KEYNOTFOUND>')
68  elif (ref[k] != chk[k]):
69  diff[k] = (ref[k], chk[k])
70  # Check all keys in chk dict to find missing
71  for k in chk.iterkeys():
72  if not (k in ref):
73  diff[k] = ('<KEYNOTFOUND>', chk[k])
74  return diff
75 
76 def cmp_component_db(ref, chk, verbose=True):
77  """ compare 2 dicts of components
78  dicts are of the form:
79  { 'comp_type' : <name of component>,
80  'cxx_type' : <C++ type of the component>,
81  'props' : { 'property-name' : 'property-value', }
82  }
83  """
84  common_keys = []
85  ref_keys = set(ref.keys())
86  chk_keys = set(chk.keys())
87 
88  common_keys = ref_keys & chk_keys
89  ref_only_keys = ref_keys - chk_keys
90  chk_only_keys = chk_keys - ref_keys
91 
92  print ("::: components in both files: [%5s]" % (len(common_keys),))
93  print ("::: components in ref only: [%5s]" % (len(ref_only_keys),))
94  if len(ref_only_keys)>0:
95  dump_seq(ref_only_keys)
96  print ("="*80)
97  print ("::: components in chk only: [%5s]" % (len(chk_only_keys),))
98  if len(chk_only_keys)>0:
99  dump_seq(chk_only_keys)
100  print ("="*80)
101 
102  diff = []
103  for comp_name in common_keys:
104  comp_ref = ref[comp_name]
105  comp_chk = chk[comp_name]
106 
107  ref_props = sorted([(k,v) for k,v in comp_ref['props'].iteritems()])
108  chk_props = sorted([(k,v) for k,v in comp_chk['props'].iteritems()])
109  if ref_props != chk_props:
110  diff.append((comp_name, ref_props, chk_props,
111  dict_diff(ref=comp_ref['props'],
112  chk=comp_chk['props'])))
113  pass
114 
115  print ("::: components with different properties: [%5s]" % (len(diff),))
116  for name, ref_props, chk_props, diff_props in diff:
117  print ("::: - component: [%s]" % (name,))
118  for prop_name, prop_value in diff_props.iteritems():
119  ref_value = prop_value[0]
120  chk_value = prop_value[1]
121  if isinstance(ref_value, list):
122  ref_value = sorted(ref_value)
123  if isinstance(chk_value, list):
124  chk_value = sorted(chk_value)
125 
126  if isinstance(ref_value, list) and isinstance(chk_value, list):
127  dref_value = set(ref_value) - set(chk_value)
128  dchk_value = set(chk_value) - set(ref_value)
129  ref_value = sorted(list(dref_value))
130  chk_value = sorted(list(dchk_value))
131  print ("-%s: %r" %(prop_name, ref_value,))
132  print ("+%s: %r" %(prop_name, chk_value,))
133 
134 
135  if (len(ref_only_keys) > 0 or
136  len(chk_only_keys) > 0 or
137  len(diff) > 0):
138  return 1
139  return 0
140 
141 if __name__ == "__main__":
142 
143  parser = OptionParser(
144  usage="usage: %prog [options] [-r] ref.josvc.ascii [-f] chk.josvc.ascii"
145  )
146  _add = parser.add_option
147 
148  _add( "-r",
149  "--ref",
150  dest = "ref_fname",
151  help = "The path to the first josvc.ascii file to analyze" )
152 
153  _add( "-f",
154  "--file",
155  dest = "chk_fname",
156  help = "The path to the second josvc.ascii file to analyze" )
157 
158 # _add("-o", "--output",
159 # dest = "o_fname",
160 # default = "cfg.diff",
161 # help = "file where to store the output of `which diff` run on the input files given to %PROG")
162 
163  _add( "-v",
164  "--verbose",
165  action = "store_true",
166  dest = "verbose",
167  default = False,
168  help = "Switch to activate verbose printout" )
169 
170 
171  (options, args) = parser.parse_args()
172 
173  if len(args) > 0 and args[0][0] != "-":
174  options.ref_fname = args[0]
175  pass
176  if len(args) > 1 and args[1][0] != "-":
177  options.chk_fname = args[1]
178  pass
179 
180  if (options.chk_fname is None or options.ref_fname is None) :
181  str(parser.print_help() or "")
182  sys.exit(1)
183  pass
184 
185  chk_fname = os.path.expandvars(os.path.expanduser(options.chk_fname))
186  ref_fname = os.path.expandvars(os.path.expanduser(options.ref_fname))
187 
188  print (":"*80)
189  print ("::: comparing configurations")
190  print ("::: ref: %s" % ref_fname)
191  ref_db = load_cfg_file(ref_fname)
192  print ("::: -> [%d] components" % (len(ref_db.keys()),))
193  print ("::: chk: %s" % chk_fname)
194  chk_db = load_cfg_file(chk_fname)
195  print ("::: -> [%d] components" % (len(chk_db.keys()),))
196 
197  sc = cmp_component_db(ref_db, chk_db, options.verbose)
198 
199  if sc==0:
200  print ("::: all good")
201  else:
202  print ("::: configurations differ !")
203  print ("::: bye.")
204  print (":"*80)
205  sys.exit(sc)
read
IovVectorMap_t read(const Folder &theFolder, const SelectionCriterion &choice, const unsigned int limit=10)
Definition: openCoraCool.cxx:569
DerivationFramework::TriggerMatchingUtils::sorted
std::vector< typename R::value_type > sorted(const R &r, PROJ proj={})
Helper function to create a sorted vector from an unsorted range.
python.Bindings.iteritems
iteritems
Definition: Control/AthenaPython/python/Bindings.py:823
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
diff-jobo-cfg.dict_diff
def dict_diff(ref, chk)
Definition: diff-jobo-cfg.py:56
diff-jobo-cfg.dump_seq
def dump_seq(seq)
Definition: diff-jobo-cfg.py:24
histSizes.list
def list(name, path='/')
Definition: histSizes.py:38
CxxUtils::set
constexpr std::enable_if_t< is_bitmask_v< E >, E & > set(E &lhs, E rhs)
Convenience function to set bits in a class enum bitmask.
Definition: bitmask.h:232
diff-jobo-cfg.cxx_sort
def cxx_sort(dpp)
Definition: diff-jobo-cfg.py:29
Trk::open
@ open
Definition: BinningType.h:40
diff-jobo-cfg.load_cfg_file
def load_cfg_file(fname)
Definition: diff-jobo-cfg.py:44
str
Definition: BTagTrackIpAccessor.cxx:11
diff-jobo-cfg._add
_add
Definition: diff-jobo-cfg.py:146
diff-jobo-cfg.cmp_component_db
def cmp_component_db(ref, chk, verbose=True)
Definition: diff-jobo-cfg.py:76