14 from AthenaConfiguration.iconfTool.models.loaders
import loadConfigFile, baseParser, componentRenamingDict, loadDifferencesFile, isReference
18 knowndifference=
"\033[35m"
39 "-q",
"--quiet", action=
"store_true", help=
"Don't print command arguments"
42 "-p",
"--printConf", action=
"store_true", help=
"Prints entire configuration"
45 "--printComps", action=
"store_true", help=
"Prints only the components"
48 "--diff", dest=
"diff", action=
"store_true", help=
"Diffs two files"
50 parser.add_argument(
"--toJSON", action=
"store_true", help=
"Convert pickle to JSON file")
51 parser.add_argument(
"--toYAML", action=
"store_true", help=
"Convert pickle to YAML file")
52 parser.add_argument(
"--toPickle", action=
"store_true", help=
"Convert JSON to pickle file")
54 parser.add_argument(
"file", nargs=
"+", help=
"Files to work with")
57 help=
"Don't report components existing in only one of the two configurations",
65 help=
"Ignore order for sequence properties matching regex",
68 "--allComponentPrint",
69 help=
"Print all component if there are differences in any of its properties",
73 "--printIdenticalComponents",
74 help=
"Print all components even, if there are no differences.",
78 "--printIdenticalPerParameter",
79 help=
"Print all parameters in component with difference even, if there are no differences.",
82 parser.add_argument(
"--knownDifferencesFile",
83 help=
"Ignore differences enlisted in file (to be used only with diffing)")
85 parser.add_argument(
"--color",
86 help=
"Use colored output even for file output (useful when piping to less -R to to HTML conversion",
89 parser.add_argument(
"-s",
"--structured",
92 help=
"Print only a single component, in a structured manner (reflecting components parent children)")
94 parser.add_argument(
"--includeClassesSub",
97 help=
"Below the top-level, also include the components selected by the given classname (anchored regular expression)")
99 parser.add_argument(
"--includeSequences",
100 help=
"Include sequences in the structured printout",
103 parser.add_argument(
"--classes",
105 help=
"Only show class names")
107 parser.add_argument(
"--uniqueClasses",
109 help=
"Only show unique classes")
111 parser.add_argument(
"--showComponentName",
112 help=
"Show component name with --classes",
115 parser.add_argument(
"--maxDepth",
116 help=
"Maximum depth for structured printout",
120 args = parser.parse_args()
122 print(
"Run with arguments:")
123 print(
"confTool.py",
" ".
join(sys.argv[1:]))
131 if not args.quiet
and args.ignoreIrrelevant:
132 print(f
"Properties to ignore: {args.ignore}")
134 if not sys.stdout.isatty()
and not args.color:
137 for fileName
in args.file:
142 for fileName
in args.file:
147 for fileName
in args.file:
151 if args.toJSON
or args.toYAML
or args.toPickle:
152 if len(args.file) != 1:
154 "ERROR, can convert single file at a time, got: %s" % args.file
159 oFileName = args.file[0].
replace(
".pkl",
".json")
160 with open(oFileName,
"w")
as oFile:
161 json.dump(conf, oFile, indent=4, sort_keys=
True, ensure_ascii=
True)
164 oFileName = args.file[0].
replace(
".pkl",
".yml")
165 with open(oFileName,
'w')
as oFile:
166 yaml.dump(conf, oFile, default_flow_style=
False)
169 oFileName = args.file[0].
replace(
".json",
".pkl")
170 with open(oFileName,
"wb")
as oFile:
172 pickle.dump(item, oFile)
173 print(
"Wrote " + args.file[0] +
" to " + oFileName)
176 if len(args.file) != 2:
178 "ERROR, can diff exactly two files at a time, got: %s"
183 args.renameComps=
None
184 args.renameCompsFile=
None
186 global knownDifferences
187 if args.knownDifferencesFile:
189 exit_code =
_compareConfig(configRef, configChk, args, color) != 0
195 for k, settings
in conf.items():
196 print(f
"{color.component}{k}{color.reset}")
197 if isinstance(settings, dict):
198 for prop,val
in settings.items():
199 print(f
" {color.property}{prop} = {color.value}{val}")
204 for k, item
in conf.items():
205 if isinstance(item, dict):
209 showClasses = args.classes
or args.uniqueClasses
210 def _oneCompPrint(d, comp, cl, done={}, depth=0, indent = "") -> list:
211 show = ((
not showClasses
or cl
is not None)
and
212 (depth>0
and any([re.match(f
"({regex})$",cl)
for regex
in args.includeClassesSub]))
or
213 ((
not args.includeClasses
or any([re.match(f
"({regex})$",cl)
for regex
in args.includeClasses]))
and
214 not any([re.match(f
"({regex})$",cl)
for regex
in args.excludeClasses])
and
215 not any([re.match(f
"({regex})$",comp)
for regex
in args.excludeComponents])))
217 if not args.uniqueClasses:
219 cc = cl
if (
not args.showComponentName)
else f
"{cl}/{comp}"
220 print(f
"{indent}{cc}")
222 print(f
"{indent}{comp}")
223 newindent = indent +
" - "
225 elif args.includeSequences
and ((len(d) == 1
and "Members" in d)
or (comp ==
"ApplicationMgr" and "TopAlg" in d)):
229 if showClasses
and args.maxDepth
is not None and depth > args.maxDepth:
233 for prop,val
in sorted(d.items()):
234 if not showClasses
and show:
235 print(f
"{indent} {prop} = {val}")
236 if args.maxDepth
is not None and depth > args.maxDepth:
238 if not args.includeSequences
and (prop ==
"Members" or prop ==
"TopAlg"):
241 if showClasses
and not c:
243 if ref
in conf
and ref
not in done:
244 r = _oneCompPrint(conf[ref], ref, c
or ref, {**done, ref:
True}, depth, newindent)
245 if args.uniqueClasses:
248 return [[cl, comp, sub]]
254 for ref,cl
in isReference(start,
None, conf)
or [(
None,
None)]:
255 if ref
and ref
in conf:
256 settings = conf.get(ref)
257 if isinstance(settings, dict):
258 r = _oneCompPrint(settings, ref, cl
or ref)
260 elif not showClasses:
263 print(f
"{comp} is absent in the config, unfortunately the name has to be exact")
265 if not args.uniqueClasses:
269 """copy structure without the component names, so equality tests for the same class names."""
276 from collections
import defaultdict
277 cls = defaultdict(list)
278 def _oneCompPrune(top):
279 """Duplicate classes (with identical structure below) are listed, without substructure
280 - or not listed at all if they already showed up at this level.
281 Classes with the same name are grouped next to each other and shown with their component name."""
284 ord = defaultdict(list)
285 for cl,comp,sub
in top:
286 ord[cl].
append([cl,comp,sub])
290 dupcls = defaultdict(int)
291 for o
in ord.values():
292 for cl,comp,sub
in o:
293 comp = comp.split(
'.')[-1]
295 if cl
in cls
and nsub
in cls[cl]:
301 r = _oneCompPrune(sub)
303 new.append([cl,comp,r])
306 if not args.showComponentName:
312 def _structPrint(top, indent = ""):
313 for cl,comp,sub
in top:
314 cc = f
"{cl}/{comp}" if args.showComponentName
else \
315 f
"{cl} ({comp})" if comp
is not None else cl
316 print(f
"{indent}{cc}")
318 _structPrint(sub, indent +
" - ")
320 _structPrint(_oneCompPrune(sub))
325 allComps =
list(
set(configRef.keys()) |
set(configChk.keys()))
328 print(
"Step 1: reference file #components:", len(configRef))
329 print(
"Step 2: file to check #components:", len(configChk))
331 print(f
"{color.difference}Differences in components {color.first}Settings in 1st file {color.second}Settings in 2nd file{color.reset}")
333 componentReverseRenamig = {v: k
for k, v
in componentRenamingDict.items()}
334 def _componentDescription(comp_name):
335 return (comp_name+
" renamed from " + componentReverseRenamig[comp_name])
if comp_name
in componentReverseRenamig
else comp_name
338 for component
in allComps:
339 if component
not in configRef:
340 if not args.ignoreMissing:
342 f
"\n{color.second} Component ",
343 _componentDescription(component),
344 f
"{color.reset} only in 2nd file {color.reset} \n",
348 if component
not in configChk:
349 if not args.ignoreMissing:
351 f
"\n{color.first} Component",
352 _componentDescription(component),
353 f
"{color.reset}only in 1st file {color.reset} \n",
357 refValue = configRef[component]
358 chkValue = configChk[component]
360 if chkValue == refValue:
361 if args.printIdenticalComponents:
362 print(
"Component", _componentDescription(component),
"identical")
364 print(f
"{color.difference}Component", _componentDescription(component), f
"may differ{color.reset}")
365 if not args.allComponentPrint:
366 countDifferent =
_compareComponent(refValue, chkValue,
"\t", args, component,
"", color)
367 if countDifferent == 0:
368 print(
" but all are suppressed by renaming/known differences/...")
370 print(f
" {color.difference} {countDifferent} relevant differences{color.reset}")
373 f
"\t{color.first}Ref{color.reset}\t",
374 sorted(configRef[component].
items(), key=
lambda kv: kv[0]),
377 f
"\t{color.second}Chk{color.reset}\t",
378 sorted(configChk[component].
items(), key=
lambda kv: kv[0]),
380 return countDifferent
384 """Ensure all numeric values are of the same type (int or float)"""
385 if any(isinstance(val, float)
for val
in values):
386 return tuple(
float(val)
for val
in values)
387 elif all(isinstance(val, int)
for val
in values):
388 return tuple(
int(val)
for val
in values)
394 if comp
in knownDifferences:
395 if prop
in knownDifferences[comp]:
396 acceptedDifference = knownDifferences[comp][prop]
397 if acceptedDifference == (
None,
None):
399 if acceptedDifference[0]
is None:
400 return chkVal == acceptedDifference[1]
401 if acceptedDifference[1]
is None:
402 return refVal == acceptedDifference[0]
404 return refVal == acceptedDifference[0]
and chkVal == acceptedDifference[1]
408 """ Rename values in reference as long as they are hashable (and in renamingDict)
409 It is a bit of heuristics that is invoved, the assumption is that the property has value of the form A/B if it is name of the compoemnt or it is a just single string"""
410 refList = refVal
if isinstance(refVal, list)
else [refVal]
413 if isinstance(v, str):
414 if "/" in v
and len(v.split(
"/")) == 2:
415 compType,compName = v.split(
"/")
416 newName = componentRenamingDict.get(compName, compName)
417 updatedRef.append( f
"{compType}/{newName}" )
419 updatedRef.append( componentRenamingDict.get(v, v) )
422 return updatedRef
if isinstance(refVal, list)
else updatedRef[0]
426 if isinstance(compRef, dict):
427 allProps =
list(
set(compRef.keys()) |
set(compChk.keys()))
430 for prop
in allProps:
431 if prop
not in compRef.keys():
433 print(f
"{prefix}{color.property}{prop} = {color.second}{compChk[prop]} {color.reset} only in 2nd file {color.reset}")
436 print(f
"{prefix}known difference in: {prop}")
439 if prop
not in compChk.keys():
441 print(f
"{prefix}{color.property}{prop} = {color.first}{compRef[prop]} {color.reset} only in 1st file {color.reset}")
444 print(f
"{prefix}known difference in: {prop}")
447 refVal = compRef[prop]
448 chkVal = compChk[prop]
451 refVal = ast.literal_eval(
str(refVal))
if refVal
else ""
452 chkVal = ast.literal_eval(
str(chkVal))
if chkVal
else ""
463 if not args.printIdenticalPerParameter:
466 print(f
"{prefix}known difference in: {prop}")
467 if not args.printIdenticalPerParameter:
470 diffmarker = f
" {color.difference}<<{color.reset}"
472 if not (component ==
"IOVDbSvc" and prop ==
"Folders"):
473 print(f
"{prefix}{color.property}{prop} = {color.first} {refVal} {color.reset} vs {color.second} {chkVal} {color.reset} {diffmarker}")
475 if refVal
and ( isinstance(refVal, list)
or isinstance(refVal, dict) ):
476 if component ==
"IOVDbSvc" and prop ==
"Folders":
480 refVal, chkVal,
"\t" + prefix +
">> ", args, component, prop, color
483 elif isinstance(compRef, (list, tuple, set))
and len(compRef) > 1:
485 if isinstance(compRef[0], list):
488 if len(compChk) > 0
and isinstance(compChk[0], list):
492 if compRef == compChk:
493 return countDifferent
499 print(f
"{prefix} {color.reset}only in 1st file : {color.first} {diffRef} {color.reset}")
502 print(f
"{prefix} {color.reset}only in 2nd file : {color.second} {diffChk} {color.reset}")
505 if len(compRef) == len(compChk):
507 if any(re.match(f
"^{regex}$",f
"{component}.{propname}")
for regex
in args.ignoreOrder):
508 print(f
"{prefix} : {color.knowndifference} ^^ Different order ignored ^^ {color.reset}")
510 print(f
"{prefix} : {color.difference} ^^ Different order ^^ {color.reset}")
513 for i, (refVal, chkVal)
in enumerate(zip(compRef, compChk)):
515 print(f
"{prefix} : {color.first} {refVal} {color.reset} vs {color.second} {chkVal} {color.reset} {color.difference}<< at index {i} {color.reset}")
517 refVal, chkVal,
"\t" + prefix +
">> ", args,
"",
"", color
519 return countDifferent
524 db_match = re.search(
r"<db>(.*)</db>", definition)
526 result[
"db"] = db_match.group(1)
527 definition = definition.replace(db_match.group(0),
"")
529 key_match = re.search(
r"<key>(.*)</key>", definition)
531 result[
"key"] = key_match.group(1)
532 definition = definition.replace(key_match.group(0),
"")
534 tag_match = re.search(
r"<tag>(.*)</tag>", definition)
536 result[
"tag"] = tag_match.group(1)
537 definition = definition.replace(tag_match.group(0),
"")
539 cache_match = re.search(
r"<cache>(.*)</cache>", definition)
541 definition = definition.replace(cache_match.group(0),
"")
543 noover_match = re.search(
r"<noover/>", definition)
545 result[
"noover"] =
True
546 definition = definition.replace(noover_match.group(0),
"")
548 result[
"name"] = definition.strip()
550 return json.dumps(result)
563 if __name__ ==
"__main__":