10 This module contains tools to write a DQConfiguration in han format
12 from __future__
import print_function
15 from DQConfMakerBase.Helpers
import BaseException, toList
16 from DQConfMakerBase.DQElements
import DQBase
17 from functools
import reduce
25 This exception is thrown when an error writing the han configuration is encountered
29 BaseException.__init__(
30 self,
"Cannot create the han file ("+
str(reason)+
")")
38 This exception is thrown when an error creating the han configuration is encountered
42 BaseException.__init__(
43 self,
"Cannot create configuration ("+
str(reason)+
")")
49 class Node(DQHanConfMaker.Node):
51 This is the base class for each element that can be written in a valid han configuration file.
53 - subnodes : a list of Node instances
54 - attributes : a list of attributes associated with the node
55 - acceptChild : a list of Node types that can be accepted as sub nodes
56 - nodeType : the type of the node
57 - name : the name of the node
68 self.
acceptChild = [Node.ALGORITHM, Node.DIR, Node.DOCUMENT, Node.HIST, Node.OUTPUT,
69 Node.REFERENCE, Node.THRESHOLD, Node.LIMIT, Node.COMPALG]
78 convert the object in a valid han script block
89 convert the object in a formatted han string
95 writer = io.BytesIO()
if six.PY2
else io.StringIO()
96 if encoding
is not None:
98 writer = codecs.lookup(encoding)[3](writer)
99 self.
writehan(writer,
"", indent, newl)
100 return writer.getvalue()
107 def writehan(self, writer, indent="", addindent="", newl=""):
109 Converts the object in a han string and writes it in the writer object
112 indent+DQHanConfMaker._get_NodeType(self.
nodeType)+
" "+self.
name)
115 writer.write(
" { %s" % (newl))
118 writer.write(
"%s %s = %s%s" % (indent, key, attribute, newl))
121 node.writehan(writer, indent+addindent, addindent, newl)
123 writer.write(
"%s}%s" % (indent, newl))
130 Add a sub node to this node
134 msg =
" Node: "+DQHanConfMaker._get_NodeType(self.
nodeType)
135 msg +=
" Cannot have a child of type:", child.nodeType
139 "Object:"+
str(child)+
" is not a valid Node")
149 The attribute identified by key is added to this node
160 Removes attribute identified by key
172 Gets the attribute identified by the key, None if not found
184 Returns the sub-node identified by name and nodetype
188 if sn.nodeType == nodetype
and sn.name == name:
200 A han histogram element is a Node with attributes:
202 - the algorithm to be used
203 - the output where to store the dq result
213 def __init__(self, histogram, algorithm, annotations={}, output=DQHanConfMaker._default_output, attributes={}):
215 Creates a han histogram configuration element
217 Node.__init__(self, histogram)
222 if 'Weight' in attributes:
224 for annotation
in annotations:
231 Retrieves the output attribute
239 Retrieves the algorithm attribute
249 The han representation of a DQAlgorithm
259 def __init__(self, name, algoname, libname='libdqm_algorithms.so', reference=None, thresholds=None,
262 Creates a han algorithm configuration element
264 Node.__init__(self, name)
274 for parameter
in parameters:
283 The han representation of a CompositeAlgorithm
290 def __init__(self, name, subalgnames=[], libnames=[]):
292 Creates a han composite algorithm configuration element
294 Node.__init__(self, name)
306 The han representation of a DQReference
313 def __init__(self, name, histogramname, file, info=None):
315 Creates a han reference configuration element
317 Node.__init__(self, name)
331 The han representation of a limit for a HanThreshold
340 Creates a han limit object
342 Node.__init__(self, name)
354 The han representation of a DQThreshold
361 Creates a han threshold configuration element
363 Node.__init__(self, name)
369 Adds a limit to the threshold
370 @param name: the limits name
371 @param warning: the limit value for the warning level
372 @param error: the limit value for the error level
376 "The limit: "+name+
" already exists for threshold: "+self.
name)
377 limit =
HanLimit(name, warning, error)
386 Class representing a dir element
387 A han dir element is a Node with sub nodes of type HanDir or HanHisto
394 Creates a han directory element
396 Node.__init__(self, name)
405 Adds a subdirectory called name to the current directory and returns the newly created sub directory
418 def addHistogram(self, histogram, algorithm, annotations={}, output=DQHanConfMaker._default_output, attributes={}):
420 Adds to the directory a histogram specifying the histogram name, the algorithm and the output path.
423 annotations, output, attributes)
431 returns the sub-directory called name
441 Class representing a output element
442 A han output element is a Node with sub nodes of type output and with the optional attribute algorithm
444 - father: the HanOutput that contains this HanOutput
453 def __init__(self, name, algorithm=None, father=None, annotations={}, attributes={}):
455 Creates an output han element
457 Node.__init__(self, name)
463 if 'Weight' in attributes:
465 for annotation
in annotations:
472 Gets the algorithm attribute
483 def addOutput(self, name, algorithm, annotations, attributes):
485 Adds a sub output element with name and algorithm
487 subnode =
HanOutput(name, algorithm, self, annotations, attributes)
497 Creates the complete path of this output directory
513 Class representing a han document
518 def __init__(self, top_level_algorithm='WorstCaseSummary'):
520 Creates an empty han document with a top level output
522 HanDir.__init__(self,
'')
523 HanOutput.__init__(self,
'')
525 self.
acceptChild = [Node.ALGORITHM, Node.DIR, Node.HIST, Node.OUTPUT, Node.REFERENCE, Node.THRESHOLD,
536 def addOutput(self, name, algorithm, annotations={}, attributes={}):
538 Adds a sub output element with name and algorithm
541 return HanOutput.addOutput(self.
root_output_level, name, algorithm, annotations, attributes)
548 retrieves the algorithm han object associated with name
557 retrieves the han reference associated with name
569 Add a DQParameter object to a HanDocument, Doing this it creates the tree of
573 if dqparameter.type !=
"DQParameter":
575 "Object: "+
str(dqparameter)+
" type does not match 'DQParameter'")
577 histoname = dqparameter.getInput().
split(
'/')
580 for directory
in histoname[0:len(histoname)-1]:
581 subdir = theDir.getSubDir(directory)
583 subdir = theDir.addSubDir(directory)
586 algo = dqparameter.getAlgorithm()
588 raise Exception(
'DQParameter without DQAlgorithm')
589 dqalgoname, dqalgolibrary = algo.id, algo.getLibraryName()
591 dqrefname, dqreference, refs =
"", dqparameter.getReference(), []
592 if dqreference != [
None]:
593 refs =
'[%s]' % (
','.
join(_.id
for _
in dqreference))
594 for iref
in dqreference:
597 if handocument.getReference(ref)
is None:
599 refcompletename = iref.getReference()
600 filename = refcompletename.rpartition(
':')[0]
601 refhistoname = refcompletename.rpartition(
':')[
604 ref, histogramname=refhistoname, file=filename,
605 info=iref.getAnnotation(
'info'))
606 handocument.appendChild(hanref)
611 warnings = toList(dqparameter.getWarnings())
612 errors = toList(dqparameter.getErrors())
613 thresholds, hanthreshold, thresholdsname =
'',
None,
''
614 if (warnings != [
None])
and (errors != [
None]):
616 thresholds = dqparameter.id.replace(
617 '_',
'__').
replace(
'/',
'_')+
'_thr'
619 thresholdsname = thresholds
620 thresholds =
'_'+thresholds
622 def getCorrespondingRed(warningname):
624 if e.getAttribute(
'Name')[1] == warningname:
627 for warning
in warnings:
628 warningname = warning.getAttribute(
'Name')[1]
629 error = getCorrespondingRed(warningname)
632 'Cannot find corresponding error for warning names: '+warning.id)
633 hanthreshold.addLimit(warningname, warning.getAttribute(
634 'Value')[1], error.getAttribute(
'Value')[1])
636 handocument.appendChild(hanthreshold)
639 parameterstr =
'_'.
join([x.id
for x
in toList(
640 dqparameter.getAlgorithmParameters())
if x
is not None])
642 for x
in toList(dqparameter.getAlgorithmParameters()):
645 parameters[x.getName()[1]] = x.getValue()[1][0]
647 hanalgoname = (
"%s_%s%s%s%s" % (dqalgolibrary, dqalgoname,
648 dqrefname, thresholds.replace(
'/',
'_'), parameterstr))
650 theDir.addHistogram(histogram=histoname[len(histoname)-1], algorithm=hanalgoname,
651 annotations=dqparameter.getAllAnnotations(), output=output, attributes=dqparameter.getAllAttributes())
653 if handocument.getAlgorithm(hanalgoname)
is None:
655 node =
HanAlgorithm(name=hanalgoname, algoname=dqalgoname, libname=dqalgolibrary,
656 reference=refs, thresholds=thresholdsname, parameters=parameters)
657 handocument.appendChild(node)
659 compalg = algo.getCompositeAlgorithm()
660 if compalg
is not None:
662 libnames=compalg.getLibNames())
663 handocument.appendChild(node)
664 except Exception
as msg:
666 "Object: "+
str(dqparameter)+
" is not a valid DQParameter. Reason: "+
str(msg))
677 This helper return the list of objects that are the roots in the tree represented by input.
682 if o.type == objecttype:
683 if len(o.isReferencedBy) == 0:
687 "object %s is not a valid DQBase object" % o)
698 This returns all DQBase objects reachable from the rootlist as a set.
705 if isinstance(dqbase, list):
707 if not isinstance(dqbase, DQBase):
709 '%s is not a valid DQBase object; this should never happen' % dqbase)
711 for rel
in dqbase.getAllRelatedObjects():
716 for dqbase
in toList(rootlist):
726 Writes the configuration to a han file.
727 The configuration has to be passed as a list of roots in han configuration.
731 for element
in configList:
733 if not DQHanConfMaker._isAceptedHanElement(element):
734 print(
"==========WARNING===========")
737 " does not have a valid han representation, cannot save")
738 print(
"The object is referenced by:", element.isReferencedBy)
739 print(
"You may need to manually edit the configuration")
740 print(
"============================")
741 configList.remove(element)
743 print(element.isReferencedBy)
751 def _addchild(hanfather, dqregion, handocument):
752 outputalgo = dqregion.getAlgorithm()
753 if handocument.getAlgorithm(outputalgo.id)
is None:
758 outputalgo.id, outputalgo.id, outputalgo.getLibraryName()))
759 output = hanfather.addOutput(dqregion.id, outputalgo.id, annotations=dqregion.getAllAnnotations(
760 ), attributes=dqregion.getAllAttributes())
762 for subregion
in dqregion.getSubRegions():
763 if subregion
is not None:
764 _addchild(output, subregion, doc)
766 for parameter
in dqregion.getDQParameters():
767 if parameter
is not None:
769 output=output.getOutputPath())
771 for region
in rootregions:
772 _addchild(doc, region, doc)
774 fileout =
open(filename,
'w')
775 print(doc.toprettyhan(
" "), file=fileout)