10 This module contains tools to write a DQConfiguration in han format
14 from DQConfMakerBase.Helpers
import BaseException, toList
15 from DQConfMakerBase.DQElements
import DQBase
16 from functools
import reduce
24 This exception is thrown when an error writing the han configuration is encountered
28 BaseException.__init__(
29 self,
"Cannot create the han file ("+
str(reason)+
")")
37 This exception is thrown when an error creating the han configuration is encountered
41 BaseException.__init__(
42 self,
"Cannot create configuration ("+
str(reason)+
")")
48 class Node(DQHanConfMaker.Node):
50 This is the base class for each element that can be written in a valid han configuration file.
52 - subnodes : a list of Node instances
53 - attributes : a list of attributes associated with the node
54 - acceptChild : a list of Node types that can be accepted as sub nodes
55 - nodeType : the type of the node
56 - name : the name of the node
67 self.
acceptChild = [Node.ALGORITHM, Node.DIR, Node.DOCUMENT, Node.HIST, Node.OUTPUT,
68 Node.REFERENCE, Node.THRESHOLD, Node.LIMIT, Node.COMPALG]
77 convert the object in a valid han script block
88 convert the object in a formatted han string
94 writer = io.BytesIO()
if six.PY2
else io.StringIO()
95 if encoding
is not None:
97 writer = codecs.lookup(encoding)[3](writer)
98 self.
writehan(writer,
"", indent, newl)
99 return writer.getvalue()
106 def writehan(self, writer, indent="", addindent="", newl=""):
108 Converts the object in a han string and writes it in the writer object
111 indent+DQHanConfMaker._get_NodeType(self.
nodeType)+
" "+self.
name)
114 writer.write(
" { %s" % (newl))
117 writer.write(
"%s %s = %s%s" % (indent, key, attribute, newl))
120 node.writehan(writer, indent+addindent, addindent, newl)
122 writer.write(
"%s}%s" % (indent, newl))
129 Add a sub node to this node
133 msg =
" Node: "+DQHanConfMaker._get_NodeType(self.
nodeType)
134 msg +=
" Cannot have a child of type:", child.nodeType
138 "Object:"+
str(child)+
" is not a valid Node")
148 The attribute identified by key is added to this node
159 Removes attribute identified by key
171 Gets the attribute identified by the key, None if not found
183 Returns the sub-node identified by name and nodetype
187 if sn.nodeType == nodetype
and sn.name == name:
199 A han histogram element is a Node with attributes:
201 - the algorithm to be used
202 - the output where to store the dq result
212 def __init__(self, histogram, algorithm, annotations={}, output=DQHanConfMaker._default_output, attributes={}):
214 Creates a han histogram configuration element
216 Node.__init__(self, histogram)
221 if 'Weight' in attributes:
223 for annotation
in annotations:
230 Retrieves the output attribute
238 Retrieves the algorithm attribute
248 The han representation of a DQAlgorithm
258 def __init__(self, name, algoname, libname='libdqm_algorithms.so', reference=None, thresholds=None,
261 Creates a han algorithm configuration element
263 Node.__init__(self, name)
273 for parameter
in parameters:
282 The han representation of a CompositeAlgorithm
289 def __init__(self, name, subalgnames=[], libnames=[]):
291 Creates a han composite algorithm configuration element
293 Node.__init__(self, name)
305 The han representation of a DQReference
312 def __init__(self, name, histogramname, file, info=None):
314 Creates a han reference configuration element
316 Node.__init__(self, name)
330 The han representation of a limit for a HanThreshold
339 Creates a han limit object
341 Node.__init__(self, name)
353 The han representation of a DQThreshold
360 Creates a han threshold configuration element
362 Node.__init__(self, name)
368 Adds a limit to the threshold
369 @param name: the limits name
370 @param warning: the limit value for the warning level
371 @param error: the limit value for the error level
375 "The limit: "+name+
" already exists for threshold: "+self.
name)
376 limit =
HanLimit(name, warning, error)
385 Class representing a dir element
386 A han dir element is a Node with sub nodes of type HanDir or HanHisto
393 Creates a han directory element
395 Node.__init__(self, name)
404 Adds a subdirectory called name to the current directory and returns the newly created sub directory
417 def addHistogram(self, histogram, algorithm, annotations={}, output=DQHanConfMaker._default_output, attributes={}):
419 Adds to the directory a histogram specifying the histogram name, the algorithm and the output path.
422 annotations, output, attributes)
430 returns the sub-directory called name
440 Class representing a output element
441 A han output element is a Node with sub nodes of type output and with the optional attribute algorithm
443 - father: the HanOutput that contains this HanOutput
452 def __init__(self, name, algorithm=None, father=None, annotations={}, attributes={}):
454 Creates an output han element
456 Node.__init__(self, name)
462 if 'Weight' in attributes:
464 for annotation
in annotations:
471 Gets the algorithm attribute
482 def addOutput(self, name, algorithm, annotations, attributes):
484 Adds a sub output element with name and algorithm
486 subnode =
HanOutput(name, algorithm, self, annotations, attributes)
496 Creates the complete path of this output directory
512 Class representing a han document
517 def __init__(self, top_level_algorithm='WorstCaseSummary'):
519 Creates an empty han document with a top level output
521 HanDir.__init__(self,
'')
522 HanOutput.__init__(self,
'')
524 self.
acceptChild = [Node.ALGORITHM, Node.DIR, Node.HIST, Node.OUTPUT, Node.REFERENCE, Node.THRESHOLD,
535 def addOutput(self, name, algorithm, annotations={}, attributes={}):
537 Adds a sub output element with name and algorithm
540 return HanOutput.addOutput(self.
root_output_level, name, algorithm, annotations, attributes)
547 retrieves the algorithm han object associated with name
556 retrieves the han reference associated with name
568 Add a DQParameter object to a HanDocument, Doing this it creates the tree of
572 if dqparameter.type !=
"DQParameter":
574 "Object: "+
str(dqparameter)+
" type does not match 'DQParameter'")
576 histoname = dqparameter.getInput().
split(
'/')
579 for directory
in histoname[0:len(histoname)-1]:
580 subdir = theDir.getSubDir(directory)
582 subdir = theDir.addSubDir(directory)
585 algo = dqparameter.getAlgorithm()
587 raise Exception(
'DQParameter without DQAlgorithm')
588 dqalgoname, dqalgolibrary = algo.id, algo.getLibraryName()
590 dqrefname, dqreference, refs =
"", dqparameter.getReference(), []
591 if dqreference != [
None]:
592 refs =
'[%s]' % (
','.
join(_.id
for _
in dqreference))
593 for iref
in dqreference:
596 if handocument.getReference(ref)
is None:
598 refcompletename = iref.getReference()
599 filename = refcompletename.rpartition(
':')[0]
600 refhistoname = refcompletename.rpartition(
':')[
603 ref, histogramname=refhistoname, file=filename,
604 info=iref.getAnnotation(
'info'))
605 handocument.appendChild(hanref)
610 warnings = toList(dqparameter.getWarnings())
611 errors = toList(dqparameter.getErrors())
612 thresholds, hanthreshold, thresholdsname =
'',
None,
''
613 if (warnings != [
None])
and (errors != [
None]):
615 thresholds = dqparameter.id.replace(
616 '_',
'__').
replace(
'/',
'_')+
'_thr'
618 thresholdsname = thresholds
619 thresholds =
'_'+thresholds
621 def getCorrespondingRed(warningname):
623 if e.getAttribute(
'Name')[1] == warningname:
626 for warning
in warnings:
627 warningname = warning.getAttribute(
'Name')[1]
628 error = getCorrespondingRed(warningname)
631 'Cannot find corresponding error for warning names: '+warning.id)
632 hanthreshold.addLimit(warningname, warning.getAttribute(
633 'Value')[1], error.getAttribute(
'Value')[1])
635 handocument.appendChild(hanthreshold)
638 parameterstr =
'_'.
join([x.id
for x
in toList(
639 dqparameter.getAlgorithmParameters())
if x
is not None])
641 for x
in toList(dqparameter.getAlgorithmParameters()):
644 parameters[x.getName()[1]] = x.getValue()[1][0]
646 hanalgoname = (
"%s_%s%s%s%s" % (dqalgolibrary, dqalgoname,
647 dqrefname, thresholds.replace(
'/',
'_'), parameterstr))
649 theDir.addHistogram(histogram=histoname[len(histoname)-1], algorithm=hanalgoname,
650 annotations=dqparameter.getAllAnnotations(), output=output, attributes=dqparameter.getAllAttributes())
652 if handocument.getAlgorithm(hanalgoname)
is None:
654 node =
HanAlgorithm(name=hanalgoname, algoname=dqalgoname, libname=dqalgolibrary,
655 reference=refs, thresholds=thresholdsname, parameters=parameters)
656 handocument.appendChild(node)
658 compalg = algo.getCompositeAlgorithm()
659 if compalg
is not None:
661 libnames=compalg.getLibNames())
662 handocument.appendChild(node)
663 except Exception
as msg:
665 "Object: "+
str(dqparameter)+
" is not a valid DQParameter. Reason: "+
str(msg))
676 This helper return the list of objects that are the roots in the tree represented by input.
681 if o.type == objecttype:
682 if len(o.isReferencedBy) == 0:
686 "object %s is not a valid DQBase object" % o)
697 This returns all DQBase objects reachable from the rootlist as a set.
704 if isinstance(dqbase, list):
706 if not isinstance(dqbase, DQBase):
708 '%s is not a valid DQBase object; this should never happen' % dqbase)
710 for rel
in dqbase.getAllRelatedObjects():
715 for dqbase
in toList(rootlist):
725 Writes the configuration to a han file.
726 The configuration has to be passed as a list of roots in han configuration.
730 for element
in configList:
732 if not DQHanConfMaker._isAceptedHanElement(element):
733 print(
"==========WARNING===========")
736 " does not have a valid han representation, cannot save")
737 print(
"The object is referenced by:", element.isReferencedBy)
738 print(
"You may need to manually edit the configuration")
739 print(
"============================")
740 configList.remove(element)
742 print(element.isReferencedBy)
750 def _addchild(hanfather, dqregion, handocument):
751 outputalgo = dqregion.getAlgorithm()
752 if handocument.getAlgorithm(outputalgo.id)
is None:
757 outputalgo.id, outputalgo.id, outputalgo.getLibraryName()))
758 output = hanfather.addOutput(dqregion.id, outputalgo.id, annotations=dqregion.getAllAnnotations(
759 ), attributes=dqregion.getAllAttributes())
761 for subregion
in dqregion.getSubRegions():
762 if subregion
is not None:
763 _addchild(output, subregion, doc)
765 for parameter
in dqregion.getDQParameters():
766 if parameter
is not None:
768 output=output.getOutputPath())
770 for region
in rootregions:
771 _addchild(doc, region, doc)
773 fileout =
open(filename,
'w')
774 print(doc.toprettyhan(
" "), file=fileout)