python.hanwriter Namespace Reference

Classes
class	Document
class	HanAlgorithm
class	HanCannotCreateConf
class	HanCompositeAlgorithm
class	HanDir
class	HanHistogram
class	HanLimit
class	HanOutput
class	HanReference
class	HanThreshold
class	HanWritingError
class	Node

Functions
	_hanAddDQParameter (handocument, dqparameter, output=DQHanConfMaker._default_output)
	_findRoots (input=[], objecttype="DQRegion")
	_findAllDQBaseObjects (rootlist)
	writeHanConfiguration (filename='dq.han.config', roots=[])

Detailed Description

This module contains tools to write a DQConfiguration in han format

Function Documentation

_findAllDQBaseObjects()

python.hanwriter._findAllDQBaseObjects ( rootlist )

protected

This returns all DQBase objects reachable from the rootlist as a set.

Definition at line 694 of file hanwriter.py.

def _findAllDQBaseObjects(rootlist):
    """
    This returns all DQBase objects reachable from the rootlist as a set.
    """
    import operator
 
    def recurse(dqbase):
        if dqbase is None:
            return set()
        if isinstance(dqbase, list):
            return reduce(operator.ior, [recurse(x) for x in dqbase])
        if not isinstance(dqbase, DQBase):
            raise ValueError(
                '%s is not a valid DQBase object; this should never happen' % dqbase)
        retset = {dqbase}
        for rel in dqbase.getAllRelatedObjects():
            retset |= recurse(rel)
        return retset
 
    topset = set()
    for dqbase in toList(rootlist):
        topset |= recurse(dqbase)
    return list(topset)
 
# Writes the han configuration to a file
# @param roots: the list of all roots of han trees
 
 

_findRoots()

python.hanwriter._findRoots ( input = [], objecttype = "DQRegion" )

protected

This helper return the list of objects that are the roots in the tree represented by input.

Definition at line 673 of file hanwriter.py.

def _findRoots(input=[], objecttype="DQRegion"):
    """
    This helper return the list of objects that are the roots in the tree represented by input.
    """
    result = []
    for o in input:
        try:
            if o.type == objecttype:
                if len(o.isReferencedBy) == 0:
                    result += [o]
        except Exception:
            raise HanCannotCreateConf(
                "object %s is not a valid DQBase object" % o)
    return result
 
# Gets all DQBase objects that corresponds to the input han objects
# @param rootlist: a list of han object trees
# @return: the list of all DQBaseObjects associated to each node of the input tree
# @note: This function is used internally and should not be used by users of this module
 
 

_hanAddDQParameter()

python.hanwriter._hanAddDQParameter ( handocument, dqparameter, output = DQHanConfMaker._default_output )

protected

Add a DQParameter object to a HanDocument, Doing this it creates the tree of 
dir and output.

Definition at line 565 of file hanwriter.py.

def _hanAddDQParameter(handocument, dqparameter, output=DQHanConfMaker._default_output):
    """
    Add a DQParameter object to a HanDocument, Doing this it creates the tree of 
    dir and output.
    """
    try:
        if dqparameter.type != "DQParameter":  # This is not valid
            raise HanCannotCreateConf(
                "Object: "+str(dqparameter)+" type does not match 'DQParameter'")
        # gets the directory structure of the histogram and add it, if needed to the document
        histoname = dqparameter.getInput().split('/')
        theDir = handocument
        # add directories to han document if needed
        for directory in histoname[0:len(histoname)-1]:
            subdir = theDir.getSubDir(directory)
            if subdir is None:
                subdir = theDir.addSubDir(directory)
            theDir = subdir
        # prepare to build the han algorithm element
        algo = dqparameter.getAlgorithm()
        if algo is None:
            raise Exception('DQParameter without DQAlgorithm')
        dqalgoname, dqalgolibrary = algo.id, algo.getLibraryName()
        # Build the han reference, if any, associated to this DQParameter
        dqrefname, dqreference, refs = "", dqparameter.getReference(), []
        if dqreference != [None]:  # References are defined
            refs = '[%s]' % (','.join(_.id for _ in dqreference))
            for iref in dqreference:
                ref = iref.id
                # search if reference is already in the han document
                if handocument.getReference(ref) is None:
                    # it's not found, let's create it
                    refcompletename = iref.getReference()
                    filename = refcompletename.rpartition(':')[0]
                    refhistoname = refcompletename.rpartition(':')[
                        2].lstrip('/')
                    hanref = HanReference(
                        ref, histogramname=refhistoname, file=filename, 
                        info=iref.getAnnotation('info'))
                    handocument.appendChild(hanref)
                    dqrefname = '_'+ref
        # build the han thresholds
        # here it's quite tricky: we have to get two sets of DQThresholds and combine them in a single
        # han thresholds element
        warnings = toList(dqparameter.getWarnings())
        errors = toList(dqparameter.getErrors())
        thresholds, hanthreshold, thresholdsname = '', None, ''
        if (warnings != [None]) and (errors != [None]):
            # Found errors and warnings limits
            thresholds = dqparameter.id.replace(
                '_', '__').replace('/', '_')+'_thr'
            hanthreshold = HanThreshold(thresholds)
            thresholdsname = thresholds
            thresholds = '_'+thresholds
 
            def getCorrespondingRed(warningname):
                for e in errors:
                    if e.getAttribute('Name')[1] == warningname:
                        return e
                return None
            for warning in warnings:
                warningname = warning.getAttribute('Name')[1]
                error = getCorrespondingRed(warningname)
                if error is None:
                    raise HanCannotCreateConf(
                        'Cannot find corresponding error for warning names: '+warning.id)
                hanthreshold.addLimit(warningname, warning.getAttribute(
                    'Value')[1], error.getAttribute('Value')[1])
            #print DQHanConfMaker._get_NodeType(hanthreshold.nodeType),hanthreshold.name
            handocument.appendChild(hanthreshold)
 
        # build the parameter id string and key, value pairs
        parameterstr = '_'.join([x.id for x in toList(
            dqparameter.getAlgorithmParameters()) if x is not None])
        parameters = {}
        for x in toList(dqparameter.getAlgorithmParameters()):
            if x is None:
                continue
            parameters[x.getName()[1]] = x.getValue()[1][0]
        # create a unique string identifying this han algorithm element so far
        hanalgoname = ("%s_%s%s%s%s" % (dqalgolibrary, dqalgoname,
                                        dqrefname, thresholds.replace('/', '_'), parameterstr))
        # Finally add the histogram han element
        theDir.addHistogram(histogram=histoname[len(histoname)-1], algorithm=hanalgoname,
                            annotations=dqparameter.getAllAnnotations(), output=output, attributes=dqparameter.getAllAttributes())
        # search the han algorithm in the document
        if handocument.getAlgorithm(hanalgoname) is None:
            # This algorithm has never been added before in the document
            node = HanAlgorithm(name=hanalgoname, algoname=dqalgoname, libname=dqalgolibrary,
                                reference=refs, thresholds=thresholdsname, parameters=parameters)
            handocument.appendChild(node)
            # check for possible composite algorithms
            compalg = algo.getCompositeAlgorithm()
            if compalg is not None:
                node = HanCompositeAlgorithm(name=compalg.id, subalgnames=compalg.getSubAlgNames(),
                                             libnames=compalg.getLibNames())
                handocument.appendChild(node)
    except Exception as msg:
        raise HanCannotCreateConf(
            "Object: "+str(dqparameter)+" is not a valid DQParameter. Reason: "+str(msg))
 
# Finds the root han objects of the input collection
# @param input: the list of objects @c input: a list of DQBase objects
# @raise HanCannotCreateConf: in case of errors
# @return: the list of roots
# @note: This function is used internally and should not be used by users of this module
 
 

writeHanConfiguration()

python.hanwriter.writeHanConfiguration	(		filename = 'dq.han.config',
			roots = [] )

Writes the configuration to a han file. 
The configuration has to be passed as a list of roots in han configuration.

Definition at line 722 of file hanwriter.py.

def writeHanConfiguration(filename='dq.han.config', roots=[]):
    """
    Writes the configuration to a han file. 
    The configuration has to be passed as a list of roots in han configuration.
    """
    configList = _findAllDQBaseObjects(roots)
    # First remove the configuration elements that cannot be saved as han
    for element in configList:
        try:
            if not DQHanConfMaker._isAceptedHanElement(element):
                print("==========WARNING===========")
                print("Object: " + \
                    str(element) + \
                    " does not have a valid han representation, cannot save")
                print("The object is referenced by:", element.isReferencedBy)
                print("You may need to manually edit the configuration")
                print("============================")
                configList.remove(element)
        except Exception:
            print(element.isReferencedBy)
            raise HanCannotCreateConf("Not valid object:" + str(element))
    # Let's create the han document
    doc = Document()
    # Find all root regions
    rootregions = _findRoots(configList)
 
    # Navigate through DQRegions tree and add it to the han document
    def _addchild(hanfather, dqregion, handocument):
        outputalgo = dqregion.getAlgorithm()
        if handocument.getAlgorithm(outputalgo.id) is None:
            # Add to the document the summary maker algorithm
            # summary makers do not have thresholds or parameters
            # @todo: in the future extend this to make in enough general to support parameter
            handocument.appendChild(HanAlgorithm(
                outputalgo.id, outputalgo.id, outputalgo.getLibraryName()))  # , reference, thresholds)
        output = hanfather.addOutput(dqregion.id, outputalgo.id, annotations=dqregion.getAllAnnotations(
        ), attributes=dqregion.getAllAttributes())
        # recursively add subregions
        for subregion in dqregion.getSubRegions():
            if subregion is not None:
                _addchild(output, subregion, doc)
        # Add each DQParameter to the han document with the helper
        for parameter in dqregion.getDQParameters():
            if parameter is not None:
                _hanAddDQParameter(handocument, parameter,
                                   output=output.getOutputPath())
 
    for region in rootregions:
        _addchild(doc, region, doc)
 
    fileout = open(filename, 'w')
    print(doc.toprettyhan(" "), file=fileout)