python.TriggerConfigAccessBase.ConfigDBLoader Class Reference

Inheritance diagram for python.TriggerConfigAccessBase.ConfigDBLoader:

Collaboration diagram for python.TriggerConfigAccessBase.ConfigDBLoader:

Public Member Functions
def	__init__ (self, configType, dbalias, dbkey)
def	setQuery (self, query)
def	getQueryDefinition (self, schemaVersion)
def	load (self)
def	getWriteFilename (self)
def	confirmConfigType (self, config)

Static Public Member Functions
def	getResolvedFileName (filename, pathenv="")
def	getConnectionParameters (dbalias)
def	getSchema (connStr)
def	readSchemaVersion (qdict, session)
def	getCoralQuery (session, queryStr, qdict=None)

Public Attributes
	dbalias
	dbkey
	query
	schema
	configType

Detailed Description

Definition at line 89 of file TriggerConfigAccessBase.py.

Constructor & Destructor Documentation

__init__()

def python.TriggerConfigAccessBase.ConfigDBLoader.__init__	(		self,
			configType,
			dbalias,
			dbkey
	)

Definition at line 90 of file TriggerConfigAccessBase.py.

    def __init__(self, configType, dbalias, dbkey):
        super().__init__(configType)
        self.dbalias = dbalias
        self.dbkey = dbkey
        self.query = None
        self.schema = None
 

Member Function Documentation

confirmConfigType()

def python.TriggerConfigAccessBase.ConfigLoader.confirmConfigType (   self,   config  )

inherited

checks that the in-file specification of the configuration type matches the expected type

Definition at line 51 of file TriggerConfigAccessBase.py.

    def confirmConfigType(self,config):
        """
        checks that the in-file specification of the configuration type matches the expected type
        """
        if config['filetype'] != self.configType:
            raise RuntimeError("Can not load file with filetype '%s' when expecting '%s'" % (config['filetype'], self.configType.filetype))
 

getConnectionParameters()

def python.TriggerConfigAccessBase.ConfigDBLoader.getConnectionParameters (   dbalias )

static

Definition at line 117 of file TriggerConfigAccessBase.py.

    def getConnectionParameters(dbalias):
        dblookupFile = ConfigDBLoader.getResolvedFileName("dblookup.xml", "CORAL_DBLOOKUP_PATH")
        dbp = ET.parse(dblookupFile)
        listOfServices = None
        for logSvc in dbp.iter("logicalservice"):
            if logSvc.attrib["name"] != dbalias:
                continue
            listOfServices = [ serv.attrib["name"] for serv in logSvc.iter("service") ]
            if len(listOfServices) == 0:
                raise RuntimeError("DB %s has no services listed in %s" % (dbalias, dblookupFile))
            break
        if listOfServices is None:
            raise RuntimeError("DB %s not available in %s" % (dbalias, dblookupFile))
        
        if "FRONTIER_SERVER" not in os.environ:
            # remove all frontier connnections in the list if the environment FRONTIER_SERVER variable does not exist
            # this speeds up the resolution of the connection specification (dbalias)
            listOfServices = [svc for svc in listOfServices if not svc.startswith("frontier:")]
 
        # now get the account and pw for oracle connections
        credentials = odict().fromkeys(listOfServices)
 
        for svc in filter(lambda s : s.startswith("frontier:"), listOfServices):
            credentials[svc] = dict()
            credentials[svc]["user"] = svc
            credentials[svc]["password"] = ""
 
        try:
            authFile = ConfigDBLoader.getResolvedFileName("authentication.xml", "CORAL_AUTH_PATH")
        except Exception as e:
            log.warning("File authentication.xml is not available! Oracle connection cannot be established. Exception message is: %s",e)
        else:
            for svc in filter(lambda s : s.startswith("oracle:"), listOfServices):
                ap = ET.parse(authFile)
                count = 0
                for con in filter( lambda c: c.attrib["name"]==svc, ap.iter("connection")):
                    credentials[svc] = dict([(par.attrib["name"],par.attrib["value"]) for par in con])
                    count += 1
                if count==0:
                    raise RuntimeError("No credentials found for connection %s from service %s for db %s" % (con,svc,dbalias))
                if count>1:
                    raise RuntimeError("More than 1 connection found in %s for service %s" % (authFile, svc))
        
        return credentials
 

getCoralQuery()

def python.TriggerConfigAccessBase.ConfigDBLoader.getCoralQuery (   session,   queryStr,   qdict = None  )

static

Parse output, tables and condition from the query string into coral query object

Definition at line 208 of file TriggerConfigAccessBase.py.

    def getCoralQuery(session, queryStr, qdict = None):
        ''' Parse output, tables and condition from the query string into coral query object'''
        query = session.nominalSchema().newQuery()
 
        if qdict is not None:
            queryStr = queryStr.format(**qdict)
 
        # bind vars
        bindVars = coral.AttributeList()
        bindVarsInQuery = re.findall(r":(\w*)", queryStr)
        if len(bindVarsInQuery) > 0 and qdict is None:
            log.error("Query has bound-variable syntax but no value dictionary is provided. Query: %s", queryStr)
        for k in bindVarsInQuery:
            bindVars.extend(k, "int")
            bindVars[k].setData(qdict[k])
 
        output = queryStr.split("SELECT")[1].split("FROM")[0]
        for field in output.split(','):
            query.addToOutputList(field)
 
        log.debug("Conversion for Coral of query: %s", queryStr)
 
        for table in queryStr.split("FROM")[1].split("WHERE")[0].split(","):
            tableSplit = list(filter(None, table.split(" ")))
            # Schema name is stripped from TableList in Coral query
            query.addToTableList(tableSplit[0].split(".")[1], tableSplit[1])
 
        if "WHERE" in queryStr:
            cond = queryStr.split("WHERE")[1]
            m = re.match("(.*)(?i: ORDER *BY )(.*)", cond) # check for "order by" clause
            if m:
                where, order = m.groups()
                query.setCondition(where, bindVars)
                query.addToOrderList(order)
            else:
                query.setCondition(cond, bindVars)
 
        return query
 

getQueryDefinition()

def python.TriggerConfigAccessBase.ConfigDBLoader.getQueryDefinition	(		self,
			schemaVersion
	)

Choose query based on schema version, based on TrigConf::TrigDBLoader::getQueryDefinition 

Definition at line 247 of file TriggerConfigAccessBase.py.

    def getQueryDefinition(self, schemaVersion):
        '''Choose query based on schema version, based on TrigConf::TrigDBLoader::getQueryDefinition '''
        maxDefVersion = 0
        for vkey in self.query.keys():
            if vkey>maxDefVersion and vkey<=schemaVersion:
                maxDefVersion = vkey
 
        if maxDefVersion == 0:
            raise RuntimeError("No query available for schema version {0}".format(schemaVersion))
 
        return self.query[maxDefVersion]
 

getResolvedFileName()

def python.TriggerConfigAccessBase.ConfigDBLoader.getResolvedFileName (   filename,   pathenv = ""  )

static

looks for file, first absolute, then by resolving envvar pathenv

Definition at line 105 of file TriggerConfigAccessBase.py.

    def getResolvedFileName(filename, pathenv=""):
        """ looks for file, first absolute, then by resolving envvar pathenv"""
        if os.access(filename,os.R_OK):
            return filename
        pathlist = os.getenv(pathenv,'').split(os.pathsep)
        for path in pathlist:
            f = os.path.join( path, filename )
            if os.access( f, os.R_OK ):
                return f
        raise RuntimeError("Can't read file %s, neither locally nor in %s" % (filename, pathenv) )
 

getSchema()

def python.TriggerConfigAccessBase.ConfigDBLoader.getSchema (   connStr )

static

Read schema from connection string 

Definition at line 163 of file TriggerConfigAccessBase.py.

    def getSchema(connStr):
        ''' Read schema from connection string '''
        if connStr.startswith("oracle:"):
            [_, schema] = connStr.split("/")[-2:]
            return schema
 
        if connStr.startswith("frontier:"):
            import re
            pattern = r"frontier://ATLF/\‍(\‍)/(.*)"
            m = re.match(pattern, connStr)
            if not m:
                raise RuntimeError("connection string '%s' doesn't match the pattern '%s'?" % (connStr, pattern))
            (schema, ) = m.groups()
            return schema
 
        if connStr.startswith("sqlite_file:"):
            raise NotImplementedError("Python-loading of trigger configuration from sqlite has not yet been implemented")
 

getWriteFilename()

def python.TriggerConfigAccessBase.ConfigDBLoader.getWriteFilename

(

self

)

Definition at line 339 of file TriggerConfigAccessBase.py.

    def getWriteFilename(self):
        return "{basename}_{schema}_{dbkey}.json".format(basename = self.configType.basename, schema = self.schema, dbkey = self.dbkey)
 

load()

def python.TriggerConfigAccessBase.ConfigDBLoader.load

(

self

)

Definition at line 259 of file TriggerConfigAccessBase.py.

    def load(self):
        credentials: dict[str,Any] = ConfigDBLoader.getConnectionParameters(self.dbalias)
 
        if not credentials:
            log.error("No TriggerDB connections found for %s", self.dbalias)
            raise RuntimeError(f"No TriggerDB connections found for {self.dbalias}")
 
        svc = coral.ConnectionService() 
        svcconfig = svc.configuration()
        svcconfig.disablePoolAutomaticCleanUp()
        svcconfig.setConnectionTimeOut(0)
 
        failureMode = 0
        for credential in credentials:
            log.debug("Trying credentials %s",credential)
 
            try: 
                session = svc.connect(credential, coral.access_ReadOnly)
            except Exception as e:
                log.warning("Failed to establish connection: %s",e)
                failureMode = max(1, failureMode)
                continue
 
            # Check that the FRONTIER_SERVER is set properly, if not reduce the retrial period and time out values
            if 'FRONTIER_SERVER' in os.environ and os.environ['FRONTIER_SERVER']:
                svcconfig.setConnectionRetrialPeriod(300)
                svcconfig.setConnectionRetrialTimeOut(3600)
            else:
                svcconfig.setConnectionRetrialPeriod(1)
                svcconfig.setConnectionRetrialTimeOut(1)
 
            try:
                session.transaction().start(True) # readOnly
                self.schema = ConfigDBLoader.getSchema(credential)
                qdict = { "schema" : self.schema, "dbkey" : self.dbkey }
                
                # Choose query based on schema
                schemaVersion = ConfigDBLoader.readSchemaVersion(qdict, session)
                qstr = self.getQueryDefinition(schemaVersion)
                # execute coral query
                query = ConfigDBLoader.getCoralQuery(session, qstr, qdict)
                cursor = query.execute()
 
            except Exception as e:
                log.warning(f"DB query on {credential} failed to execute.")
                log.warning("Exception message: %r", e)
                failureMode = max(2, failureMode)
                continue # to next source
 
            # Read query result
            if not cursor.next():
                # empty result
                log.warning(f"DB query on {credential} returned empty result, likely due to non-existing key {self.dbkey}")
                failureMode = 3
                continue # to next source
 
            configblob = cursor.currentRow()[0].data()
            if type(configblob) is not str:
                configblob = configblob.readline()
            config = json.loads(configblob, object_pairs_hook = odict)
            session.transaction().commit()
            
            self.confirmConfigType(config)
            return config
 
        if failureMode == 1:
            log.error("TriggerDB query: could not connect to any source for %s", self.configType.basename)
            log.error("Considered sources: %s", ", ".join(credentials))
            raise RuntimeError("TriggerDB query: could not connect to any source", self.configType.basename)
        if failureMode == 2:
            log.error("Query failed due to wrong definition for %s", self.configType.basename)
            log.error("DB query was: %s", qstr.format(**qdict))
            raise RuntimeError("Query failed due to wrong definition", self.configType.basename)
        elif failureMode == 3:
            log.error("DB key %s does not exist for %s", self.dbkey, self.configType.basename)
            raise KeyError("DB key does not exist", self.dbkey, self.configType.basename)
        else:
            raise RuntimeError("Query failed for unknown reason")
 

readSchemaVersion()

def python.TriggerConfigAccessBase.ConfigDBLoader.readSchemaVersion (   qdict,   session  )

static

Read schema version form database, based on TrigConf::TrigDBLoader::schemaVersion 

Definition at line 182 of file TriggerConfigAccessBase.py.

    def readSchemaVersion(qdict, session):
        ''' Read schema version form database, based on TrigConf::TrigDBLoader::schemaVersion '''
        try:
            q = "SELECT TS_TAG FROM {schema}.TRIGGER_SCHEMA TS"
            query = ConfigDBLoader.getCoralQuery(session, q.format(**qdict))
            cursor = query.execute()
            cursor.next()
 
            versionTag = cursor.currentRow()[0].data()
 
            versionTagPrefix = "Trigger-Run3-Schema-v"
            if not versionTag.startswith(versionTagPrefix):
                raise RuntimeError( "Tag format error: Trigger schema version tag %s does not start with %s", versionTag, versionTagPrefix) 
 
            vstr = versionTag[len(versionTagPrefix)]
 
            if not vstr.isdigit():
                raise RuntimeError( "Invalid argument when interpreting the version part %s of schema tag %s is %s", vstr, versionTag, type(vstr))
 
            log.debug("Found schema version %s", vstr)
            return int(vstr)
 
        except Exception as e:
            log.warning("Failed to read schema version: %r", e)
 

setQuery()

def python.TriggerConfigAccessBase.ConfigDBLoader.setQuery	(		self,
			query
	)

query template is a dictionary of queries, identified by schema version, 
similar to TrigConf::TrigDBMenuLoader::m_hltQueries and TrigConf::TrigDBMenuLoader::m_l1Queries

Definition at line 97 of file TriggerConfigAccessBase.py.

    def setQuery(self, query):
        """
        query template is a dictionary of queries, identified by schema version, 
        similar to TrigConf::TrigDBMenuLoader::m_hltQueries and TrigConf::TrigDBMenuLoader::m_l1Queries
        """
        self.query = query
 

Member Data Documentation

configType

python.TriggerConfigAccessBase.ConfigLoader.configType

inherited

Definition at line 50 of file TriggerConfigAccessBase.py.

dbalias

python.TriggerConfigAccessBase.ConfigDBLoader.dbalias

Definition at line 92 of file TriggerConfigAccessBase.py.

dbkey

python.TriggerConfigAccessBase.ConfigDBLoader.dbkey

Definition at line 93 of file TriggerConfigAccessBase.py.

query

python.TriggerConfigAccessBase.ConfigDBLoader.query

Definition at line 94 of file TriggerConfigAccessBase.py.

schema

python.TriggerConfigAccessBase.ConfigDBLoader.schema

Definition at line 95 of file TriggerConfigAccessBase.py.

---

The documentation for this class was generated from the following file:

• TriggerConfigAccessBase.py