python.TriggerConfigAccessBase.ConfigDBLoader Class Reference

Inheritance diagram for python.TriggerConfigAccessBase.ConfigDBLoader:

Collaboration diagram for python.TriggerConfigAccessBase.ConfigDBLoader:

Public Member Functions
def	__init__ (self, configType, dbalias, dbkey)
def	setQuery (self, query)
def	getQueryDefinition (self, schemaVersion)
dict[str, Any]	load (self)
str	getWriteFilename (self)
def	confirmConfigType (self, config)

Static Public Member Functions
def	getResolvedFileName (filename, pathenv="")
def	getConnectionParameters (dbalias)
def	getSchema (connStr)
def	readSchemaVersion (qdict, session)
def	getCoralQuery (session, queryStr, qdict=None)

Public Attributes
	dbalias
	dbkey
	query
	schema

Detailed Description

Definition at line 101 of file TriggerConfigAccessBase.py.

Constructor & Destructor Documentation

__init__()

def python.TriggerConfigAccessBase.ConfigDBLoader.__init__	(		self,
			configType,
			dbalias,
			dbkey
	)

Definition at line 102 of file TriggerConfigAccessBase.py.

    def __init__(self, configType, dbalias, dbkey):
        super().__init__(configType)
        self.dbalias = dbalias
        self.dbkey = dbkey
        self.query = {}
        self.schema = None
 

Member Function Documentation

confirmConfigType()

def python.TriggerConfigAccessBase.ConfigLoader.confirmConfigType (   self,   config  )

inherited

checks that the in-file specification of the configuration type matches the expected type

Definition at line 51 of file TriggerConfigAccessBase.py.

    def confirmConfigType(self,config):
        """
        checks that the in-file specification of the configuration type matches the expected type
        """
        if config['filetype'] != self.configType:
            raise RuntimeError("Can not load file with filetype '%s' when expecting '%s'" % (config['filetype'], self.configType.filetype))
 

getConnectionParameters()

def python.TriggerConfigAccessBase.ConfigDBLoader.getConnectionParameters (   dbalias )

static

Definition at line 129 of file TriggerConfigAccessBase.py.

    def getConnectionParameters(dbalias):
        dblookupFile = ConfigDBLoader.getResolvedFileName("dblookup.xml", "CORAL_DBLOOKUP_PATH")
        dbp = ET.parse(dblookupFile)
        listOfServices = []
        foundAlias = False
        for logSvc in dbp.iter("logicalservice"):
            if logSvc.attrib["name"] != dbalias:
                continue
            foundAlias = True
            listOfServices = [ serv.attrib["name"] for serv in logSvc.iter("service") ]
            if len(listOfServices) == 0:
                raise RuntimeError("DB %s has no services listed in %s" % (dbalias, dblookupFile))
            break
        if not foundAlias:
            raise RuntimeError("DB %s not available in %s" % (dbalias, dblookupFile))
        
        if "FRONTIER_SERVER" not in os.environ:
            # remove all frontier connnections in the list if the environment FRONTIER_SERVER variable does not exist
            # this speeds up the resolution of the connection specification (dbalias)
            listOfServices: list[str] = [svc for svc in listOfServices if not svc.startswith("frontier:")]
 
        # now get the account and pw for oracle connections
        credentials: dict[str, Any] = dict.fromkeys(listOfServices)
 
        for svc in filter(lambda s : s.startswith("frontier:"), listOfServices):
            credentials[svc] = {}
            credentials[svc]["user"] = svc
            credentials[svc]["password"] = ""
 
        try:
            authFile = ConfigDBLoader.getResolvedFileName("authentication.xml", "CORAL_AUTH_PATH")
        except Exception as e:
            log.warning("File authentication.xml is not available! Oracle connection cannot be established. Exception message is: %s",e)
        else:
            for svc in filter(lambda s : s.startswith("oracle:"), listOfServices):
                ap = ET.parse(authFile)
                count = 0
                for con in filter( lambda c: c.attrib["name"]==svc, ap.iter("connection")):
                    credentials[svc] = dict([(par.attrib["name"],par.attrib["value"]) for par in con])
                    count += 1
                if count==0:
                    raise RuntimeError("No credentials found for connection %s from service %s for db %s" % (con,svc,dbalias))
                if count>1:
                    raise RuntimeError("More than 1 connection found in %s for service %s" % (authFile, svc))
        
        return credentials
 

getCoralQuery()

def python.TriggerConfigAccessBase.ConfigDBLoader.getCoralQuery (   session,   queryStr,   qdict = None  )

static

Parse output, tables and condition from the query string into coral query object

Definition at line 222 of file TriggerConfigAccessBase.py.

    def getCoralQuery(session, queryStr, qdict = None):
        ''' Parse output, tables and condition from the query string into coral query object'''
        query = session.nominalSchema().newQuery()
 
        if qdict is not None:
            queryStr = queryStr.format(**qdict)
 
        # bind vars
        bindVars = coral.AttributeList()  # type: ignore
        bindVarsInQuery = re.findall(r":(\w*)", queryStr)
        if len(bindVarsInQuery) > 0 and qdict is None:
            log.error("Query has bound-variable syntax but no value dictionary is provided. Query: %s", queryStr)
        for k in bindVarsInQuery:
            bindVars.extend(k, "int")
            bindVars[k].setData(qdict[k])
 
        output = queryStr.split("SELECT")[1].split("FROM")[0]
        for field in output.split(','):
            query.addToOutputList(field)
 
        log.debug("Conversion for Coral of query: %s", queryStr)
 
        for table in queryStr.split("FROM")[1].split("WHERE")[0].split(","):
            tableSplit = list(filter(None, table.split(" ")))
            # Schema name is stripped from TableList in Coral query
            query.addToTableList(tableSplit[0].split(".")[1], tableSplit[1])
 
        if "WHERE" in queryStr:
            cond = queryStr.split("WHERE")[1]
            m = re.match("(.*)(?i: ORDER *BY )(.*)", cond) # check for "order by" clause
            if m:
                where, order = m.groups()
                query.setCondition(where, bindVars)
                query.addToOrderList(order)
            else:
                query.setCondition(cond, bindVars)
 
        return query
 

getQueryDefinition()

def python.TriggerConfigAccessBase.ConfigDBLoader.getQueryDefinition	(		self,
			schemaVersion
	)

Choose query based on schema version, based on TrigConf::TrigDBLoader::getQueryDefinition 

Definition at line 261 of file TriggerConfigAccessBase.py.

    def getQueryDefinition(self, schemaVersion):
        '''Choose query based on schema version, based on TrigConf::TrigDBLoader::getQueryDefinition '''
        maxDefVersion = 0
        for vkey in self.query.keys():
            if vkey>maxDefVersion and vkey<=schemaVersion:
                maxDefVersion = vkey
 
        if maxDefVersion == 0:
            raise RuntimeError("No query available for schema version {0}".format(schemaVersion))
 
        return self.query[maxDefVersion]
 

getResolvedFileName()

def python.TriggerConfigAccessBase.ConfigDBLoader.getResolvedFileName (   filename,   pathenv = ""  )

static

looks for file, first absolute, then by resolving envvar pathenv

Definition at line 117 of file TriggerConfigAccessBase.py.

    def getResolvedFileName(filename, pathenv=""):
        """ looks for file, first absolute, then by resolving envvar pathenv"""
        if os.access(filename,os.R_OK):
            return filename
        pathlist = os.getenv(pathenv,'').split(os.pathsep)
        for path in pathlist:
            f = os.path.join( path, filename )
            if os.access( f, os.R_OK ):
                return f
        raise RuntimeError("Can't read file %s, neither locally nor in %s" % (filename, pathenv) )
 

getSchema()

def python.TriggerConfigAccessBase.ConfigDBLoader.getSchema (   connStr )

static

Read schema from connection string 

Definition at line 177 of file TriggerConfigAccessBase.py.

    def getSchema(connStr):
        ''' Read schema from connection string '''
        if connStr.startswith("oracle:"):
            [_, schema] = connStr.split("/")[-2:]
            return schema
 
        if connStr.startswith("frontier:"):
            import re
            pattern = r"frontier://ATLF/\‍(\‍)/(.*)"
            m = re.match(pattern, connStr)
            if not m:
                raise RuntimeError("connection string '%s' doesn't match the pattern '%s'?" % (connStr, pattern))
            (schema, ) = m.groups()
            return schema
 
        if connStr.startswith("sqlite_file:"):
            raise NotImplementedError("Python-loading of trigger configuration from sqlite has not yet been implemented")
 

getWriteFilename()

str python.TriggerConfigAccessBase.ConfigDBLoader.getWriteFilename

(

self

)

Reimplemented from python.TriggerConfigAccessBase.ConfigLoader.

Definition at line 353 of file TriggerConfigAccessBase.py.

    def getWriteFilename(self) -> str:
        return "{basename}_{schema}_{dbkey}.json".format(basename = self.configType.basename, schema = self.schema, dbkey = self.dbkey)
 

load()

dict[str, Any] python.TriggerConfigAccessBase.ConfigDBLoader.load

(

self

)

Reimplemented from python.TriggerConfigAccessBase.ConfigLoader.

Definition at line 273 of file TriggerConfigAccessBase.py.

    def load(self) -> dict[str, Any]:
        credentials: dict[str,Any] = ConfigDBLoader.getConnectionParameters(self.dbalias)
 
        if not credentials:
            log.error("No TriggerDB connections found for %s", self.dbalias)
            raise RuntimeError(f"No TriggerDB connections found for {self.dbalias}")
 
        svc = coral.ConnectionService() # type: ignore
        svcconfig = svc.configuration()
        svcconfig.disablePoolAutomaticCleanUp()
        svcconfig.setConnectionTimeOut(0)
 
        failureMode = 0
        for credential in credentials:
            log.debug("Trying credentials %s",credential)
 
            try: 
                session = svc.connect(credential, coral.access_ReadOnly) # type: ignore
            except Exception as e:
                log.warning("Failed to establish connection: %s",e)
                failureMode = max(1, failureMode)
                continue
 
            # Check that the FRONTIER_SERVER is set properly, if not reduce the retrial period and time out values
            if 'FRONTIER_SERVER' in os.environ and os.environ['FRONTIER_SERVER']:
                svcconfig.setConnectionRetrialPeriod(300)
                svcconfig.setConnectionRetrialTimeOut(3600)
            else:
                svcconfig.setConnectionRetrialPeriod(1)
                svcconfig.setConnectionRetrialTimeOut(1)
 
            try:
                session.transaction().start(True) # readOnly
                self.schema = ConfigDBLoader.getSchema(credential)
                qdict = { "schema" : self.schema, "dbkey" : self.dbkey }
                
                # Choose query based on schema
                schemaVersion = ConfigDBLoader.readSchemaVersion(qdict, session)
                qstr = self.getQueryDefinition(schemaVersion)
                # execute coral query
                query = ConfigDBLoader.getCoralQuery(session, qstr, qdict)
                cursor = query.execute()
 
            except Exception as e:
                log.warning(f"DB query on {credential} failed to execute.")
                log.warning("Exception message: %r", e)
                failureMode = max(2, failureMode)
                continue # to next source
 
            # Read query result
            if not cursor.next():
                # empty result
                log.warning(f"DB query on {credential} returned empty result, likely due to non-existing key {self.dbkey}")
                failureMode = 3
                continue # to next source
 
            configblob = cursor.currentRow()[0].data()
            if type(configblob) is not str:
                configblob = configblob.readline()
            config = json.loads(configblob)
            session.transaction().commit()
            
            self.confirmConfigType(config)
            return config
 
        if failureMode == 1:
            log.error("TriggerDB query: could not connect to any source for %s", self.configType.basename)
            log.error("Considered sources: %s", ", ".join(credentials))
            raise RuntimeError("TriggerDB query: could not connect to any source", self.configType.basename)
        if failureMode == 2:
            log.error("Query failed due to wrong definition for %s", self.configType.basename)
            log.error("DB query was: %s", qstr.format(**qdict))
            raise RuntimeError("Query failed due to wrong definition", self.configType.basename)
        elif failureMode == 3:
            log.error("DB key %s does not exist for %s", self.dbkey, self.configType.basename)
            raise KeyError("DB key does not exist", self.dbkey, self.configType.basename)
        else:
            raise RuntimeError("Query failed for unknown reason")
 

readSchemaVersion()

def python.TriggerConfigAccessBase.ConfigDBLoader.readSchemaVersion (   qdict,   session  )

static

Read schema version form database, based on TrigConf::TrigDBLoader::schemaVersion 

Definition at line 196 of file TriggerConfigAccessBase.py.

    def readSchemaVersion(qdict, session):
        ''' Read schema version form database, based on TrigConf::TrigDBLoader::schemaVersion '''
        try:
            q = "SELECT TS_TAG FROM {schema}.TRIGGER_SCHEMA TS"
            query = ConfigDBLoader.getCoralQuery(session, q.format(**qdict))
            cursor = query.execute()
            cursor.next()
 
            versionTag = cursor.currentRow()[0].data()
 
            versionTagPrefix = "Trigger-Run3-Schema-v"
            if not versionTag.startswith(versionTagPrefix):
                raise RuntimeError( "Tag format error: Trigger schema version tag %s does not start with %s", versionTag, versionTagPrefix) 
 
            vstr = versionTag[len(versionTagPrefix)]
 
            if not vstr.isdigit():
                raise RuntimeError( "Invalid argument when interpreting the version part %s of schema tag %s is %s", vstr, versionTag, type(vstr))
 
            log.debug("Found schema version %s", vstr)
            return int(vstr)
 
        except Exception as e:
            log.warning("Failed to read schema version: %r", e)
 

setQuery()

def python.TriggerConfigAccessBase.ConfigDBLoader.setQuery	(		self,
			query
	)

query template is a dictionary of queries, identified by schema version, 
similar to TrigConf::TrigDBMenuLoader::m_hltQueries and TrigConf::TrigDBMenuLoader::m_l1Queries

Reimplemented from python.TriggerConfigAccessBase.ConfigLoader.

Definition at line 109 of file TriggerConfigAccessBase.py.

    def setQuery(self, query):
        """
        query template is a dictionary of queries, identified by schema version, 
        similar to TrigConf::TrigDBMenuLoader::m_hltQueries and TrigConf::TrigDBMenuLoader::m_l1Queries
        """
        self.query = query
 

Member Data Documentation

dbalias

python.TriggerConfigAccessBase.ConfigDBLoader.dbalias

Definition at line 104 of file TriggerConfigAccessBase.py.

dbkey

python.TriggerConfigAccessBase.ConfigDBLoader.dbkey

Definition at line 105 of file TriggerConfigAccessBase.py.

query

python.TriggerConfigAccessBase.ConfigDBLoader.query

Definition at line 106 of file TriggerConfigAccessBase.py.

schema

python.TriggerConfigAccessBase.ConfigDBLoader.schema

Definition at line 107 of file TriggerConfigAccessBase.py.

---

The documentation for this class was generated from the following file:

• TriggerConfigAccessBase.py