TileCalibCrest.py

Go to the documentation of this file.

#!/bin/env python
 
# Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
# TileCalibCrest.py
# Sanya Solodkov <Sanya.Solodkov@cern.ch>, 2025-02-04
# Laura Sargsyan <Laura.Sargsyan@cern.ch>, 2025-09-16
# Siarhei Harkusha <Siarhei.Harkusha@cern.ch>, 2025-05-19
 
"""
Python helper module for managing CREST DB connections and TileCalibBlobs.
"""
 
import os, re, cppyy, base64, json, time, datetime
 
from PyCool import cool # noqa: F401
Blob = cppyy.gbl.coral.Blob
 
from pycrest.api.crest_api import CrestApi
 
from TileCalibBlobObjs.Classes import TileCalibUtils, TileCalibDrawerCmt, \
     TileCalibDrawerInt, TileCalibDrawerOfc, TileCalibDrawerBch, \
     TileCalibDrawerFlt, TileCalibType
from TileCalibBlobPython import TileCalibTools
 
#=== get a logger
from TileCalibBlobPython.TileCalibLogger import TileCalibLogger, getLogger
log = getLogger("TileCalibCrest")
 
 
#======================================================================
#===
#=== Global helper functions
#===
#======================================================================
 
#
#______________________________________________________________________
#=== useful constants
MINRUN      = 0
MINLBK      = 0
MAXRUN      = (1<<31)-1
MAXLBK      = (1<<32)-1
MAXRUNLUMI  = (MAXRUN<<32)+MAXLBK
# empty Tile channel for storing laser partition variation. DO NOT CHANGE.
LASPARTCHAN = 43
 
 
class TileBlobReaderCrest(TileCalibLogger):
    """
    TileCalibBlobReader is a helper class, managing the details of CREST interactions for
    the user of TileCalibBlobs.
    """
 
    #____________________________________________________________________
    def __init__(self, db, folder='', tag='', run=None, lumi=0, modmin=0, modmax=275, copyBlob=False):
        """
        Input:
        - db    : server connection string or file name
        - folder: full folder path
        - tag   : The folder tag, e.g. \"UPD4-24\" or full tag
        - run   : Run number (if known)
        - lumi  : Lumi block number
        - modmin: Minimal module (COOL channel number)
        - modmax: Maximal module (COOL channel number)
        - copyBlob: save payload from CREST (Default:False, True to copy payload to json file)
        """
          #=== initialize base class
        TileCalibLogger.__init__(self,"TileBlobReader")
        self.payload = {}
        self.__db = db
        self.__folder = folder
        self.__tag = tag
        self.__copyBlob = copyBlob
 
        self.__iovList = []
        self.__iov = (-1,0)
        self.__commentBlob = None
        self.__drawerBlob = [None]*276
        self.__comment = None
        self.__drawer = [None]*276
        self.__modmin = modmin
        self.__modmax = modmax+1
 
        #=== try to open db
        self.__remote = (("http://" in db) or ("https://" in db) or ("CREST" in db))
        if self.__remote:
            if 'http' not in self.__db:
                self.__db = os.getenv(db,os.getenv('CREST_HOST',os.getenv('CREST_SERVER_PATH','http://crest-j23.cern.ch:8080/api-v5.0')))
            self.log().info('Host %s' , (self.__db))
            self.__api_instance = CrestApi(host=self.__db)
            socks = os.getenv('CREST_SOCKS', 'False')
            if socks == 'True': self.__api_instance.socks()
            self.__tag = self.getFolderTag(folder,None,tag)
            if run is not None and lumi is not None:
                log.info("Initializing for run %d, lumiblock %d", run,lumi)
                self.__getIov((run,lumi),True)
        else:
            self.log().info('File %s' , (self.__db))
            self.__api_instance = None
            self.__tag = 'unknown'
            self.__iovList.append(((MINRUN,MINLBK),(MAXRUN, MAXLBK)))
            self.__iov = self.__runlumi2iov(self.__iovList[-1])
            with open(self.__db, 'r') as the_file:
                jdata = json.load(the_file)
                for chan in range(self.__modmin,self.__modmax):
                    try:
                        blob=jdata[str(chan)][0]
                    except Exception:
                        blob=None
                    self.__create_drawer(blob,chan)
                try:
                    blob=jdata['1000'][0]
                except Exception:
                    blob=None
                self.__create_comment(blob)
 
    #____________________________________________________________________
    def getTag(self):
        return self.__tag
 
    #____________________________________________________________________
    def getFolderTag(self, folder, prefix, globalTag, api=None):
        if globalTag=='CURRENT' or globalTag=='UPD4' or globalTag=='' or globalTag=='HEAD':
            globalTag=TileCalibTools.getAliasFromFile('Current')
            log.info("Resolved CURRENT globalTag to \'%s\'", globalTag)
        elif globalTag=='CURRENTES' or globalTag=='UPD1':
            globalTag=TileCalibTools.getAliasFromFile('CurrentES')
            log.info("Resolved CURRENT ES globalTag to \'%s\'", globalTag)
        elif globalTag=='NEXT':
            globalTag=TileCalibTools.getAliasFromFile('Next')
            log.info("Resolved NEXT globalTag to \'%s\'", globalTag)
        elif globalTag=='NEXTES':
            globalTag=TileCalibTools.getAliasFromFile('NextES')
            log.info("Resolved NEXT ES globalTag to \'%s\'", globalTag)
        globalTag=globalTag.replace('*','')
        if prefix is None:
            prefix = ''
            for f in folder.split('/'):
                if re.findall('[a-z]+',f) != [] and f!='CellNoise':
                    prefix+=f
                else:
                    prefix+=f.capitalize()
        else:
            prefix=prefix.strip('-').split('-')[0]
        if prefix.startswith('Calo') and 'NoiseCell' not in prefix:
            prefix='CALO'+prefix[4:]
        if 'UPD1' in globalTag or 'UPD4' in globalTag or 'COND' not in globalTag:
            if prefix != '':
                if globalTag.startswith(prefix) or globalTag.startswith(prefix.upper()):
                    tag=globalTag
                else:
                    tag=prefix+'-'+globalTag
                self.log().info("Resolved localTag \'%s\' to folderTag \'%s\'", globalTag,tag)
            elif folder!='' and not (globalTag.upper().startswith('TILE') or globalTag.upper().startswith('CALO')):
                tag = TileCalibUtils.getFullTag(folder, globalTag)
                if tag.startswith('Calo') and 'NoiseCell' not in tag:
                    tag='CALO'+tag[4:]
                self.log().info("Resolved localTag \'%s\' to folderTag \'%s\'", globalTag,tag)
            else:
                tag=globalTag
                self.log().info("Use localTag \'%s\' as is", tag)
        else:
            tag=None
            if api:
                tags=api.find_global_tag_map(globalTag)
            else:
                tags=self.__api_instance.find_global_tag_map(globalTag)
            if tags.size==0:
                raise Exception( "globalTag %s not found" % (globalTag) )
            else:
                for i in range(tags.size):
                    t=tags.resources[i].tag_name
                    l=tags.resources[i].label
                    if (prefix!='' and t.startswith(prefix)) or l==folder:
                        tag=t
                        self.log().info("Resolved globalTag \'%s\' to folderTag \'%s\'", globalTag,tag)
                        #taginfo = self.__api_instance.find_tag(name=tag)
                        #print(taginfo)
        return tag
 
    #____________________________________________________________________
    def getIovs(self,since=(MINRUN,MINLBK),until=(MAXRUN,MAXLBK)):
        if self.__api_instance is None:
            return [self.__iovList[0][0]]
        else:
            run_lumi1=str((since[0]<<32)+since[1]+1)
            run_lumi2=str((until[0]<<32)+until[1]+1)
            MAXRUNLUMI1=str(MAXRUNLUMI+1)
            iovs1=self.__api_instance.select_iovs(self.__tag,"0",run_lumi1,sort='id.since:DESC,id.insertionTime:DESC',size=1,snapshot=0)
            iovs2=self.__api_instance.select_iovs(self.__tag,run_lumi2,MAXRUNLUMI1,sort='id.since:ASC,id.insertionTime:DESC',size=1,snapshot=0)
            since1=0 if iovs1.size==0 else iovs1.resources[0].since
            until1=MAXRUNLUMI if iovs2.size==0 else iovs2.resources[0].since
            iovs=self.__api_instance.select_iovs(self.__tag,str(since1),str(until1),sort='id.since:ASC,id.insertionTime:DESC',size=999999,snapshot=0)
            iovList=[]
            if iovs.size==0:
                raise Exception( "IOV for tag %s IOV [%s,%s] - (%s,%s) not found" % (self.__tag,since[0],since[1],until[0],until[1]) )
            else:
                for i in range(iovs.size):
                    iov=iovs.resources[i]
                    since=int(iov.since)
                    runS=since>>32
                    lumiS=since&0xFFFFFFFF
                    iovList.append((runS,lumiS))
            return iovList
 
    #____________________________________________________________________
    def getIov(self, option=0):
        if option!=0:
            return self.__iov
        else:
            return self.__iovList[-1]
 
    #____________________________________________________________________
    def __getIov(self,runlumi,dbg=False):
        if self.__api_instance is None:
            pass
        else:
            run_lumi1=str((runlumi[0]<<32)+runlumi[1]+1)
            MAXRUNLUMI1=str(MAXRUNLUMI+1)
            iovs1=self.__api_instance.select_iovs(self.__tag,"0",run_lumi1,sort='id.since:DESC,id.insertionTime:DESC',size=1,snapshot=0)
            iovs2=self.__api_instance.select_iovs(self.__tag,run_lumi1,MAXRUNLUMI1,sort='id.since:ASC,id.insertionTime:DESC',size=1,snapshot=0)
            if iovs1.size==0:
                raise Exception( "IOV for tag %s run,lumi (%s,%s) not found" % (self.__tag,runlumi[0],runlumi[1]) )
            else:
                iov=iovs1.resources[0]
                since=int(iov.since)
                runS=since>>32
                lumiS=since&0xFFFFFFFF
                until=MAXRUNLUMI if iovs2.size==0 else iovs2.resources[0].since
                runU=until>>32
                lumiU=until&0xFFFFFFFF
                hash=iov.payload_hash
                if dbg:
                    #self.log().info('Run,Lumi (%d,%d)' , runlumi)
                    self.log().info('IOV [%d,%d] - (%d,%d)' , runS,lumiS,runU,lumiU)
                    self.log().info('Insertion time %s' , iov.insertion_time)
                    self.log().info('Hash %s' , hash)
                payload = self.__api_instance.get_payload(hash=hash).decode('utf-8')
                jdata=json.loads(payload)
                #with open("payload.json", 'w') as the_file:
                #    the_file.write(payload)
                #    the_file.write('\n')
                if self.__copyBlob:
                    self.payload = jdata
                    return
                self.__iovList.append(((runS,lumiS),(runU, lumiU)))
                self.__iov = self.__runlumi2iov(self.__iovList[-1])
                for chan in range(self.__modmin,self.__modmax):
                    try:
                        blob=jdata[str(chan)][0]
                    except Exception:
                        blob=None
                    self.__create_drawer(blob,chan)
                try:
                    blob=jdata['1000'][0]
                except Exception:
                    blob=None
                self.__create_comment(blob)
        return
 
    #____________________________________________________________________
    def __runlumi2iov(self,runlumi):
        since = (runlumi[0][0]<<32) + runlumi[0][1]
        until = (runlumi[1][0]<<32) + runlumi[1][1]
        return (since,until)
 
    #____________________________________________________________________
    def __checkIov(self,runlumi):
        point = (runlumi[0]<<32) + runlumi[1]
        inrange = point>=self.__iov[0] and point<self.__iov[1]
        return inrange
 
    #____________________________________________________________________
    def __make_blob(self,string):
        b = Blob()
        b.write(string)
        b.seek(0)
        return b
 
    #____________________________________________________________________
    def __create_comment(self,b64string):
        if b64string is None or len(b64string)==0:
            if b64string is None:
                self.__commentBlob = None
            else:
                self.__commentBlob = 0
            self.__comment = None
        else:
            blob1 = base64.decodebytes(bytes(b64string,'ascii'))
            self.__commentBlob = self.__make_blob(blob1)
            self.__comment = TileCalibDrawerCmt.getInstance(self.__commentBlob)
        return
 
    #____________________________________________________________________
    def __create_drawer(self,b64string,chan):
        if b64string is None or isinstance(b64string, (int, float)) or len(b64string)==0:
            if b64string is None:
                self.__drawerBlob[chan] = None
            else:
                self.__drawerBlob[chan] = 0
            self.__drawer[chan] = None
            return
        blob1 = base64.decodebytes(bytes(b64string,'ascii'))
        self.__drawerBlob[chan] = self.__make_blob(blob1)
        cmt = TileCalibDrawerCmt.getInstance(self.__drawerBlob[chan])
        typeName = TileCalibType.getClassName(cmt.getObjType())
        del cmt
        #=== create calibDrawer depending on type
        if   typeName=='TileCalibDrawerFlt':
            self.__drawer[chan] = TileCalibDrawerFlt.getInstance(self.__drawerBlob[chan])
            self.log().debug( "typeName = Flt " )
        elif typeName=='TileCalibDrawerInt':
            self.__drawer[chan] = TileCalibDrawerInt.getInstance(self.__drawerBlob[chan])
            self.log().debug( "typeName = Int " )
        elif typeName=='TileCalibDrawerBch':
            self.__drawer[chan] = TileCalibDrawerBch.getInstance(self.__drawerBlob[chan])
            self.log().debug( "typeName = Bch " )
        elif typeName=='TileCalibDrawerOfc':
            self.__drawer[chan] = TileCalibDrawerOfc.getInstance(self.__drawerBlob[chan])
            self.log().debug( "typeName = Ofc " )
        elif typeName=='TileCalibDrawerCmt':
            self.__drawer[chan] = cppyy.gbl.CaloCondBlobFlt.getInstance(self.__drawerBlob[chan])
            self.log().debug( "typeName = CaloFlt " )
        else:
            self.__drawer[chan] = None
            self.log().warn("Unknown blob type for chan %d - ignoring", chan)
        return
 
    #____________________________________________________________________
    def getBlob(self,ros, mod, runlumi=None, dbg=False):
 
        if self.__remote and runlumi is not None and not self.__checkIov(runlumi):
            self.__getIov(runlumi,dbg)
 
        if ros<0:
            chanNum = mod
        else:
            chanNum = TileCalibUtils.getDrawerIdx(ros,mod)
 
        if (chanNum>=0 and chanNum<len(self.__drawer)):
            return self.__drawerBlob[chanNum]
        else:
            raise Exception( "Invalid drawer requested: %s %s" % (ros,mod) )
 
    #____________________________________________________________________
    def getDrawer(self,ros, mod, runlumi=None, dbg=False, useDefault=True):
 
        if self.__remote and runlumi is not None and not self.__checkIov(runlumi):
            self.__getIov(runlumi,dbg)
 
        if ros<0:
            chanNum = mod
        else:
            chanNum = TileCalibUtils.getDrawerIdx(ros,mod)
 
        if (chanNum>=0 and chanNum<len(self.__drawer)):
            drawer=self.__drawer[chanNum]
            if not useDefault and drawer is None:
                if self.__drawerBlob[chanNum] is None:
                    return None
                else:
                    return 0
            while drawer is None:
                #=== no default at all?
                if ros==0 and drawer==0:
                    raise Exception('No default available')
                #=== follow default policy
                ros,mod = self.getDefault(ros,mod)
                chanNum = TileCalibUtils.getDrawerIdx(ros,mod)
                drawer=self.__drawer[chanNum]
            return drawer
        elif (chanNum == 1000):
            return self.__comment
        else:
            raise Exception( "Invalid drawer requested: %s %s" % (ros,mod) )
 
    #____________________________________________________________________
    def getComment(self,runlumi=None,split=False):
 
        if self.__remote and runlumi is not None and not self.__checkIov(runlumi):
            self.__getIov(runlumi)
        if self.__comment is not None:
            if split:
                return (self.__comment.getAuthor(),self.__comment.getComment(),self.__comment.getDate())
            else:
                return self.__comment.getFullComment()
        else:
            return "<no comment found>"
 
    #____________________________________________________________________
    def getDefault(self, ros, drawer):
        """
        Returns a default drawer number (among first 20 COOL channels) for any drawer in any partition
        """
        if ros==0:
            if drawer<=4 or drawer==12 or drawer>=20:
                drawer1=0
            elif drawer<12:
                drawer1=4
            else:
                drawer1=12
        elif ros==1 or ros==2:
            drawer1=4
        elif ros==3:
            OffsetEBA = [ 0, 0, 0, 0, 0, 0, 3, 2, #// Merged E+1: EBA07; Outer MBTS: EBA08
                          0, 0, 0, 0, 7, 6, 5, 7, #// D+4: EBA13, EBA16; Special D+4: EBA14; Special D+40: EBA15
                          7, 6, 6, 7, 0, 0, 0, 2, #// D+4: EBA17, EBA20; Special D+4: EBA18, EBA19; Outer MBTS: EBA24
                          3, 0, 0, 0, 0, 0, 0, 0, #// Merged E+1:  EBA25
                          0, 0, 0, 0, 0, 0, 1, 1, #// Inner MBTS + special C+10: EBA39, EBA40
                          1, 1, 2, 3, 0, 0, 0, 0, #// Inner MBTS + special C+10: EBA41, EBA42; Outer MBTS: EBA43; Merged E+1: EBA44
                          0, 0, 0, 0, 3, 2, 1, 1, #// Merged E+1: EBA53; Outer MBTS: EBA54; Inner MBTS + special C+10: EBA55, EBA56
                          1, 1, 0, 0, 0, 0, 0, 0] #// Inner MBTS + special C+10: EBA57, EBA58
            drawer1 = 12 + OffsetEBA[drawer]
        elif ros==4:
            OffsetEBC = [ 0, 0, 0, 0, 0, 0, 3, 2, #// Merged E-1: EBC07; Outer MBTS: EBC08
                          0, 0, 0, 0, 7, 6, 6, 7, # // D-4: EBC13, EBC16; Special D-4: EBC14, EBC15;
                          7, 5, 6, 7, 0, 0, 0, 2, #// D-4: EBC17, EBC20; Special D-40 EBC18; Special D-4: EBC19; Outer MBTS: EBC24
                          3, 0, 0, 3, 4, 0, 3, 4, #// Merged E-1:  EBC25, EBC28, EBC31; E-4': EBC29, EBC32
                          0, 4, 3, 0, 4, 3, 1, 1, #// E-4': EBC34, EBC37; Merged E-1: EBC35, EBC38; Inner MBTS + special C-10: EBC39, EBC40
                          1, 1, 2, 3, 0, 0, 0, 0, #// Inner MBTS + special C-10: EBC41, EBC42; Outer MBTS: EBC43; Merged E-1: EBC44
                          0, 0, 0, 0, 3, 2, 1, 1, #// Merged E-1: EBC53; Outer MBTS: EBC54; Inner MBTS + special C-10: EBC55, EBC56
                          1, 1, 0, 0, 0, 0, 0, 0] #// Inner MBTS + special C-10: EBC57, EBC58
            drawer1 = 12 + OffsetEBC[drawer]
        else:
            drawer1=0
 
        return (0,drawer1)
 
class TileBlobWriterCrest(TileCalibLogger):
    """
    TileBlobWriterCrest is a helper class, managing the details of
    CREST interactions for the user of TileCalibBlobs.
    """
 
    #____________________________________________________________________
    def __init__(self, db, folderPath, calibDrawerType, isMultiVersionFolder=True):
        """
        Input:
        - db        : db should be a database connection
        - folderPath: full folder path to create or update
        """
 
        #=== initialize base class
        TileCalibLogger.__init__(self, "TileBlobWriter")
 
        #=== store db
        self.__db = db
        self.__folderPath = folderPath
 
        #=== create default vectors based on calibDrawerType
        self.__calibDrawerType = calibDrawerType
        if calibDrawerType in ['TileCalibDrawerFlt', 'Flt']:
            self.__TileCalibDrawer = TileCalibDrawerFlt
            self.__defVec = cppyy.gbl.std.vector('std::vector<float>')()
        elif calibDrawerType in ['TileCalibDrawerBch', 'Bch']:
            self.__TileCalibDrawer = TileCalibDrawerBch
            self.__defVec = cppyy.gbl.std.vector('std::vector<unsigned int>')()
        elif calibDrawerType in ['TileCalibDrawerInt', 'Int']:
            self.__TileCalibDrawer = TileCalibDrawerInt
            self.__defVec = cppyy.gbl.std.vector('std::vector<unsigned int>')()
        else:
            raise Exception("Unknown calibDrawerType: %s" % calibDrawerType)
 
        # Always all drawers should be written
        self.__drawerBlob = {drawerIdx:Blob() for drawerIdx in range(0, TileCalibUtils.max_draweridx())}
        self.__drawer = {}
    #____________________________________________________________________
    def register(self, since=(MINRUN,MINLBK), tag=""):
        """
        Registers the folder in the database.
        - since: lower limit of IOV
        - tag  : The tag to write to
 
        The interpretation of the 'since' inputs depends on their type:
        - tuple(int,int) : run and lbk number
        """
 
        jdata = {}
        for drawerIdx,blob in self.__drawerBlob.items():
            if blob is None or blob==0:
                b64string = ''
            else:
                b64string = str(base64.b64encode(blob.read()), 'ascii')
            jdata[drawerIdx] = [b64string]
 
        (sinceRun, sinceLumi) = since
 
        if not self.__db or (self.__db and self.__db.endswith('.json')):
            # Writting into the json file
            fullTag = tag
            if self.__folderPath and not (tag.upper().startswith('TILE') or tag.upper().startswith('CALO')):
                fullTag = TileCalibUtils.getFullTag(self.__folderPath, tag)
            fileName = f"{fullTag}.{sinceRun}.{sinceLumi}.json"
            if self.__db:
                fileName = f'{self.__db[:-5]}.{fileName}'
 
            with open(fileName, 'w') as the_file:
                json.dump(jdata, the_file)
                the_file.write('\n')
 
        #=== print info
        self.log().info( 'Writting tag "%s"', fullTag)
        self.log().info( '... since             : [%s,%s]' , sinceRun, sinceLumi)
        self.log().info( '... with comment field: "%s"', self.getComment())
        self.log().info( '... into file         : %s' , fileName)
 
    #____________________________________________________________________
    def setComment(self, author, comment=None):
        """
        Sets a general comment in the comment channel.
        """
        drawerIdx = TileCalibUtils.getCommentChannel()
        commentBlob = self.__drawer.get(drawerIdx, None)
        if not commentBlob:
            commentBlob = Blob()
            self.__drawerBlob[drawerIdx] = commentBlob
 
        if isinstance(author, tuple) and len(author) == 3:
            tm = time.mktime(datetime.datetime.strptime(author[2], "%a %b %d %H:%M:%S %Y").timetuple())
            self.__drawer[drawerIdx] = TileCalibDrawerCmt.getInstance(commentBlob, author[0], author[1], int(tm))
        else:
            self.__drawer[drawerIdx] = TileCalibDrawerCmt.getInstance(commentBlob, author, comment)
 
    #____________________________________________________________________
    def getComment(self, split=False):
        """
        Returns the general comment (default if none is set)
        """
        drawerIdx = TileCalibUtils.getCommentChannel()
        comment = self.__drawer.get(drawerIdx, None)
        if comment:
            if split:
                return (comment.getAuthor(), self.__comment.getComment(), self.__comment.getDate())
            else:
                return comment.getFullComment()
        else:
            return "<No general comment!>"
 
    #____________________________________________________________________
    def getDrawer(self, ros, drawer, calibDrawerTemplate=None):
        """
        Returns a TileCalibDrawer object of requested type
        for the given ROS and drawer.
        """
 
        try:
            drawerIdx = TileCalibUtils.getDrawerIdx(ros, drawer)
            calibDrawer = self.__drawer.get(drawerIdx, None)
 
            if not calibDrawer:
                calibDrawer = self.__TileCalibDrawer.getInstance(self.__drawerBlob[drawerIdx], self.__defVec,0,0)
                self.__drawer[drawerIdx] = calibDrawer
 
            #=== clone if requested
            if calibDrawerTemplate:
                calibDrawer.clone(calibDrawerTemplate)
 
            return calibDrawer
 
        except Exception as e:
            self.log().critical( e )
            return None
 
    #____________________________________________________________________
    def zeroBlob(self, ros, drawer):
        """
        Resets blob size to zero
        """
        try:
            drawerIdx = TileCalibUtils.getDrawerIdx(ros, drawer)
            blob = self.__drawerBlob[drawerIdx]
            blob.resize(0)
        except Exception as e:
            self.log().critical( e )
            return None