11Python helper module for managing CREST DB connections and TileCalibBlobs.
14import os, re, cppyy, base64, json, time, datetime
16from PyCool
import cool
17Blob = cppyy.gbl.coral.Blob
20chai.set_log_level(chai.LogLevel.WARNING)
22from TileCalibBlobObjs.Classes
import TileCalibUtils, TileCalibDrawerCmt, \
23 TileCalibDrawerInt, TileCalibDrawerOfc, TileCalibDrawerBch, \
24 TileCalibDrawerFlt, TileCalibType
25from TileCalibBlobPython
import TileCalibTools
28from TileCalibBlobPython.TileCalibLogger
import TileCalibLogger, getLogger
29log = getLogger(
"TileCalibCrest")
31CREST_HOST = os.getenv(
'CREST_HOST',os.getenv(
'CREST_SERVER',os.getenv(
'CHAI_CREST_SERVER',os.getenv(
'CREST_SERVER_PATH',os.getenv(
'CREST_API_URL',
'https://atlas-crest-dev.cern.ch:443/api-v6.4')))))
52 TileCalibBlobReader is a helper class, managing the details of CREST interactions for
53 the user of TileCalibBlobs.
57 def __init__(self, db, folder='', tag='', run=None, lumi=0, modmin=0, modmax=275, copyBlob=False):
60 - db : server connection string or file name
61 - folder: full folder path
62 - tag : The folder tag, e.g. \"UPD4-24\" or full tag
63 - run : Run number (if known)
64 - lumi : Lumi block number
65 - modmin: Minimal module (COOL channel number)
66 - modmax: Maximal module (COOL channel number)
67 - copyBlob: save payload from CREST (Default:False, True to copy payload to json file)
70 TileCalibLogger.__init__(self,
"TileBlobReader")
72 if db
is None or (
'CREST' in db
and 'http' not in db):
94 if 'http' in self.
__db:
95 self.log().info(
'Host %s' , (self.
__db))
98 self.log().info(
'Local db %s' , (self.
__db))
105 if run
is not None and lumi
is not None:
106 log.info(
"Initializing for run %d, lumiblock %d", run,lumi)
108 elif run
is None and lumi
is not None:
109 log.info(
"Initializing for hash %s", lumi)
112 self.log().info(
'File %s' , (self.
__db))
115 self.
__tag =
'unknown'
116 self.
__iovList.append(((MINRUN,MINLBK),(MAXRUN, MAXLBK)))
118 with open(self.
__db,
'r')
as the_file:
119 jdata = json.load(the_file)
125 blob=jdata[str(chan)][0]
130 blob=jdata[
'1000'][0]
141 if globalTag==
'CURRENT' or globalTag==
'UPD4' or globalTag==
'' or globalTag==
'HEAD':
142 globalTag=TileCalibTools.getAliasFromFile(
'Current')
143 log.info(
"Resolved CURRENT globalTag to \'%s\'", globalTag)
144 elif globalTag==
'CURRENTES' or globalTag==
'UPD1':
145 globalTag=TileCalibTools.getAliasFromFile(
'CurrentES')
146 log.info(
"Resolved CURRENT ES globalTag to \'%s\'", globalTag)
147 elif globalTag==
'NEXT':
148 globalTag=TileCalibTools.getAliasFromFile(
'Next')
149 log.info(
"Resolved NEXT globalTag to \'%s\'", globalTag)
150 elif globalTag==
'NEXTES':
151 globalTag=TileCalibTools.getAliasFromFile(
'NextES')
152 log.info(
"Resolved NEXT ES globalTag to \'%s\'", globalTag)
153 globalTag=globalTag.replace(
'*',
'')
156 for f
in folder.split(
'/'):
157 if re.findall(
'[a-z]+',f) != []
and f!=
'CellNoise':
160 prefix+=f.capitalize()
162 prefix=prefix.strip(
'-').
split(
'-')[0]
163 if prefix.startswith(
'Calo')
and 'NoiseCell' not in prefix:
164 prefix=
'CALO'+prefix[4:]
165 prefix=prefix.replace(
'Pileupnoiselumi',
'PileUpNoiseLumi')
166 if 'UPD1' in globalTag
or 'UPD4' in globalTag
or (
'COND' not in globalTag
and 'CREST' not in globalTag):
168 if prefix
in globalTag
or prefix.upper()
in globalTag:
171 tag=prefix+
'-'+globalTag
172 self.log().info(
"Resolved localTag \'%s\' to folderTag \'%s\'", globalTag,tag)
173 elif folder!=
'' and not (globalTag.upper().startswith(
'TILE')
or globalTag.upper().startswith(
'CALO')):
175 if tag.startswith(
'Calo')
and 'NoiseCell' not in tag:
177 tag=tag.replace(
'Pileupnoiselumi',
'PileUpNoiseLumi')
178 self.log().info(
"Resolved localTag \'%s\' to folderTag \'%s\'", globalTag,tag)
181 self.log().info(
"Use localTag \'%s\' as is", tag)
186 gtag = api.get_global_tag(globalTag)
190 api = chai.Database(f
"crest:{CREST_HOST}")
191 gtag = api.get_global_tag(globalTag)
196 raise Exception(
"globalTag %s not found" % (globalTag) )
200 if (prefix!=
'' and t.startswith(prefix))
or l==folder:
202 self.log().info(
"Resolved globalTag \'%s\' to folderTag \'%s\'", globalTag,tag)
208 def getIovs(self,since=(MINRUN,MINLBK),until=(MAXRUN,MAXLBK)):
212 run_lumi1=(since[0]<<32)+since[1]
213 run_lumi2=(until[0]<<32)+until[1]
219 lumiS=iov.since&0xFFFFFFFF
220 if (runS,lumiS)
not in iovList:
221 iovList.append((runS,lumiS))
223 raise Exception(
"IOV for tag %s IOV [%s,%s] - (%s,%s) not found" % (self.
__tag,since[0],since[1],until[0],until[1]) )
250 if isinstance(hash,int)
and hash>=0
and hash<len(self.
__payload_hash):
255 raise Exception(
"Payload for hash %s not found" % hash )
256 jdata = json.loads(payload.to_json())
262 blob=jdata[str(chan)][0]
267 blob=jdata[
'1000'][0]
283 usehash =
not (isinstance(runlumi,list)
or isinstance(runlumi,tuple))
287 res=self.
__tag_instance.get_payload_at((runlumi[0]<<32)+runlumi[1])
290 raise Exception(
"IOV for tag %s run,lumi (%s,%s) not found" % (self.
__tag,runlumi[0],runlumi[1]) )
292 raise Exception(
"IOV for tag %s hash %s not found" % (self.
__tag,runlumi) )
295 (runS,lumiS,runU,lumiU) = (0,0,MAXRUN,MAXLBK)
298 jdata = json.loads(res.to_json())
300 self.log().info(
'hash %s' , runlumi)
303 lumiS=res.since&0xFFFFFFFF
304 if res.until
is None:
309 lumiU=res.until&0xFFFFFFFF
312 self.log().info(
'IOV [%d,%d] - (%d,%d)' , runS,lumiS,runU,lumiU)
317 if iov.since == res.since:
321 self.log().info(
'Insertion time %s Hash %s' , iov.insertion_time, iov.payload_hash)
324 jdata = json.loads(res.payload.to_json())
328 self.
__iovList.append(((runS,lumiS),(runU, lumiU)))
335 blob=jdata[str(chan)][0]
340 blob=jdata[
'1000'][0]
348 since = (runlumi[0][0]<<32) + runlumi[0][1]
349 until = (runlumi[1][0]<<32) + runlumi[1][1]
354 if isinstance(runlumi,list)
or isinstance(runlumi,tuple):
355 run = runlumi[0]
if isinstance(runlumi[0],int)
else 0
356 lumi = runlumi[1]
if isinstance(runlumi[1],int)
else 0
357 point = (run<<32) + lumi
359 point = runlumi
if isinstance(runlumi,int)
else 0
360 inrange = point>=self.
__iov[0]
and point<self.
__iov[1]
372 if b64string
is None or len(b64string)==0:
373 if b64string
is None:
379 blob1 = base64.decodebytes(bytes(b64string,
'ascii'))
386 if b64string
is None or isinstance(b64string, (int, float))
or len(b64string)==0:
387 if b64string
is None:
393 blob1 = base64.decodebytes(bytes(b64string,
'ascii'))
399 if typeName==
'TileCalibDrawerFlt':
401 self.log().
debug(
"typeName = Flt " )
402 elif typeName==
'TileCalibDrawerInt':
404 self.log().
debug(
"typeName = Int " )
405 elif typeName==
'TileCalibDrawerBch':
407 self.log().
debug(
"typeName = Bch " )
408 elif typeName==
'TileCalibDrawerOfc':
410 self.log().
debug(
"typeName = Ofc " )
411 elif typeName==
'TileCalibDrawerCmt':
413 self.log().
debug(
"typeName = CaloFlt " )
416 self.log().warn(
"Unknown blob type for chan %d - ignoring", chan)
428 def getBlob(self,ros, mod, runlumi=None, dbg=False):
435 if (chanNum>=0
and chanNum<len(self.
__drawer)):
438 raise Exception(
"Invalid drawer requested: %s %s" % (ros,mod) )
441 def getDrawer(self,ros, mod, runlumi=None, dbg=False, useDefault=True):
448 if (chanNum>=0
and chanNum<len(self.
__drawer)):
450 if not useDefault
and drawer
is None:
455 while drawer
is None:
457 if ros==0
and drawer==0:
458 raise Exception(
'No default available')
464 elif (chanNum == 1000):
467 raise Exception(
"Invalid drawer requested: %s %s" % (ros,mod) )
480 return "<no comment found>"
485 Returns a default drawer number (among first 20 COOL channels) for any drawer in any partition
488 if drawer<=4
or drawer==12
or drawer>=20:
494 elif ros==1
or ros==2:
497 OffsetEBA = [ 0, 0, 0, 0, 0, 0, 3, 2,
498 0, 0, 0, 0, 7, 6, 5, 7,
499 7, 6, 6, 7, 0, 0, 0, 2,
500 3, 0, 0, 0, 0, 0, 0, 0,
501 0, 0, 0, 0, 0, 0, 1, 1,
502 1, 1, 2, 3, 0, 0, 0, 0,
503 0, 0, 0, 0, 3, 2, 1, 1,
504 1, 1, 0, 0, 0, 0, 0, 0]
505 drawer1 = 12 + OffsetEBA[drawer]
507 OffsetEBC = [ 0, 0, 0, 0, 0, 0, 3, 2,
508 0, 0, 0, 0, 7, 6, 6, 7,
509 7, 5, 6, 7, 0, 0, 0, 2,
510 3, 0, 0, 3, 4, 0, 3, 4,
511 0, 4, 3, 0, 4, 3, 1, 1,
512 1, 1, 2, 3, 0, 0, 0, 0,
513 0, 0, 0, 0, 3, 2, 1, 1,
514 1, 1, 0, 0, 0, 0, 0, 0]
515 drawer1 = 12 + OffsetEBC[drawer]
522 def dumpIovs(self, iovList, rosmin, rosmax, drawermin, drawermax, option=1, comment=False, usenames=True):
524 Dumps statistics - how many non-empty modules exists in different IOVs
532 rosrange=list(range(rosmin,rosmax))
535 for since
in iovList:
536 iov=
"(%s,%s)" % since
544 (dmin,dmax) = (drawermin,drawermax)
549 for drawer
in range(dmin,dmax):
550 flt = self.
getDrawer(ros, drawer, since,
False,
False)
560 if mod
not in allmods:
567 allmod +=
" " + mod +
"_zero"
568 allmods[mod] +=
" " + iov +
"_zero"
571 allmods[mod] +=
" " + iov
575 word =
'module' if usenames
else 'COOL channel'
576 if miss==0
and nmod>1:
577 alliovs[iov] =
" All %s" %
plural(nmod,word)
578 elif miss>0
and miss<10:
579 alliovs[iov] =
" %s present, %s missing:%s" % (
plural(nmod-miss,word),
plural(miss,word),missmod)
581 alliovs[iov] =
"%s ; %s present, %s missing" % (allmod,
plural(nmod-miss,word),
plural(miss,word))
582 zeroiovs[iov] = (zero,zeromod)
586 for key,value
in allmods.items():
589 print(
"%s\t%s" % (key,value))
592 for key,value
in alliovs.items():
595 if zeroiovs[key]
and zeroiovs[key][0]>0:
596 if "_zero" not in value:
597 print(
"%s\t%s ; zero-sized blobs for %d modules:%s" % (key,value,zeroiovs[key][0],zeroiovs[key][1]))
599 print(
"%s\t%s ; zero-sized blobs for %d modules" % (key,value,zeroiovs[key][0]))
601 print(
"%s\t%s" % (key,value))
603 print(
"\nNo IOVs found")
608 TileBlobWriterCrest is a helper class, managing the details of
609 CREST interactions for the user of TileCalibBlobs.
613 def __init__(self, db, folderPath, calibDrawerType, payload=None):
616 - db : db should be a database connection
617 - folderPath: full folder path to create or update
621 TileCalibLogger.__init__(self,
"TileBlobWriter")
624 if db
is not None and 'CREST' in db
and 'http' not in db:
630 if payload
is not None:
635 if calibDrawerType
in [
'TileCalibDrawerFlt',
'Flt']:
637 self.
__defVec = cppyy.gbl.std.vector(
'std::vector<float>')()
638 elif calibDrawerType
in [
'TileCalibDrawerBch',
'Bch']:
640 self.
__defVec = cppyy.gbl.std.vector(
'std::vector<unsigned int>')()
641 elif calibDrawerType
in [
'TileCalibDrawerInt',
'Int']:
643 self.
__defVec = cppyy.gbl.std.vector(
'std::vector<unsigned int>')()
644 elif calibDrawerType
in [
'CaloCondBlobFlt',
'CaloFlt']:
646 self.
__defVec = cppyy.gbl.std.vector(
'std::vector<float>')()
648 raise Exception(
"Unknown calibDrawerType: %s" % calibDrawerType)
654 def register(self, since=(MINRUN,MINLBK), tag=
"", chan=-1, payload=
None):
656 Registers the folder in the database.
657 - since: lower limit of IOV
658 - tag : The tag to write to
659 - chan : COOL channel to write, if negative - all COOL channels are written
660 Comment channel is always written
662 The interpretation of the 'since' inputs depends on their type:
663 - tuple(int,int) : run and lbk number
666 if self.
__payload is None and payload
is None:
669 if chan<0
or drawerIdx==chan
or drawerIdx==1000:
670 if blob
is None or blob==0:
674 b64string = str(base64.b64encode(blob.read()),
'ascii')
675 jdata[drawerIdx] = [b64string]
677 if payload
is not None:
681 (sinceRun, sinceLumi) = since
683 if self.
__folderPath and not (tag.upper().startswith(
'TILE')
or tag.upper().startswith(
'CALO')):
686 if not self.
__db or self.
__db.endswith(
'.json'):
688 fileName = f
"{fullTag}.{sinceRun}.{sinceLumi}.json"
690 fileName = f
'{self.__db[:-5]}.{fileName}'
692 with open(fileName,
'w')
as the_file:
693 json.dump(jdata, the_file, separators=(
',',
':'), sort_keys=
True)
695 if 'http' in self.
__db:
696 self.log().info(
'Host %s' , (self.
__db))
698 fileName =
'remote host'
700 self.log().info(
'Local db %s' , (self.
__db))
702 fileName = self.
__db +
'/' + fullTag
711 '<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type="71" clid="1238547719" /></addrHeader><typeName>CondAttrListCollection</typeName>',
712 chai.IovType.RunNumberLumiBlock,
713 "crest-json-single-iov",
714 chai.Synchronization.All,
715 chai.TagStatus.Unlocked
717 except Exception
as e:
721 except Exception
as e2:
722 self.log().
error(f
"Failed to create or get tag: {e}")
723 self.log().
error(f
"Additional error: {e2}")
724 raise Exception(
"Failed to write data to %s" % (self.
__db) )
726 new_tag.add_payload(container, (sinceRun<<32)+sinceLumi)
729 self.log().info(
'Writting tag "%s"', fullTag)
730 self.log().info(
'... since : [%s,%s]' , sinceRun, sinceLumi)
732 self.log().info(
'... with comment field: "%s"', self.
getComment())
733 self.log().info(
'... into file : %s' , fileName)
738 Create a container with the blob data (base64 encoded).
741 jdata: all base64-encoded blob data in json format
742 name: name for field specification
743 type: type for field specification
746 Payload specification with all channels found in jdata
747 Container with the base64-encoded blob data
749 This method uses String type with base64 encoding instead of Blob type
750 due to limitations in nanobind's handling of BlobData during serialization
754 name=
'CaloCondBlob16M'
758 field_spec = chai.FieldSpec([chai.Field(name,type)])
761 for id
in jdata.items():
762 spec += [chai.Channel(int(id[0]),
"")]
763 channel_spec = chai.ChannelSpec(spec)
766 payload_spec = chai.PayloadSpec(field_spec, channel_spec)
769 container = chai.Container(payload_spec)
771 for id
in jdata.items():
772 container[int(id[0])].push_string(id[1][0])
774 return (payload_spec,container)
779 Sets a general comment in the comment channel.
784 commentBlob.resize(0)
789 if isinstance(author, tuple)
and len(author) == 3:
790 tm = time.mktime(datetime.datetime.strptime(author[2],
"%a %b %d %H:%M:%S %Y").timetuple())
798 Returns the general comment (default if none is set)
804 return (comment.getAuthor(), self.__comment.
getComment(), self.__comment.getDate())
806 return comment.getFullComment()
808 return "<no comment found>"
811 def getDrawer(self, ros, drawer, calibDrawerTemplate=None):
813 Returns a TileCalibDrawer object of requested type
814 for the given ROS and drawer.
826 self.
__drawer[drawerIdx] = calibDrawer
829 if calibDrawerTemplate:
830 calibDrawer.clone(calibDrawerTemplate)
834 except Exception
as e:
835 self.log().critical( e )
841 Resets blob size to zero
847 except Exception
as e:
848 self.log().critical( e )
855 Calculates TileCalibDrawer index
856 from ros number and drawer number
869 merges integer number and text
870 and adds 's' at the end of text
void print(char *figname, TCanvas *c1)
static const TileCalibDrawerBch * getInstance(const coral::Blob &blob)
Returns a pointer to a const TileCalibDrawerBch.
static const TileCalibDrawerCmt * getInstance(const coral::Blob &blob)
Returns a pointer to a const TileCalibDrawerCmt.
static const TileCalibDrawerFlt * getInstance(const coral::Blob &blob)
Returns a pointer to a const TileCalibDrawerFlt.
static const TileCalibDrawerInt * getInstance(const coral::Blob &blob)
Returns a pointer to a const TileCalibDrawerBch.
static TileCalibDrawerOfc * getInstance(coral::Blob &blob, uint16_t objVersion, uint32_t nSamples, int32_t nPhases, uint16_t nChans, uint16_t nGains, std::string_view author="", std::string_view comment="", uint64_t timeStamp=0)
Returns a pointer to a non-const TileCalibDrawerOfc.
static std::string getClassName(TileCalibType::TYPE type)
Returns the class name.
static std::string getDrawerString(unsigned int ros, unsigned int drawer)
Return the drawer name, e.g.
static unsigned int max_draweridx()
Python compatibility function.
static unsigned int getDrawerIdx(unsigned int ros, unsigned int drawer)
Returns a drawer hash.
static unsigned int getMaxDrawer(unsigned int ros)
Returns the maximal channel number for a given drawer.
static std::string getFullTag(const std::string &folder, const std::string &tag)
Returns the full tag string, composed of camelized folder name and tag part.
static unsigned int getCommentChannel()
Returns the COOL channel number for the comment channel.
getPayloadByHash(self, hash)
__create_comment(self, b64string)
getIovs(self, since=(MINRUN, MINLBK), until=(MAXRUN, MAXLBK))
__create_drawer(self, b64string, chan)
readIov(self, runlumi=None, dbg=False)
getFolderTag(self, folder, prefix, globalTag, api=None)
getPayload(self, runlumi=None, dbg=False)
__init__(self, db, folder='', tag='', run=None, lumi=0, modmin=0, modmax=275, copyBlob=False)
__checkIov(self, runlumi)
getDrawer(self, ros, mod, runlumi=None, dbg=False, useDefault=True)
__getIov(self, runlumi, dbg=False)
dumpIovs(self, iovList, rosmin, rosmax, drawermin, drawermax, option=1, comment=False, usenames=True)
getComment(self, runlumi=None, split=False)
getBlob(self, ros, mod, runlumi=None, dbg=False)
__make_blob(self, string)
__runlumi2iov(self, runlumi)
getDefault(self, ros, drawer)
getDrawer(self, ros, drawer, calibDrawerTemplate=None)
register(self, since=(MINRUN, MINLBK), tag="", chan=-1, payload=None)
zeroBlob(self, ros, drawer)
setComment(self, author, comment=None)
getComment(self, split=False)
__init__(self, db, folderPath, calibDrawerType, payload=None)
create_container_with_blob(self, jdata, name=None, type=chai.Type.String)
T * get(TKey *tobj)
get a TObject* from a TKey* (why can't a TObject be a TKey?)
std::vector< std::string > split(const std::string &s, const std::string &t=":")
getDrawerIdx(ros, drawer)