Classes
class	cfile

class	logFile

Functions
def	getFolderList (dbstring, retdb=False, cleanlist=False)

def	chmkDir (path)

def	get_latest_run ()

def	athListFormat (vals)

def	printAndRun (cmd, outlogpath=None, runNow=True)

def	run_merge (insqlite, inkeys, outsqlite, outpool, Ncoll=0, poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True)

def	run_fillofcphase (phases_txt, outkey="LArSCOFCPhase", default_phase=22, folder="/LAR/ElecCalibOflSC/OFCBin/PhysShift", tag="LARElecCalibOflSCOFCBinPhysShift-10", outsql="SCOFCPhase.db", outpool="SC_OFC_Phase_10.pool.root", poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True)

def	run_ofcpick (insqlite, outsqlite, phase_sql, run, BCsnapshotDB, outpdir="./", outrdir="./", outname="Picked_phase24052024", tag="LARElecCalibOflSCOFCBinPhysShift-10", Ncoll=0, poolcat="mergedPoolCat.xml", isPhys=True, outlogpath=None, runNow=True)

def	run_toCoolInline (mergedDB, outDB, infolders="ConvertToInlineSC", globalTag="LARCALIB-RUN2-00", poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True)

def	poolCatalog (poolDir, catalog=None, runNow=True)

def	folderNamesTags (sql, mustr="mu-60", dbname="CONDBR2", verbose=False)

Variables
dictionary	folderInfo = {}

	parser = argparse.ArgumentParser()

	type

	str

	help

	dest

	default

	int

	nargs

	None

	action

	args = parser.parse_args()

	runNow = not args.dryRun

string	outtag = f"mu{args.Ncoll}_{datetime.datetime.now().strftime('%y%m%d')}"

string	logPath = f"{args.logDir}/{args.inputLog}"

bool	doPicking = True

	theLog = logFile(logPath)

	insqlite = theLog.sqldbs

	bcsnapshots = theLog.bcsnapshots

	outdir

string	outdir_root = f"{args.outdir}/rootFiles"

string	outdir_pool = f"{args.outdir}/poolFiles"

string	outdir_logs = f"{args.outdir}/logs"

string	mergedDB = f"{outdir_pool}/mergeSC_{outtag}.db"

string	mergedPool = f"{outdir_pool}/merged_SC_{outtag}.pool.root"

string	finalDB_flat = f"{outdir_pool}/freshConstants_1.5_{outtag}.db"

string	poolcat = f"{outdir_pool}/mergedPoolCat.xml"

string	phase_pool = f"{outdir_pool}/SC_OFC_Phase_10_{outtag}.pool.root"

	pd

	inkeys

	folders

	outsqlite

	outpool

	Ncoll

	outlogpath

string	phaseDBname = f"SCOFCPhase_{outtag}.db"

	phase_db

	phases_txt

	phase_txt

	outkey

	default_phase

	folder

	tag

	outsql

	phase_sql

	run

	BCsnapshotDB

	outpdir

	outrdir

	outname

	isPhys

def	foldersTags = folderNamesTags(mergedDB, mustr="", verbose=True)

string	folderScript = f"{args.outdir}/ConvertToInlineSC"

list	lines = ["inputFolders=[]"]

	theFolder = f

def	theTag = foldersTags[f]

dictionary	theKey = folderInfo[f]["key"]

dictionary	theClassType = folderInfo[f]["classtype"]

string	theline = f'inputFolders.append(("{theFolder}","{theTag}","{theKey}","{theClassType}"))'

	infolders

	outDB

	globalTag

Function Documentation

◆ athListFormat()

def makeDTCalibBlob_pickPhase.athListFormat ( vals )

Definition at line 144 of file makeDTCalibBlob_pickPhase.py.

 def athListFormat(vals):
     if isinstance(vals, list):
         vals = ",".join(vals)
     return vals
  
  

◆ chmkDir()

def makeDTCalibBlob_pickPhase.chmkDir ( path )

Definition at line 68 of file makeDTCalibBlob_pickPhase.py.

 def chmkDir( path ):
     os.umask(0)
     try:
         print( "mkdir "+path )
         os.makedirs(path)
     except OSError as exception:
         if exception.errno != errno.EEXIST:
             raise
         pass
  

◆ folderNamesTags()

def makeDTCalibBlob_pickPhase.folderNamesTags	(	sql,
		mustr = `"mu-60"`,
		dbname = `"CONDBR2"`,
		verbose = `False`
	)

Definition at line 232 of file makeDTCalibBlob_pickPhase.py.

 def folderNamesTags(sql, mustr="mu-60", dbname="CONDBR2", verbose=False):
     dbstring=f"sqlite://;schema={sql};dbname={dbname}"
     folderlist, db = getFolderList(dbstring, retdb=True, cleanlist=True)
     ftags = {}
     for f in folderlist:
         try:
             myfolder=db.getFolder(f)
             # list tags in folder
             if verbose: print("*"*20)
             if verbose: print(f"Reading folder {f}")
             if verbose: print("*"*20)
             if verbose: print("Checking tags...")
             taglist=myfolder.listTags()
             if verbose: print( f"{len(taglist)} Tags in folder:" )
             if verbose:
                 for i in taglist: print( i )
             thetag = ""
             if len(taglist) > 0:
                 taglist = [ str(t) for t in taglist ]
                 if mustr != "":
                     posstag = [ t for t in taglist if mustr in t ]
                 else:
                     posstag = [ t for t in taglist if "mu-" not in t ]
                 if len(posstag) == 1:
                     thetag = posstag[0]
                 else:
                     if verbose: print(len(posstag), "Choosing first tag")
                     thetag = taglist[0]
             if verbose: print(f"Will use tag {thetag}")
             ftags[f] = thetag
  
         except Exception as e:
             continue
     return ftags
     

◆ get_latest_run()

def makeDTCalibBlob_pickPhase.get_latest_run ( )

Definition at line 78 of file makeDTCalibBlob_pickPhase.py.

 def get_latest_run():
     dqmsite = "atlasdqm.cern.ch"
     dqmpassfile = "/afs/cern.ch/user/l/larmon/public/atlasdqmpass.txt"
     if not os.path.isfile(dqmpassfile):
         return 555555
     dqmpass = None
     dqmapi = None
     with open(dqmpassfile, "r") as f:
         dqmpass = f.readlines()[0].strip()
         if ":" not in dqmpass:
             print("Problem reading dqmpass")
             sys.exit()
         try:
             dqmapi = None #xmlrpc.client.ServerProxy("https://"+dqmpass+"@"+dqmsite)
         except:
             print("Failed to connect to atlasdqm, therefore giving a fake run number 555555")
             return 555555
     if dqmapi is not None:
         return dqmapi.get_latest_run()
     else:
         return 555555
  
     

◆ getFolderList()

def makeDTCalibBlob_pickPhase.getFolderList	(	dbstring,
		retdb = `False`,
		cleanlist = `False`
	)

Definition at line 29 of file makeDTCalibBlob_pickPhase.py.

 def getFolderList(dbstring, retdb=False, cleanlist=False):
     # get database service and open database
     dbSvc=cool.DatabaseSvcFactory.databaseService()
     # database accessed via logical name, can also use physical connections
     try:
         db=dbSvc.openDatabase(dbstring)
     except Exception as e:
         print(f"Problem opening database {e}")
         sys.exit(-1)
     print(f"Opened database {dbstring}")
     # now list the folders
     folderlist= [ str(s) for s in db.listAllNodes()]
     if cleanlist:
         # Careful... some folders have similar names e.g. OFC and OFCCali
         # folderlist = [j for j in folderlist if all(j not in k if j!=k else True for k in folderlist)]
         folderlist = [f for f in folderlist if f.count("/") >= 3]
  
     if retdb:
         return folderlist, db
     else:
         return folderlist
  
  
 #from mp import parallel_exec 
 #sys.path.append("/afs/cern.ch/user/l/larmon/public/prod/LArPage1/makeJIRA")
 #import jiraComment as jc
  

◆ poolCatalog()

def makeDTCalibBlob_pickPhase.poolCatalog	(	poolDir,
		catalog = `None`,
		runNow = `True`
	)

Definition at line 211 of file makeDTCalibBlob_pickPhase.py.

 def poolCatalog(poolDir, catalog=None, runNow=True):
     print(f"** Adding pool files from {poolDir} with pool_insertFileToCatalog **")
     for f in [os.path.join(poolDir, f) for f in os.listdir(poolDir) if f.endswith(".pool.root")]:
         cmd1 = "pool_extractFileIdentifier "+f
         process = subprocess.Popen(cmd1,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         out,err = process.communicate()
         retcode = process.returncode
         ident=out.decode('ascii').split(" ")[0]        
         cmd=f"pool_insertFileToCatalog.py {f}"
         if catalog is not None:
             cmd += f" --catalog='xmlcatalog_file:{catalog}'"
         print(f"---- {cmd} ({ident})")
         if runNow:
             process = subprocess.Popen(cmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             out,err = process.communicate()
             retcode = process.returncode
             if retcode != 0:
                 print(err.decode('ascii')) #print("ERROR: inserting file to catalogue failed")
             #print(out.decode('ascii'))
             #print(err.decode('ascii'))
  

◆ printAndRun()

def makeDTCalibBlob_pickPhase.printAndRun	(	cmd,
		outlogpath = `None`,
		runNow = `True`
	)

Definition at line 150 of file makeDTCalibBlob_pickPhase.py.

 def printAndRun(cmd, outlogpath=None, runNow=True):
     print("**** RUNNING THE FOLLOWING COMMAND ****")
     print(cmd)
     print("***************************************")
     if runNow:
         process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         out,err = process.communicate()        
         retcode = process.returncode
         print("**** Return code:", retcode, "*"*10)
         log = None
         if outlogpath is not None:
             print(f"See log: {outlogpath}")
             log = cfile(outlogpath)
             log.wl("~~~~~~~~ OUTPUT ~~~~~~~~")
             log.wl(out.decode('ascii'))
             log.wl("~~~~~~~~~~~~~~~~~~~~~~~~")
             log.wl("~~~~~~~~ ERROR ~~~~~~~~")
             log.wl(err.decode('ascii'))
             log.wl("~~~~~~~~~~~~~~~~~~~~~~~~")
         else:
             print("~~~~~~~~ OUTPUT ~~~~~~~~")
             print(out.decode('ascii'))
             print("~~~~~~~~~~~~~~~~~~~~~~~~")
             print("~~~~~~~~ ERROR ~~~~~~~~")
             print(err.decode('ascii'))
             print("~~~~~~~~~~~~~~~~~~~~~~~~")
         if retcode != 0:
             print("!!!!!!!! DID NOT TERMINATE SUCCESSFULLY... exiting")
             sys.exit()
  

◆ run_fillofcphase()

def makeDTCalibBlob_pickPhase.run_fillofcphase	(	phases_txt,
		outkey = `"LArSCOFCPhase"`,
		default_phase = `22`,
		folder = `"/LAR/ElecCalibOflSC/OFCBin/PhysShift"`,
		tag = `"LARElecCalibOflSCOFCBinPhysShift-10"`,
		outsql = `"SCOFCPhase.db"`,
		outpool = `"SC_OFC_Phase_10.pool.root"`,
		poolcat = `"mergedPoolCat.xml"`,
		outlogpath = `None`,
		runNow = `True`
	)

Step 2 *if needed* - make an sql file from picked OFC phase txt file

Definition at line 187 of file makeDTCalibBlob_pickPhase.py.

 def run_fillofcphase(phases_txt, outkey="LArSCOFCPhase", default_phase=22, folder="/LAR/ElecCalibOflSC/OFCBin/PhysShift", tag="LARElecCalibOflSCOFCBinPhysShift-10", outsql="SCOFCPhase.db", outpool="SC_OFC_Phase_10.pool.root", poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True):
     ''' Step 2 *if needed* - make an sql file from picked OFC phase txt file '''
     cmd = f"LArNewCalib_FillOFCPhase.py --infile {phases_txt} --outkey {outkey} --isSC --hasid --default {default_phase} --folder {folder} --tag {tag} --outsql {outsql} --outp {outpool} --poolcat {poolcat}"
     printAndRun(cmd,outlogpath, runNow=runNow)
  
  

◆ run_merge()

def makeDTCalibBlob_pickPhase.run_merge	(	insqlite,
		inkeys,
		outsqlite,
		outpool,
		Ncoll = `0`,
		poolcat = `"mergedPoolCat.xml"`,
		outlogpath = `None`,
		runNow = `True`
	)

Step 1 of the merging.

Definition at line 180 of file makeDTCalibBlob_pickPhase.py.

 def run_merge(insqlite, inkeys, outsqlite, outpool, Ncoll=0, poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True):
     ''' Step 1 of the merging. '''
     insqlite = athListFormat(insqlite)
     inkeys = athListFormat(inkeys)
     cmd = f"python -m LArCalibProcessing.LArNewCalib_MergeDB --insqlite {insqlite} --inkeys {inkeys} --outsqlite {outsqlite} --poolfile {outpool} --isSC --poolcat {poolcat} --Ncoll {Ncoll}"
     printAndRun(cmd,outlogpath, runNow=runNow)
  

◆ run_ofcpick()

def makeDTCalibBlob_pickPhase.run_ofcpick	(	insqlite,
		outsqlite,
		phase_sql,
		run,
		BCsnapshotDB,
		outpdir = `"./"`,
		outrdir = `"./"`,
		outname = `"Picked_phase24052024"`,
		tag = `"LARElecCalibOflSCOFCBinPhysShift-10"`,
		Ncoll = `0`,
		poolcat = `"mergedPoolCat.xml"`,
		isPhys = `True`,
		outlogpath = `None`,
		runNow = `True`
	)

Step 3, picking the phase in the DB

Definition at line 193 of file makeDTCalibBlob_pickPhase.py.

 def run_ofcpick(insqlite, outsqlite, phase_sql, run, BCsnapshotDB, outpdir="./", outrdir="./", outname="Picked_phase24052024", tag="LARElecCalibOflSCOFCBinPhysShift-10", Ncoll=0, poolcat="mergedPoolCat.xml", isPhys=True, outlogpath=None, runNow=True):  # insqlite  mergeSC.db, outssqlite mergeSCOnl_1.5_phase24052024.db , subdet Picked_phase24052024, phase_sql SCOFCPhase.d)
     ''' Step 3, picking the phase in the DB '''
     cmd=f"LArNewCalib_PhysOFCPhasePicker.py --run {run} -b {BCsnapshotDB} --insqlite {insqlite} --poolcat {poolcat} --outsqlite {outsqlite} --isSC --outpdir {outpdir} --outrdir {outrdir} --subdet {outname} --ofcphasetag {tag} --ofcphasesqlite {phase_sql} --Ncoll {Ncoll}" 
     if not isPhys:
         cmd += " --isCalib --outprefix LArOFCCaliOnePhase"
         
     printAndRun(cmd,outlogpath, runNow=runNow)
  
  

◆ run_toCoolInline()

def makeDTCalibBlob_pickPhase.run_toCoolInline	(	mergedDB,
		outDB,
		infolders = `"ConvertToInlineSC"`,
		globalTag = `"LARCALIB-RUN2-00"`,
		poolcat = `"mergedPoolCat.xml"`,
		outlogpath = `None`,
		runNow = `True`
	)

Step 4: flattening the DB

Definition at line 202 of file makeDTCalibBlob_pickPhase.py.

 def run_toCoolInline(mergedDB, outDB, infolders="ConvertToInlineSC", globalTag="LARCALIB-RUN2-00", poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True):# outDB freshConstantsOnl_1.5.db
     ''' Step 4: flattening the DB '''
     #  connecting all folder level tags to some new global tag..... In calibration processing it is used to be able to define some calib. global tag in sqlite file, and then job do not need to know all individual folder level tags, athena automaticaly uses the one connected to defined global tag...
     cmd=f"/afs/cern.ch/user/l/larcalib/LArDBTools/python/BuildTagHierarchy.py {mergedDB} {globalTag}"
     printAndRun(cmd,outlogpath, runNow=runNow)
     cmd=f"LArCalib_ToCoolInlineConfig.py --insqlite {mergedDB} --infolders {infolders} --poolcat {poolcat} --outsqlite {outDB} --isSC"
     printAndRun(cmd,outlogpath, runNow=runNow)
  
  