Classes
class	cfile
class	logFile

Functions
	getFolderList (dbstring, retdb=False, cleanlist=False)
	chmkDir (path)
	get_latest_run ()
	athListFormat (vals)
	printAndRun (cmd, outlogpath=None, runNow=True)
	run_merge (insqlite, inkeys, outsqlite, outpool, Ncoll=0, poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True)
	run_fillofcphase (phases_txt, outkey="LArSCOFCPhase", default_phase=22, folder="/LAR/ElecCalibOflSC/OFCBin/PhysShift", tag="LARElecCalibOflSCOFCBinPhysShift-10", outsql="SCOFCPhase.db", outpool="SC_OFC_Phase_10.pool.root", poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True)
	run_ofcpick (insqlite, outsqlite, phase_sql, run, BCsnapshotDB, outpdir="./", outrdir="./", outname="Picked_phase24052024", tag="LARElecCalibOflSCOFCBinPhysShift-10", Ncoll=0, poolcat="mergedPoolCat.xml", isPhys=True, outlogpath=None, runNow=True)
	run_toCoolInline (mergedDB, outDB, infolders="ConvertToInlineSC", globalTag="LARCALIB-RUN2-00", poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True)
	poolCatalog (poolDir, catalog=None, runNow=True)
	folderNamesTags (sql, mustr="mu-60", dbname="CONDBR2", verbose=False)

Variables
dict	folderInfo = {}
	parser = argparse.ArgumentParser()
	type
	str
	help
	dest
	default
	int
	nargs
	None
	action
	args = parser.parse_args()
	runNow = not args.dryRun
str	outtag = f"mu{args.Ncoll}_{datetime.datetime.now().strftime('%y%m%d')}"
str	logPath = f"{args.logDir}/{args.inputLog}"
bool	doPicking = True
	theLog = logFile(logPath)
	insqlite = theLog.sqldbs
	bcsnapshots = theLog.bcsnapshots
	outdir
str	outdir_root = f"{args.outdir}/rootFiles"
str	outdir_pool = f"{args.outdir}/poolFiles"
str	outdir_logs = f"{args.outdir}/logs"
str	mergedDB = f"{outdir_pool}/mergeSC_{outtag}.db"
str	mergedPool = f"{outdir_pool}/merged_SC_{outtag}.pool.root"
str	finalDB_flat = f"{outdir_pool}/freshConstants_1.5_{outtag}.db"
str	poolcat = f"{outdir_pool}/mergedPoolCat.xml"
str	phase_pool = f"{outdir_pool}/SC_OFC_Phase_10_{outtag}.pool.root"
	pd
	inkeys
	folders
	outsqlite
	outpool
	Ncoll
	outlogpath
str	phaseDBname = f"SCOFCPhase_{outtag}.db"
	phase_db
	phases_txt
	phase_txt
	outkey
	default_phase
	folder
	tag
	outsql
	phase_sql
	run
	BCsnapshotDB
	outpdir
	outrdir
	outname
	isPhys
	foldersTags = folderNamesTags(mergedDB, mustr="", verbose=True)
str	folderScript = f"{args.outdir}/ConvertToInlineSC"
list	lines = ["inputFolders=[]"]
	theFolder = f
	theTag = foldersTags[f]
dict	theKey = folderInfo[f]["key"]
dict	theClassType = folderInfo[f]["classtype"]
str	theline = f'inputFolders.append(("{theFolder}","{theTag}","{theKey}","{theClassType}"))'
	infolders
	outDB
	globalTag

Function Documentation

◆ athListFormat()

makeDTCalibBlob_pickPhase.athListFormat ( vals )

Definition at line 144 of file makeDTCalibBlob_pickPhase.py.

def athListFormat(vals):
    if isinstance(vals, list):
        vals = ",".join(vals)
    return vals
 
 

◆ chmkDir()

makeDTCalibBlob_pickPhase.chmkDir ( path )

Definition at line 68 of file makeDTCalibBlob_pickPhase.py.

def chmkDir( path ):
    os.umask(0)
    try:
        print( "mkdir "+path )
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise
        pass
 

◆ folderNamesTags()

makeDTCalibBlob_pickPhase.folderNamesTags	(	sql,
		mustr = "mu-60",
		dbname = "CONDBR2",
		verbose = False )

Definition at line 232 of file makeDTCalibBlob_pickPhase.py.

def folderNamesTags(sql, mustr="mu-60", dbname="CONDBR2", verbose=False):
    dbstring=f"sqlite://;schema={sql};dbname={dbname}"
    folderlist, db = getFolderList(dbstring, retdb=True, cleanlist=True)
    ftags = {}
    for f in folderlist:
        try:
            myfolder=db.getFolder(f)
            # list tags in folder
            if verbose: print("*"*20)
            if verbose: print(f"Reading folder {f}")
            if verbose: print("*"*20)
            if verbose: print("Checking tags...")
            taglist=myfolder.listTags()
            if verbose: print( f"{len(taglist)} Tags in folder:" )
            if verbose:
                for i in taglist: print( i )
            thetag = ""
            if len(taglist) > 0:
                taglist = [ str(t) for t in taglist ]
                if mustr != "":
                    posstag = [ t for t in taglist if mustr in t ]
                else:
                    posstag = [ t for t in taglist if "mu-" not in t ]
                if len(posstag) == 1:
                    thetag = posstag[0]
                else:
                    if verbose: print(len(posstag), "Choosing first tag")
                    thetag = taglist[0]
            if verbose: print(f"Will use tag {thetag}")
            ftags[f] = thetag
 
        except Exception as e:
            continue
    return ftags
    

◆ get_latest_run()

makeDTCalibBlob_pickPhase.get_latest_run ( )

Definition at line 78 of file makeDTCalibBlob_pickPhase.py.

def get_latest_run():
    dqmsite = "atlasdqm.cern.ch"
    dqmpassfile = "/afs/cern.ch/user/l/larmon/public/atlasdqmpass.txt"
    if not os.path.isfile(dqmpassfile):
        return 555555
    dqmpass = None
    dqmapi = None
    with open(dqmpassfile, "r") as f:
        dqmpass = f.readlines()[0].strip()
        if ":" not in dqmpass:
            print("Problem reading dqmpass")
            sys.exit()
        try:
            dqmapi = None #xmlrpc.client.ServerProxy("https://"+dqmpass+"@"+dqmsite)
        except:
            print("Failed to connect to atlasdqm, therefore giving a fake run number 555555")
            return 555555
    if dqmapi is not None:
        return dqmapi.get_latest_run()
    else:
        return 555555
 
    

◆ getFolderList()

makeDTCalibBlob_pickPhase.getFolderList	(	dbstring,
		retdb = False,
		cleanlist = False )

Definition at line 29 of file makeDTCalibBlob_pickPhase.py.

def getFolderList(dbstring, retdb=False, cleanlist=False):
    # get database service and open database
    dbSvc=cool.DatabaseSvcFactory.databaseService()
    # database accessed via logical name, can also use physical connections
    try:
        db=dbSvc.openDatabase(dbstring)
    except Exception as e:
        print(f"Problem opening database {e}")
        sys.exit(-1)
    print(f"Opened database {dbstring}")
    # now list the folders
    folderlist= [ str(s) for s in db.listAllNodes()]
    if cleanlist:
        # Careful... some folders have similar names e.g. OFC and OFCCali
        # folderlist = [j for j in folderlist if all(j not in k if j!=k else True for k in folderlist)]
        folderlist = [f for f in folderlist if f.count("/") >= 3]
 
    if retdb:
        return folderlist, db
    else:
        return folderlist
 
 
#from mp import parallel_exec 
#sys.path.append("/afs/cern.ch/user/l/larmon/public/prod/LArPage1/makeJIRA")
#import jiraComment as jc
 

◆ poolCatalog()

makeDTCalibBlob_pickPhase.poolCatalog	(	poolDir,
		catalog = None,
		runNow = True )

Definition at line 211 of file makeDTCalibBlob_pickPhase.py.

def poolCatalog(poolDir, catalog=None, runNow=True):
    print(f"** Adding pool files from {poolDir} with pool_insertFileToCatalog **")
    for f in [os.path.join(poolDir, f) for f in os.listdir(poolDir) if f.endswith(".pool.root")]:
        cmd1 = "pool_extractFileIdentifier "+f
        process = subprocess.Popen(cmd1,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out,err = process.communicate()
        retcode = process.returncode
        ident=out.decode('ascii').split(" ")[0]        
        cmd=f"pool_insertFileToCatalog.py {f}"
        if catalog is not None:
            cmd += f" --catalog='xmlcatalog_file:{catalog}'"
        print(f"---- {cmd} ({ident})")
        if runNow:
            process = subprocess.Popen(cmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out,err = process.communicate()
            retcode = process.returncode
            if retcode != 0:
                print(err.decode('ascii')) #print("ERROR: inserting file to catalogue failed")
            #print(out.decode('ascii'))
            #print(err.decode('ascii'))
 

◆ printAndRun()

makeDTCalibBlob_pickPhase.printAndRun	(	cmd,
		outlogpath = None,
		runNow = True )

Definition at line 150 of file makeDTCalibBlob_pickPhase.py.

def printAndRun(cmd, outlogpath=None, runNow=True):
    print("**** RUNNING THE FOLLOWING COMMAND ****")
    print(cmd)
    print("***************************************")
    if runNow:
        process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out,err = process.communicate()        
        retcode = process.returncode
        print("**** Return code:", retcode, "*"*10)
        log = None
        if outlogpath is not None:
            print(f"See log: {outlogpath}")
            log = cfile(outlogpath)
            log.wl("~~~~~~~~ OUTPUT ~~~~~~~~")
            log.wl(out.decode('ascii'))
            log.wl("~~~~~~~~~~~~~~~~~~~~~~~~")
            log.wl("~~~~~~~~ ERROR ~~~~~~~~")
            log.wl(err.decode('ascii'))
            log.wl("~~~~~~~~~~~~~~~~~~~~~~~~")
        else:
            print("~~~~~~~~ OUTPUT ~~~~~~~~")
            print(out.decode('ascii'))
            print("~~~~~~~~~~~~~~~~~~~~~~~~")
            print("~~~~~~~~ ERROR ~~~~~~~~")
            print(err.decode('ascii'))
            print("~~~~~~~~~~~~~~~~~~~~~~~~")
        if retcode != 0:
            print("!!!!!!!! DID NOT TERMINATE SUCCESSFULLY... exiting")
            sys.exit()
 

◆ run_fillofcphase()

makeDTCalibBlob_pickPhase.run_fillofcphase	(	phases_txt,
		outkey = "LArSCOFCPhase",
		default_phase = 22,
		folder = "/LAR/ElecCalibOflSC/OFCBin/PhysShift",
		tag = "LARElecCalibOflSCOFCBinPhysShift-10",
		outsql = "SCOFCPhase.db",
		outpool = "SC_OFC_Phase_10.pool.root",
		poolcat = "mergedPoolCat.xml",
		outlogpath = None,
		runNow = True )

Step 2 *if needed* - make an sql file from picked OFC phase txt file

Definition at line 187 of file makeDTCalibBlob_pickPhase.py.

def run_fillofcphase(phases_txt, outkey="LArSCOFCPhase", default_phase=22, folder="/LAR/ElecCalibOflSC/OFCBin/PhysShift", tag="LARElecCalibOflSCOFCBinPhysShift-10", outsql="SCOFCPhase.db", outpool="SC_OFC_Phase_10.pool.root", poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True):
    ''' Step 2 *if needed* - make an sql file from picked OFC phase txt file '''
    cmd = f"LArNewCalib_FillOFCPhase.py --infile {phases_txt} --outkey {outkey} --isSC --hasid --default {default_phase} --folder {folder} --tag {tag} --outsql {outsql} --outp {outpool} --poolcat {poolcat}"
    printAndRun(cmd,outlogpath, runNow=runNow)
 
 

◆ run_merge()

makeDTCalibBlob_pickPhase.run_merge	(	insqlite,
		inkeys,
		outsqlite,
		outpool,
		Ncoll = 0,
		poolcat = "mergedPoolCat.xml",
		outlogpath = None,
		runNow = True )

Step 1 of the merging.

Definition at line 180 of file makeDTCalibBlob_pickPhase.py.

def run_merge(insqlite, inkeys, outsqlite, outpool, Ncoll=0, poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True):
    ''' Step 1 of the merging. '''
    insqlite = athListFormat(insqlite)
    inkeys = athListFormat(inkeys)
    cmd = f"python -m LArCalibProcessing.LArNewCalib_MergeDB --insqlite {insqlite} --inkeys {inkeys} --outsqlite {outsqlite} --poolfile {outpool} --isSC --poolcat {poolcat} --Ncoll {Ncoll}"
    printAndRun(cmd,outlogpath, runNow=runNow)
 

◆ run_ofcpick()

makeDTCalibBlob_pickPhase.run_ofcpick	(	insqlite,
		outsqlite,
		phase_sql,
		run,
		BCsnapshotDB,
		outpdir = "./",
		outrdir = "./",
		outname = "Picked_phase24052024",
		tag = "LARElecCalibOflSCOFCBinPhysShift-10",
		Ncoll = 0,
		poolcat = "mergedPoolCat.xml",
		isPhys = True,
		outlogpath = None,
		runNow = True )

Step 3, picking the phase in the DB

Definition at line 193 of file makeDTCalibBlob_pickPhase.py.

def run_ofcpick(insqlite, outsqlite, phase_sql, run, BCsnapshotDB, outpdir="./", outrdir="./", outname="Picked_phase24052024", tag="LARElecCalibOflSCOFCBinPhysShift-10", Ncoll=0, poolcat="mergedPoolCat.xml", isPhys=True, outlogpath=None, runNow=True):  # insqlite  mergeSC.db, outssqlite mergeSCOnl_1.5_phase24052024.db , subdet Picked_phase24052024, phase_sql SCOFCPhase.d)
    ''' Step 3, picking the phase in the DB '''
    cmd=f"LArNewCalib_PhysOFCPhasePicker.py --run {run} -b {BCsnapshotDB} --insqlite {insqlite} --poolcat {poolcat} --outsqlite {outsqlite} --isSC --outpdir {outpdir} --outrdir {outrdir} --subdet {outname} --ofcphasetag {tag} --ofcphasesqlite {phase_sql} --Ncoll {Ncoll}" 
    if not isPhys:
        cmd += " --isCalib --outprefix LArOFCCaliOnePhase"
        
    printAndRun(cmd,outlogpath, runNow=runNow)
 
 

◆ run_toCoolInline()

makeDTCalibBlob_pickPhase.run_toCoolInline	(	mergedDB,
		outDB,
		infolders = "ConvertToInlineSC",
		globalTag = "LARCALIB-RUN2-00",
		poolcat = "mergedPoolCat.xml",
		outlogpath = None,
		runNow = True )

Step 4: flattening the DB

Definition at line 202 of file makeDTCalibBlob_pickPhase.py.

def run_toCoolInline(mergedDB, outDB, infolders="ConvertToInlineSC", globalTag="LARCALIB-RUN2-00", poolcat="mergedPoolCat.xml", outlogpath=None, runNow=True):# outDB freshConstantsOnl_1.5.db
    ''' Step 4: flattening the DB '''
    #  connecting all folder level tags to some new global tag..... In calibration processing it is used to be able to define some calib. global tag in sqlite file, and then job do not need to know all individual folder level tags, athena automaticaly uses the one connected to defined global tag...
    cmd=f"/afs/cern.ch/user/l/larcalib/LArDBTools/python/BuildTagHierarchy.py {mergedDB} {globalTag}"
    printAndRun(cmd,outlogpath, runNow=runNow)
    cmd=f"LArCalib_ToCoolInlineConfig.py --insqlite {mergedDB} --infolders {infolders} --poolcat {poolcat} --outsqlite {outDB} --isSC"
    printAndRun(cmd,outlogpath, runNow=runNow)
 
 