python.GENtoEVGEN_Skeleton Namespace Reference

Functions
	_mk_symlink (srcfile, dstfile)
	_find_unique_file (pattern)
	_merge_lhe_files (listOfFiles, outputFile)
	_handle_input_files (generators, flags)
	_validate_sample_properties (sample)
	setupSample (runArgs, flags)
	checkBlackList (cache, generatorName, checkType)
	fromRunArgs (runArgs)

Variables
	jobPropertiesDisallowed
	evgenLog = logging.getLogger("Gen_tf")

Detailed Description

Functionality core of the Gen_tf transform

Function Documentation

_find_unique_file()

python.GENtoEVGEN_Skeleton._find_unique_file ( pattern )

protected

Definition at line 38 of file GENtoEVGEN_Skeleton.py.

def _find_unique_file(pattern):
    import glob
    files = glob.glob(pattern)
    # Check that there is exactly 1 match
    if not files:
        raise RuntimeError(f"No {pattern} file found")
    elif len(files) > 1:
        raise RuntimeError(f"More than one {pattern} file found")
    return files[0]
 
 
# This function merges a list of input LHE files into one output file.
# The header is taken from the first file, but the number of events is
# updated to equal the total number of events in all input files.

_handle_input_files()

python.GENtoEVGEN_Skeleton._handle_input_files ( generators, flags )

protected

Definition at line 99 of file GENtoEVGEN_Skeleton.py.

def _handle_input_files(generators, flags):
    from GeneratorConfig.GenConfigHelpers import gens_lhef
 
    # Name of event files produced by various generators.
    events_file_map = {
        "Alpgen": "alpgen.unw_events",
        "Protos": "protos.events",
        "ProtosLHEF": "protoslhef.events",
        "BeamHaloGenerator": "beamhalogen.events",
        "HepMCAscii": "events.hepmc",
        "ReadMcAscii": "events.hepmc",
    }
    eventsFile = None
    for gen_name, out_file in events_file_map.items():
        if gen_name in generators:
            eventsFile = out_file
            break
    if eventsFile is None:
        if gens_lhef(generators):
            eventsFile = "events.lhe"
        else:
            raise RuntimeError(f"Unknown type of ME generator: {generators}")
 
    genInputFiles = [f.strip() for f in flags.Generator.inputGeneratorFile.split(",") if f.strip()]
    if not genInputFiles:
        raise RuntimeError("Generator.inputGeneratorFile is empty while input handling is requested")
 
    def _input_root(path, keep_suffix_after_underscore=False):
        fname = os.path.basename(path)
        if any(ext in fname for ext in (".tar.", ".tgz", ".gz")):
            return re.split(r"\.tar\.|\.tgz|\.gz", fname, maxsplit=1)[0]
        parts = fname.split("._", 1)
        if keep_suffix_after_underscore and len(parts) > 1:
            return parts[0] + "._" + parts[1].split(".", 1)[0]
        return parts[0]
 
    # If there is a single file, make a symlink. If multiple files, merge them into one output eventsFile.
    if len(genInputFiles) == 1:
        inputroot = _input_root(genInputFiles[0], keep_suffix_after_underscore=False)
        if inputroot.endswith(".events"):
            inputroot = inputroot[:-7]
        realEventsFile = _find_unique_file(f"*{inputroot}.*ev*ts")
        _mk_symlink(realEventsFile, eventsFile)
        return
 
    allFiles = []
    for file in genInputFiles:
        # Since we can have multiple files from the same task, include more of the filename
        # to make the lookup unique in the plain-file case.
        inputroot = _input_root(file, keep_suffix_after_underscore=True)
        evgenLog.info("inputroot = %s", inputroot)
        realEventsFile = _find_unique_file(f"*{inputroot}.*ev*ts")
        # The only input format where merging is permitted is LHE.
        with open(realEventsFile, "r") as f:
            first_line = f.readline()
            if "LesHouche" not in first_line:
                raise RuntimeError(f"{realEventsFile} is NOT a LesHouche file")
        allFiles.append(realEventsFile)
    _merge_lhe_files(allFiles, eventsFile)
 
    # counting the number of events in LHE input
    eventsInLHE = 0
    with open(eventsFile) as f:
        for line in f:
           eventsInLHE += line.count('/event')
    return eventsInLHE
 
 
# Helper function to validate and set sample properties

_merge_lhe_files()

python.GENtoEVGEN_Skeleton._merge_lhe_files ( listOfFiles, outputFile )

protected

Definition at line 52 of file GENtoEVGEN_Skeleton.py.

def _merge_lhe_files(listOfFiles, outputFile):
    if os.path.exists(outputFile):
        print("outputFile", outputFile, "already exists. Will rename to", outputFile + ".OLD")
        os.rename(outputFile, outputFile + ".OLD")
 
    total_events = 0
    for file in listOfFiles:
        with open(file, "r") as f:
            total_events += sum(1 for line in f if "</event>" in line)
 
    wrote_header = False
    with open(outputFile, "w") as output:
        for file in listOfFiles:
            inHeader = True
            header = ""
            print("*** Starting file", file)
            with open(file, "r") as infile:
                for line in infile:
                    # Reading first event signals that we are done with all header information.
                    if "<event" in line and inHeader:
                        inHeader = False
                        if not wrote_header:
                            wrote_header = True
                            output.write(header)
                        output.write(line)
                    # Each input file ends with "</LesHouchesEvents>". We only write it once at the end.
                    elif not inHeader and "</LesHouchesEvents>" not in line:
                        output.write(line)
 
                    if inHeader:
                        # Format for storing number of events differs in MG and Powheg.
                        if "nevents" in line:
                            # MG5 format is "n = nevents".
                            parts = line.split("=")
                            if parts:
                                line = line.replace(parts[0], str(total_events), 1)
                        elif "numevts" in line:
                            # Powheg format is "numevts n".
                            parts = line.split()
                            if len(parts) > 1:
                                line = line.replace(parts[1], str(total_events), 1)
                        header += line
 
        output.write("</LesHouchesEvents>\n")
 
 
# Helper for handling input files

_mk_symlink()

python.GENtoEVGEN_Skeleton._mk_symlink ( srcfile, dstfile )

protected

Definition at line 25 of file GENtoEVGEN_Skeleton.py.

def _mk_symlink(srcfile, dstfile):
    if dstfile:
        if os.path.exists(dstfile) and not os.path.samefile(dstfile, srcfile):
            os.remove(dstfile)
        if not os.path.exists(dstfile):
            evgenLog.info(f"Symlinking {srcfile} to {dstfile}")
            print (f"Symlinking {srcfile} to {dstfile}")
            os.symlink(srcfile, dstfile)
        else:
            evgenLog.debug(f"Symlinking: {dstfile} is already the same as {srcfile}")
 
 
# Helper functions for finding input file

_validate_sample_properties()

python.GENtoEVGEN_Skeleton._validate_sample_properties ( sample )

protected

Definition at line 168 of file GENtoEVGEN_Skeleton.py.

def _validate_sample_properties(sample):
    # Required fields with lightweight, explicit validators.
    required_rules = {
        "keywords": lambda v: isinstance(v, list) and len(v) > 0,
        "contact": lambda v: isinstance(v, list) and len(v) > 0,
        "nEventsPerJob": lambda v: v is not None,
    }
    for field, validator in required_rules.items():
        value = getattr(sample, field, None)
        if not validator(value):
            raise RuntimeError(f"self.{field} should be set in Sample(EvgenConfig)")
 
    input_files_per_job = getattr(sample, "inputFilesPerJob", 0)
    me_generator = getattr(sample, "MEgenerator", None)
 
    if input_files_per_job < 0:
        raise RuntimeError("self.inputFilesPerJob should be >= 0 in Sample(EvgenConfig)")
    if input_files_per_job > 0 and not me_generator:
        raise RuntimeError("self.MEgenerator should be set when self.inputFilesPerJob > 0 in Sample(EvgenConfig)")
    if input_files_per_job == 0 and me_generator:
        raise RuntimeError("self.MEgenerator should be empty when self.inputFilesPerJob == 0 in Sample(EvgenConfig)")
 
 
# Function that reads the jO and returns an instance of Sample(EvgenCAConfig)

checkBlackList()

python.GENtoEVGEN_Skeleton.checkBlackList	(		cache,
			generatorName,
			checkType )

Definition at line 276 of file GENtoEVGEN_Skeleton.py.

def checkBlackList(cache, generatorName, checkType) :
    isError = None
    fileName = "BlackList_caches.txt" if checkType == "black" else "PurpleList_generators.txt"
    with open(f"/cvmfs/atlas.cern.ch/repo/sw/Generators/MC16JobOptions/common/{fileName}") as bfile:
        for line in bfile.readlines():
            if not line.strip():
                continue
            # Bad caches
            badCache=line.split(',')[1].strip()
            # Bad generators
            badGens=line.split(',')[2].strip()
 
            used_gens = ','.join(generatorName)
            # Match Generator and release cache
            if cache==badCache and re.search(badGens,used_gens) is not None:
                if badGens=="": badGens="all generators"
                isError=f"{cache} is {checkType}-listed for {badGens}"
                return isError
    return isError
 
 
# Main function

fromRunArgs()

python.GENtoEVGEN_Skeleton.fromRunArgs

(

runArgs

)

Definition at line 298 of file GENtoEVGEN_Skeleton.py.

def fromRunArgs(runArgs):
    # print release information
    d = release_metadata()
    evgenLog.info("using release [%(project name)s-%(release)s] [%(platform)s] [%(nightly name)s/%(nightly release)s] -- built on [%(date)s]", d)
    athenaRel = d["release"]
 
    evgenLog.info("****************** STARTING EVENT GENERATION *****************")
 
    evgenLog.info("**** Transformation run arguments")
    evgenLog.info(runArgs)
 
    evgenLog.info("**** Setting-up configuration flags")
 
    from AthenaConfiguration.AllConfigFlags import initConfigFlags
    flags = initConfigFlags()
 
    from AthenaConfiguration.Enums import ProductionStep
    flags.Common.ProductionStep = ProductionStep.Generation
 
    # Convert run arguments to global athena flags
    from PyJobTransforms.CommonRunArgsToFlags import commonRunArgsToFlags
    commonRunArgsToFlags(runArgs, flags)
 
    # Convert generator-specific run arguments to global athena flags
    from GeneratorConfig.GeneratorConfigFlags import  generatorRunArgsToFlags
    generatorRunArgsToFlags(runArgs, flags)
 
    # convert arguments to flags
    flags.fillFromArgs()
 
    # Create an instance of the Sample(EvgenCAConfig) and update global flags accordingly
    sample = setupSample(runArgs, flags)
 
    # Setup the main flags
    flags.Exec.FirstEvent = runArgs.firstEvent
    # Max events should be not set, job stopping is handled by CountHepMC
    # using the RequestedOutput property
    flags.Exec.MaxEvents = -1
 
    if hasattr(runArgs, "inputEVNT_PreFile"):
        flags.Input.Files = runArgs.inputEVNT_PreFile
    else:
        flags.Input.Files = []
        flags.Input.RunNumbers = [flags.Generator.DSID]
        flags.Input.TimeStamps = [0]
 
    flags.Output.EVNTFileName = runArgs.outputEVNTFile
 
    flags.Beam.Energy = runArgs.ecmEnergy / 2 * GeV
 
    flags.PerfMon.doFastMonMT = True
    flags.PerfMon.doFullMonMT = True
 
    # Process pre-include
    processPreInclude(runArgs, flags)
 
    # Process pre-exec
    processPreExec(runArgs, flags)
 
    # Lock flags
    flags.lock()
 
    evgenLog.info("**** Configuration flags")
    if runArgs.VERBOSE:
        flags.dump()
    else:
        flags.dump("Generator.*")
 
    # Print various stuff
    evgenLog.info(".transform =                  Gen_tf")
    evgenLog.info(".platform = " + str(os.environ["BINARY_TAG"]))
 
    # Announce start of job configuration
    evgenLog.info("**** Configuring event generation")
 
    # Main object
    from AthenaConfiguration.MainServicesConfig import MainEvgenServicesCfg
    cfg = MainEvgenServicesCfg(flags, withSequences=True)
 
    # Input file handling (if needed)
    if flags.Input.Files:
        from AthenaPoolCnvSvc.PoolReadConfig import PoolReadCfg
        cfg.merge(PoolReadCfg(flags))
 
    # EventInfoCnvAlg
    from xAODEventInfoCnv.xAODEventInfoCnvConfig import EventInfoCnvAlgCfg
    cfg.merge(EventInfoCnvAlgCfg(flags, disableBeamSpot=True, xAODKey="TMPEvtInfo"),
                                 sequenceName=EvgenSequence.Generator.value)
 
    # Set up the process
    cfg.merge(sample.setupProcess(flags))
 
    # Sort the list of generator names into standard form
    from GeneratorConfig.GenConfigHelpers import gen_sortkey
    from GeneratorConfig.Versioning import generatorsGetInitialVersionedDictionary, generatorsVersionedStringList
    generators = sorted(cfg.getService("GeneratorInfoSvc").Generators, key=gen_sortkey)
    gendict = generatorsGetInitialVersionedDictionary(generators)
    generatorsWithVersion = generatorsVersionedStringList(gendict)
 
    # Check if the setup requires steering
    from GeneratorConfig.GenConfigHelpers import gen_require_steering
    if gen_require_steering(generators):
        if hasattr(runArgs, "outputEVNTFile") and not hasattr(runArgs, "outputEVNT_PreFile"):
            raise RuntimeError("'EvtGen' found in job options name, please set '--steering=afterburn'")
 
    # LHE input handling
    if flags.Generator.inputFilesPerJob > 0:
        if not flags.Generator.inputGeneratorFile:
            raise RuntimeError(f"Sample sets inputFilesPerJob = {flags.Generator.inputFilesPerJob} but Gen_tf run without inputGeneratorFile")
        else:
            nEventsLHE = _handle_input_files(generators, flags)
 
    # Check black-list and purple-list
    blError = checkBlackList(athenaRel, generators, "black")
    plError = checkBlackList(athenaRel, generators, "purple")
    if blError is not None:
        raise RuntimeError(blError)
    if plError is not None:
        evgenLog.warning("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
        evgenLog.warning(f"!!! WARNING {plError} !!!")
        evgenLog.warning("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
 
    # Fix non-standard event features
    if not flags.Input.Files:
        from EvgenProdTools.EvgenProdToolsConfig import FixHepMCCfg
        from GeneratorConfig.GenConfigHelpers import gens_purgenoendvtx
        generatorsList = generators.copy()
        if "Pythia8" in generatorsList:
            pythia8Alg = cfg.getEventAlgo("Pythia8_i")
            if pythia8Alg.Beam1 != "PROTON" or pythia8Alg.Beam2 != "PROTON":
            # generator name is still "Pythia8", even when colliding nuclei
                generatorsList.append("Pythia8-Angantyr")
        cfg.merge(FixHepMCCfg(flags,
                              PurgeUnstableWithoutEndVtx=gens_purgenoendvtx(generatorsList)))
 
    # Sanity check the event record (not appropriate for all generators)
    from GeneratorConfig.GenConfigHelpers import gens_testhepmc
    if gens_testhepmc(generators):
        from EvgenProdTools.EvgenProdToolsConfig import TestHepMCCfg
        cfg.merge(TestHepMCCfg(flags))
 
    # Copy the event weight from HepMC to the Athena EventInfo class
    from EvgenProdTools.EvgenProdToolsConfig import CopyEventWeightCfg
    cfg.merge(CopyEventWeightCfg(flags))
 
    from EvgenProdTools.EvgenProdToolsConfig import FillFilterValuesCfg
    cfg.merge(FillFilterValuesCfg(flags))
 
    # Configure the event counting (AFTER all filters)
    from EvgenProdTools.EvgenProdToolsConfig import CountHepMCCfg
    cfg.merge(CountHepMCCfg(flags,
                            RequestedOutput=sample.nEventsPerJob if runArgs.maxEvents == -1
                                            else runArgs.maxEvents))
    evgenLog.info("Requested output events = %d", cfg.getEventAlgo("CountHepMC").RequestedOutput)
 
    # Print out the contents of the first 5 events (after filtering)
    if hasattr(runArgs, "printEvts") and runArgs.printEvts > 0:
        from TruthIO.TruthIOConfig import PrintMCCfg
        cfg.merge(PrintMCCfg(flags,
                             LastEvent=runArgs.printEvts))
 
    # PerfMon
    from PerfMonComps.PerfMonCompsConfig import PerfMonMTSvcCfg
    cfg.merge(PerfMonMTSvcCfg(flags), sequenceName=EvgenSequence.Post.value)
 
    # Estimate time needed for Simulation
    from EvgenProdTools.EvgenProdToolsConfig import SimTimeEstimateCfg
    cfg.merge(SimTimeEstimateCfg(flags))
 
    # TODO: Rivet
 
    # Extra metadata
    from EventInfoMgt.TagInfoMgrConfig import TagInfoMgrCfg
    from GeneratorConfig.GenConfigHelpers import gen_lhef
    metadata = {
        "project_name": "IS_SIMULATION",
        f"AtlasRelease_{runArgs.trfSubstepName}": flags.Input.Release or "n/a",
        "beam_energy": str(int(flags.Beam.Energy)),
        "beam_type": flags.Beam.Type.value,
        "generators": '+'.join(generatorsWithVersion),
        "tune":  cfg.getService("GeneratorInfoSvc").Tune,
        "hepmc_version": f"HepMC{os.environ['HEPMCVER']}",
        "keywords": ", ".join(sample.keywords).lower(),
        "lhefGenerator": '+'.join(filter(gen_lhef, generators)),
        "mc_channel_number": str(flags.Generator.DSID),
    }
    if hasattr(sample, "process"): metadata.update({"evgenProcess": sample.process})
    if hasattr(sample, "specialConfig"): metadata.update({"specialConfiguration": sample.specialConfig})
    if hasattr(sample, "hardPDF"): metadata.update({"hardPDF": sample.hardPDF})
    if hasattr(sample, "softPDF"): metadata.update({"softPDF": sample.softPDF})
    if hasattr(sample, "randomSeed"): metadata.update({"randomSeed": str(runArgs.randomSeed)})
    cfg.merge(TagInfoMgrCfg(flags, tagValuePairs=metadata))
 
    # Print metadata in the log
    evgenLog.info(f"HepMC version {os.environ['HEPMCVER']}")
    evgenLog.info(f"MetaData: generatorTune = {cfg.getService('GeneratorInfoSvc').Tune}")
    evgenLog.info("MetaData: generatorName = {}".format(generatorsWithVersion))
    if flags.Generator.inputGeneratorFile:
        print(f"MetaData: Number of input LHE events = {nEventsLHE}")
 
    # Configure output stream
    from OutputStreamAthenaPool.OutputStreamConfig import OutputStreamCfg
    cfg.merge(OutputStreamCfg(flags, "EVNT", ["McEventCollection#*"]))
 
    # Add in-file MetaData
    from xAODMetaDataCnv.InfileMetaDataConfig import SetupMetaDataForStreamCfg
    cfg.merge(SetupMetaDataForStreamCfg(flags, "EVNT"))
 
    # Post-include
    processPostInclude(runArgs, flags, cfg)
 
    # Post-exec
    processPostExec(runArgs, flags, cfg)
 
    # Write AMI tag into in-file MetaData
    from PyUtils.AMITagHelperConfig import AMITagCfg
    cfg.merge(AMITagCfg(flags, runArgs))
 
    # Print ComponentAccumulator components
    cfg.printConfig(prefix="Gen_tf", printSequenceTreeOnly=not runArgs.VERBOSE)
 
    # Run final ComponentAccumulator
    sys.exit(not cfg.run().isSuccess())

setupSample()

python.GENtoEVGEN_Skeleton.setupSample	(		runArgs,
			flags )

Definition at line 192 of file GENtoEVGEN_Skeleton.py.

def setupSample(runArgs, flags):
    # Only permit one jobConfig argument for evgen
    if len(runArgs.jobConfig) != 1:
        raise RuntimeError("You must supply one and only one jobConfig file argument")
 
    evgenLog.info("Using JOBOPTSEARCHPATH (as seen in skeleton) = {}".format(os.environ["JOBOPTSEARCHPATH"]))
 
    FIRST_DIR = (os.environ["JOBOPTSEARCHPATH"]).split(":")[0]
 
    # Find jO file
    jofiles = [f for f in os.listdir(FIRST_DIR) if (f.startswith("mc") and f.endswith(".py"))]
    if len(jofiles) !=1:
        raise RuntimeError("You must supply one and only one jobOption file in DSID directory")
    jofile = jofiles[0]
 
    # Perform consistency checks on the jO
    from GeneratorConfig.GenConfigHelpers import checkNaming, checkNEventsPerJob, checkKeywords, checkCategories
    checkNaming(jofile)
 
    # Import the jO as a module
    # We cannot do import BLAH directly since
    # 1. the filenames are not python compatible (mc.GEN_blah.py)
    # 2. the filenames are different for every jO
    import importlib.util
    spec = importlib.util.spec_from_file_location(
        name="sample",
        location=os.path.join(FIRST_DIR,jofile),
    )
    jo = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(jo)
    evgenLog.info("including file %s", jofile)
 
    # Create instance of Sample(EvgenCAConfig)
    sample = jo.Sample(flags)
 
    # Set up the sample properties
    sample.setupFlags(flags)
 
    # Set the random number seed
    # Need to use logic in EvgenJobTransforms.Generate_dsid_ranseed
 
    # Get DSID
    dsid = os.path.basename(runArgs.jobConfig[0])
    if dsid.startswith("Test"):
        dsid = dsid.split("Test")[-1]
 
    # Update the global flags
    if dsid.isdigit():
        flags.Generator.DSID = int(dsid)
 
    # Set nEventsPerJob
    if not sample.nEventsPerJob:
        evgenLog.info("#############################################################")
        evgenLog.info(" !!!! no sample.nEventsPerJob set !!!")
        evgenLog.info("#############################################################")
        # We don't need to set the global flag because its default is 10000
    else:
        checkNEventsPerJob(sample)
        evgenLog.info(" nEventsPerJob = " + str(sample.nEventsPerJob))
        flags.Generator.nEventsPerJob = sample.nEventsPerJob
 
    # Validate all required/conditional sample metadata with explicit rules.
    _validate_sample_properties(sample)
 
    # Propagate optional sample values to global flags.
    flags.Generator.inputFilesPerJob = sample.inputFilesPerJob
    flags.Generator.MEgenerator = sample.MEgenerator or ""
 
    # Print sample metadata in the log.
    for var, value in vars(sample).items():
        evgenLog.info("MetaData: {} = {}".format(var, value))
 
    # Keywords check
    if hasattr(sample, "keywords"):
        checkKeywords(sample, evgenLog)
 
    # L1, L2 categories check
    if hasattr(sample, "categories"):
        checkCategories(sample, evgenLog)
 
    return sample
 
 
# Function to check black-listed releases

Variable Documentation

evgenLog

python.GENtoEVGEN_Skeleton.evgenLog = logging.getLogger("Gen_tf")

Definition at line 11 of file GENtoEVGEN_Skeleton.py.

jobPropertiesDisallowed

python.GENtoEVGEN_Skeleton.jobPropertiesDisallowed

Definition at line 7 of file GENtoEVGEN_Skeleton.py.