3 """Functionality core of the Gen_tf transform"""
13 import AthenaCommon.AlgSequence
as acas
14 import AthenaCommon.AppMgr
as acam
15 from AthenaCommon.AthenaCommonFlags
import jobproperties
17 from xAODEventInfoCnv.xAODEventInfoCnvConf
import xAODMaker__EventInfoCnvAlg
18 acam.athMasterSeq += xAODMaker__EventInfoCnvAlg(xAODKey=
"TMPEvtInfo")
21 acam.athMasterSeq += acas.AlgSequence(
"EvgenGenSeq")
22 genSeq = acam.athMasterSeq.EvgenGenSeq
23 acam.athMasterSeq += acas.AlgSequence(
"EvgenPreFilterSeq")
24 prefiltSeq = acam.athMasterSeq.EvgenPreFilterSeq
25 acam.athMasterSeq += acas.AlgSequence(
"EvgenTestSeq")
26 testSeq = acam.athMasterSeq.EvgenTestSeq
28 from EvgenProdTools.LogicalExpressionFilter
import LogicalExpressionFilter
30 filtSeq = acam.athMasterSeq.EvgenFilterSeq
31 topSeq = acas.AlgSequence()
33 topSeq += acas.AlgSequence(
"EvgenPostSeq")
34 postSeq = topSeq.EvgenPostSeq
43 import AthenaCommon.AtlasUnixStandardJob
44 include(
"PartPropSvc/PartPropSvc.py")
47 from PerfMonComps.PerfMonFlags
import jobproperties
as perfmonjp
48 perfmonjp.PerfMonFlags.doFastMonMT =
True
51 from RngComps.RngCompsConf
import AthRNGSvc
55 jobproperties.AthenaCommonFlags.AllowIgnoreConfigError =
False
62 from AthenaCommon.Logging
import logging
63 evgenLog = logging.getLogger(
'Generate_ab')
71 evgenLog.debug(
"****************** CHECKING EVENT GENERATION ARGS *****************")
72 evgenLog.debug(
str(runArgs))
76 if not hasattr(runArgs,
"outputEVNTFile")
and not hasattr(runArgs,
"outputEVNT_PreFile"):
77 raise RuntimeError(
"No output evgen EVNT or EVNT_Pre file provided.")
80 if not hasattr(runArgs,
"ecmEnergy"):
81 raise RuntimeError(
"No center of mass energy provided.")
83 evgenLog.info(
'ecmEnergy = ' +
str(runArgs.ecmEnergy) )
84 if not hasattr(runArgs,
"randomSeed"):
85 raise RuntimeError(
"No random seed provided.")
86 if not hasattr(runArgs,
"firstEvent"):
87 raise RuntimeError(
"No first number provided.")
88 if (runArgs.firstEvent <= 0):
89 evgenLog.warning(
"Run argument firstEvent should be > 0")
91 if hasattr(runArgs,
"inputEVNT_PreFile"):
92 evgenLog.info(
"inputEVNT_PreFile = " +
','.
join(runArgs.inputEVNT_PreFile))
99 evgenLog.debug(
"****************** CONFIGURING EVENT GENERATION *****************")
103 from EvgenJobTransforms.EvgenConfig
import evgenConfig
104 from GeneratorConfig.GenConfigHelpers
import gens_known, gens_lhef, gen_sortkey, gens_testhepmc, gens_notune
107 from EvgenProdTools.EvgenProdToolsConf
import TestHepMC
108 testSeq += TestHepMC(CmEnergy=runArgs.ecmEnergy*Units.GeV)
109 if not hasattr(svcMgr,
'THistSvc'):
110 from GaudiSvc.GaudiSvcConf
import THistSvc
112 svcMgr.THistSvc.Output = [
"TestHepMCname DATAFILE='TestHepMC.root' OPT='RECREATE'"]
116 from EvgenProdTools.EvgenProdToolsConf
import CountHepMC
118 import AthenaPoolCnvSvc.ReadAthenaPool
119 svcMgr.EventSelector.FirstEvent = runArgs.firstEvent
121 if not hasattr(postSeq,
"CountHepMC"):
122 postSeq += CountHepMC(InputEventInfo=
"TMPEvtInfo",
123 OutputEventInfo=
"EventInfo",
124 mcEventWeightsKey=
"")
127 postSeq.CountHepMC.FirstEvent = runArgs.firstEvent
128 postSeq.CountHepMC.CorrectHepMC =
True
129 postSeq.CountHepMC.CorrectEventID =
True
130 postSeq.CountHepMC.CorrectRunNumber =
False
132 if hasattr(runArgs,
"inputEVNT_PreFile"):
133 from AthenaCommon.AppMgr
import ServiceMgr
135 if not hasattr(ServiceMgr.ToolSvc,
'IOVDbMetaDataTool'):
136 ServiceMgr.ToolSvc += CfgMgr.IOVDbMetaDataTool()
137 runNum =
int((runArgs.jobConfig[0])[-6:])
138 ServiceMgr.ToolSvc.IOVDbMetaDataTool.MinMaxRunNumbers = [runNum, runNum+1]
142 if hasattr(runArgs,
"printEvts")
and runArgs.printEvts > 0:
143 from TruthIO.TruthIOConf
import PrintMC
145 postSeq.PrintMC.McEventKey =
"GEN_EVENT"
146 postSeq.PrintMC.VerboseOutput =
True
147 postSeq.PrintMC.PrintStyle =
"Barcode"
148 postSeq.PrintMC.FirstEvent = 1
149 postSeq.PrintMC.LastEvent = runArgs.printEvts
153 if hasattr(runArgs,
"rivetAnas"):
154 from Rivet_i.Rivet_iConf
import Rivet_i
156 anaSeq.Rivet_i.Analyses = runArgs.rivetAnas
157 anaSeq.Rivet_i.AnalysisPath = os.environ[
'PWD']
158 if hasattr(runArgs,
"outputYODAFile"):
159 anaSeq.Rivet_i.HistoFile = runArgs.outputYODAFile
166 evgenLog.debug(
"****************** LOADING PRE-INCLUDES AND JOB CONFIG *****************")
169 if hasattr(runArgs,
"preInclude"):
170 for fragment
in runArgs.preInclude:
174 if hasattr(runArgs,
"preExec"):
175 evgenLog.info(
"Transform pre-exec")
176 for cmd
in runArgs.preExec:
183 if hasattr(runArgs,
"outputTXTFile"): outputTXTFile=runArgs.outputTXTFile
188 if len(runArgs.jobConfig) != 1:
189 evgenLog.error(
"You must supply one and only one jobConfig file argument")
191 evgenLog.info(
"Using JOBOPTSEARCHPATH!! = '%s'" % os.environ[
"JOBOPTSEARCHPATH"])
192 FIRST_DIR = (os.environ[
'JOBOPTSEARCHPATH']).
split(
":")[0]
194 jofiles = [f
for f
in os.listdir(FIRST_DIR)
if (f.startswith(
'mc')
and f.endswith(
'.py'))]
197 evgenLog.error(
"You must supply one and only one jobOption file in DSID directory")
201 joparts = (os.path.basename(jofile)).
split(
".")
203 if joparts[0].startswith(
"mc"):
206 if len(joparts) != 3:
207 evgenLog.error(jofile +
" name format is wrong: must be of the form mc.<physicsShort>.py: please rename.")
210 jo_physshortpart = joparts[1]
211 max_jo_physshort_length = 50
212 if len(jo_physshortpart) > max_jo_physshort_length:
213 evgenLog.error(f
"{jofile} contains a physicsShort field of more than {max_jo_physshort_length} characters: please rename.")
216 jo_physshortparts = jo_physshortpart.split(
"_")
217 if len(jo_physshortparts) < 2:
218 evgenLog.error(jofile +
" has too few physicsShort fields separated by '_': should contain <generators>(_<tune+PDF_if_available>)_<process>. Please rename.")
221 check_jofiles=
"/cvmfs/atlas.cern.ch/repo/sw/Generators/MC16JobOptions/scripts/check_jo_consistency.py"
222 if os.path.exists(check_jofiles):
224 check_naming(os.path.basename(jofile))
226 evgenLog.error(
"check_jo_consistency.py not found")
237 evgenLog.debug(
"****************** CHECKING EVGEN CONFIGURATION *****************")
240 for opt
in str(evgenConfig).
split(os.linesep):
243 evgenLog.info(
".transform = Gen_tf")
247 if evgenConfig.obsolete:
248 evgenLog.error(
"JOs or icludes are obsolete, please check them")
251 if not evgenConfig.generators:
252 evgenLog.error(
"No entries in evgenConfig.generators: invalid configuration, please check your JO")
255 if len(evgenConfig.generators) > len(
set(evgenConfig.generators)):
256 evgenLog.error(
"Duplicate entries in evgenConfig.generators: invalid configuration, please check your JO")
259 gennames =
sorted(evgenConfig.generators, key=gen_sortkey)
261 if joparts[0].startswith(
"MC"):
262 genpart = jo_physshortparts[0]
263 expectedgenpart =
''.
join(gennames)
265 expectedgenpart = expectedgenpart.replace(
"HerwigJimmy",
"Herwig")
275 if genpart !=
_norm(expectedgenpart)
and _norm2(genpart) !=
_norm(expectedgenpart):
276 evgenLog.error(
"Expected first part of JO name to be '%s' or '%s', but found '%s'" % (
_norm(expectedgenpart),
_norm(
_short2(expectedgenpart)), genpart))
277 evgenLog.error(
"gennames '%s' " %(expectedgenpart))
283 if not gens_notune(gennames)
and len(jo_physshortparts) < 3:
284 evgenLog.error(jofile +
" with generators " + expectedgenpart +
285 " has too few physicsShort fields separated by '_'." +
286 " It should contain <generators>_<tune+PDF_<process>. Please rename.")
292 if hasattr(runArgs,
'inputGeneratorFile')
and ',' in runArgs.inputGeneratorFile:
293 multiInput = runArgs.inputGeneratorFile.count(
',')+1
297 if not evgenConfig.nEventsPerJob:
298 evgenLog.info(
'#############################################################')
299 evgenLog.info(
' !!!! no nEventsPerJob set !!! The default 10000 used. !!! ')
300 evgenLog.info(
'#############################################################')
302 evgenLog.info(
' nEventsPerJob = ' +
str(evgenConfig.nEventsPerJob) )
305 if evgenConfig.minevents > 0 :
306 raise RuntimeError(
"evgenConfig.minevents is obsolete and should be removed from the JOs")
308 if evgenConfig.nEventsPerJob < 1:
309 raise RuntimeError(
"evgenConfig.nEventsPerJob must be at least 1")
310 elif evgenConfig.nEventsPerJob > 100000:
311 raise RuntimeError(
"evgenConfig.nEventsPerJob can be max. 100000")
313 allowed_nEventsPerJob_lt1000 = [1, 2, 5, 10, 20, 25, 50, 100, 200, 500, 1000]
314 msg =
"evgenConfig.nEventsPerJob = %d: " % evgenConfig.nEventsPerJob
316 if evgenConfig.nEventsPerJob >= 1000
and evgenConfig.nEventsPerJob <=10000
and (evgenConfig.nEventsPerJob % 1000 != 0
or 10000 % evgenConfig.nEventsPerJob != 0) :
317 msg +=
"nEventsPerJob in range [1K, 10K] must be a multiple of 1K and a divisor of 10K"
318 raise RuntimeError(msg)
319 elif evgenConfig.nEventsPerJob > 10000
and evgenConfig.nEventsPerJob % 10000 != 0:
320 msg +=
"nEventsPerJob >10K must be a multiple of 10K"
321 raise RuntimeError(msg)
322 elif evgenConfig.nEventsPerJob < 1000
and evgenConfig.nEventsPerJob
not in allowed_nEventsPerJob_lt1000:
323 msg +=
"nEventsPerJob in range <= 1000 must be one of %s" % allowed_nEventsPerJob_lt1000
324 raise RuntimeError(msg)
325 postSeq.CountHepMC.RequestedOutput = evgenConfig.nEventsPerJob
if runArgs.maxEvents == -1
else runArgs.maxEvents
326 evgenLog.info(
'Requested output events = '+
str(postSeq.CountHepMC.RequestedOutput))
331 if hasattr(testSeq,
"TestHepMC")
and postSeq.CountHepMC.RequestedOutput<100:
332 testSeq.TestHepMC.EffFailThreshold = postSeq.CountHepMC.RequestedOutput/(postSeq.CountHepMC.RequestedOutput+1) - 0.01
335 if evgenConfig.keywords:
336 from GeneratorConfig.GenConfigHelpers
import checkKeywords
341 if evgenConfig.findJets:
342 include(
"EvgenJobTransforms/Generate_TruthJets.py")
345 from AthenaPoolCnvSvc.WriteAthenaPool
import AthenaPoolOutputStream
346 from AthenaPoolCnvSvc.AthenaPoolCnvSvcConf
import AthenaPoolCnvSvc
349 if hasattr(runArgs,
"outputEVNTFile"):
350 poolFile = runArgs.outputEVNTFile
351 elif hasattr(runArgs,
"outputEVNT_PreFile"):
352 poolFile = runArgs.outputEVNT_PreFile
354 raise RuntimeError(
"Output pool file, either EVNT or EVNT_Pre, is not known.")
358 if hasattr(runArgs,
"inputEVNT_PreFile") :
359 svcMgr.EventSelector.InputCollections = runArgs.inputEVNT_PreFile
360 StreamEVGEN.TakeItemsFromInput =
True
361 postSeq.CountHepMC.CorrectRunNumber =
True
363 StreamEVGEN.ForceRead =
True
364 StreamEVGEN.ItemList += [
"EventInfo#*",
"xAOD::EventInfo#EventInfo*",
"xAOD::EventAuxInfo#EventInfoAux.*",
"McEventCollection#*"]
365 StreamEVGEN.RequireAlgs += [
"EvgenFilterSeq"]
367 if evgenConfig.saveJets:
368 StreamEVGEN.ItemList += [
"xAOD::JetContainer_v1#*"]
369 StreamEVGEN.ItemList += [
"xAOD::JetAuxContainer_v1#*.TruthLabelID.PartonTruthLabelID"]
370 if evgenConfig.savePileupTruthParticles:
371 StreamEVGEN.ItemList += [
"xAOD::TruthParticleContainer#TruthPileupParticles*"]
372 StreamEVGEN.ItemList += [
"xAOD::TruthParticleAuxContainer#TruthPileupParticlesAux.*"]
376 dsid = os.path.basename(runArgs.jobConfig[0])
377 if not dsid.isdigit():
379 svcMgr.EventSelector.RunNumber =
int(dsid)
381 if postSeq.CountHepMC.CorrectRunNumber ==
True:
382 postSeq.CountHepMC.NewRunNumber =
int(dsid)
383 evgenLog.info(
"Set new run number in skel NewRunNumber = " +
str(postSeq.CountHepMC.NewRunNumber))
385 evgenLog.info(
"No new run number set in skel RunNumber = " + dsid)
388 import EventInfoMgt.EventInfoMgtInit
389 svcMgr.TagInfoMgr.ExtraTagValuePairs.update({
"beam_energy":
str(
int(runArgs.ecmEnergy*Units.GeV/2.0))})
390 svcMgr.TagInfoMgr.ExtraTagValuePairs.update({
"beam_type":
'collisions'})
391 if len(evgenConfig.keywords)>0:
393 svcMgr.TagInfoMgr.ExtraTagValuePairs.update({
"keywords":
", ".
join(evgenConfig.keywords).lower()})
396 from PyUtils
import AMITagHelper
397 AMITagHelper.SetAMITag(runArgs=runArgs)
400 from OutputStreamAthenaPool.OutputStreamAthenaPoolConf
import CopyEventStreamInfo
401 streamInfoTool = CopyEventStreamInfo(
"StreamEVGEN_CopyEventStreamInfo" )
402 ToolSvc += streamInfoTool
403 svcMgr.MetaDataSvc.MetaDataTools += [ streamInfoTool ]
407 include(
"EvgenJobTransforms/Generate_ecmenergies.py")
410 include(
"EvgenJobTransforms/Generate_dsid_ranseed.py")
413 if (hasattr( runArgs,
"VERBOSE")
and runArgs.VERBOSE )
or (hasattr( runArgs,
"loglevel")
and runArgs.loglevel ==
"DEBUG")
or (hasattr( runArgs,
"loglevel")
and runArgs.loglevel ==
"VERBOSE"):
414 include(
"EvgenJobTransforms/Generate_debug_level.py")
417 svcMgr.TagInfoMgr.ExtraTagValuePairs.update({
"specialConfiguration": evgenConfig.specialConfig})
421 if hasattr(testSeq,
"TestHepMC")
and not gens_testhepmc(evgenConfig.generators):
422 evgenLog.info(
"Removing TestHepMC sanity checker")
423 del testSeq.TestHepMC
430 if hasattr(runArgs,
"postInclude"):
431 for fragment
in runArgs.postInclude:
434 if hasattr(runArgs,
"postExec"):
435 evgenLog.info(
"Transform post-exec")
436 for cmd
in runArgs.postExec:
444 acas.dumpMasterSequence()
452 evgenLog.debug(
"****************** HANDLING EVGEN INPUT FILES *****************")
456 if "McAtNlo" in evgenConfig.generators
and "Herwig" in evgenConfig.generators:
457 datFile =
"inparmMcAtNlo.dat"
458 elif "Alpgen" in evgenConfig.generators:
459 datFile =
"inparmAlpGen.dat"
460 elif "Protos" in evgenConfig.generators:
461 datFile =
"protos.dat"
462 elif "ProtosLHEF" in evgenConfig.generators:
463 datFile =
"protoslhef.dat"
464 elif "AcerMC" in evgenConfig.generators:
465 datFile =
"inparmAcerMC.dat"
466 elif "CompHep" in evgenConfig.generators:
467 datFile =
"inparmCompHep.dat"
474 "Return a matching file, provided it is unique"
476 files = glob.glob(pattern)
479 raise RuntimeError(
"No '%s' file found" % pattern)
481 raise RuntimeError(
"More than one '%s' file found" % pattern)
485 "Make a symlink safely"
487 if os.path.exists(dstfile)
and not os.path.samefile(dstfile, srcfile):
489 if not os.path.exists(dstfile):
490 evgenLog.info(
"Symlinking %s to %s" % (srcfile, dstfile))
491 os.symlink(srcfile, dstfile)
493 evgenLog.debug(
"Symlinking: %s is already the same as %s" % (dstfile, srcfile))
497 if evgenConfig.auxfiles:
499 get_files(evgenConfig.auxfiles, keepDir=
False, errorIfNotFound=
True)
507 if not hasattr(evgenConfig, attr)
or not getattr(evgenConfig, attr):
508 msg =
"evgenConfig attribute '%s' not found." % attr
510 raise RuntimeError(
"Required " + msg)
515 msg = evgenConfig.description
517 msg +=
" " + evgenConfig.notes
518 print (
"MetaData: %s = %s" % (
"physicsComment", msg))
520 print (
"MetaData: %s = %s" % (
"generatorName",
"+".
join(gennames)))
522 print (
"MetaData: %s = %s" % (
"physicsProcess", evgenConfig.process))
524 print (
"MetaData: %s = %s" % (
"generatorTune", evgenConfig.tune))
526 print (
"MetaData: %s = %s" % (
"hardPDF", evgenConfig.hardPDF))
528 print (
"MetaData: %s = %s" % (
"softPDF", evgenConfig.softPDF))
530 print (
"MetaData: %s = %s" % (
"nEventsPerJob", evgenConfig.nEventsPerJob))
532 print (
"MetaData: %s = %s" % (
"keywords",
", ".
join(evgenConfig.keywords).lower()))
534 print (
"MetaData: %s = %s" % (
"specialConfig", evgenConfig.specialConfig))
537 print (
"MetaData: %s = %s" % (
"contactPhysicist",
", ".
join(evgenConfig.contact)))
540 filterNames = [alg.getType()
for alg
in acas.iter_algseq(filtSeq)]
541 excludedNames = [
'AthSequencer',
'PyAthena::Alg',
'TestHepMC']
542 filterNames =
list(
set(filterNames) -
set(excludedNames))
543 print (
"MetaData: %s = %s" % (
"genFilterNames",
", ".
join(filterNames)))
550 from PyJobTransformsCore.runargs
import RunArguments
551 runPars = RunArguments()
552 runPars.nEventsPerJob = evgenConfig.nEventsPerJob
553 runPars.maxeventsstrategy = evgenConfig.maxeventsstrategy
554 with open(
"config.pickle",
"wb")
as f:
556 pickle.dump(runPars, f)
562 evgenLog.info(
"****************** STARTING EVENT GENERATION *****************")