130def generate(process_dir='PROC_mssm_0', grid_pack=False, gridpack_compile=False, extlhapath=None, required_accuracy=0.01, runArgs=None, flags=None, bias_module=None, requirePMGSettings=False, pdf_setting=None):
131 global my_MGC_instance
135 my_MGC_instance.setup_path_protection()
140 if 'ATHENA_CORE_NUMBER' in os.environ
and int(os.environ[
'ATHENA_CORE_NUMBER'])>0:
141 njobs = int(os.environ[
'ATHENA_CORE_NUMBER'])
142 mglog.info(
'Lucky you - you are running on a full node queue. Will re-configure for '+str(njobs)+
' jobs.')
146 if cluster_type
is not None:
150 mglog.info(
'Running event generation from gridpack (using smarter mode from generate() function)')
151 if flags
is not None:
152 generate_from_gridpack(flags=flags,extlhapath=extlhapath,gridpack_compile=gridpack_compile,requirePMGSettings=requirePMGSettings,pdf_setting=pdf_setting)
154 generate_from_gridpack(runArgs=runArgs,extlhapath=extlhapath,gridpack_compile=gridpack_compile,requirePMGSettings=requirePMGSettings,pdf_setting=pdf_setting)
157 mglog.info(
'Did not identify an input gridpack.')
159 mglog.info(
'The grid_pack flag is set, so I am expecting to create a gridpack in this job')
165 isNLO = my_MGC_instance.isNLO
176 if shutil.which(
'f2py')
is not None:
177 mglog.info(
'Found f2py, will use it for reweighting')
179 raise RuntimeError(
'Could not find f2py, needed for reweighting')
182 global MADGRAPH_COMMAND_STACK
186 mglog.info(
'Started generating gridpack at '+str(time.asctime()))
187 mglog.warning(
' >>>>>> THIS KIND OF JOB SHOULD ONLY BE RUN LOCALLY - NOT IN GRID JOBS <<<<<<')
190 my_settings = {
'nevents':
'1000'}
193 my_settings[
'req_acc']=str(required_accuracy)
196 LO_has_madspin =
False
197 if os.access(f
'{process_dir}/Cards/madspin_card.dat',os.R_OK):
198 MADGRAPH_COMMAND_STACK += [f
'mv {process_dir}/Cards/madspin_card.dat {process_dir}/Cards/madspin_card.tmp.dat']
199 os.rename(f
'{process_dir}/Cards/madspin_card.dat',f
'{process_dir}/Cards/madspin_card.tmp.dat')
200 LO_has_madspin =
True
201 my_settings = {
'gridpack':
'true'}
202 my_MGC_instance.runCardDict.update(my_settings)
205 mglog.info(
'Started generating at '+str(time.asctime()))
207 mglog.info(
'Run '+MADGRAPH_RUN_NAME+
' will be performed in mode '+str(mode)+
' with '+str(njobs)+
' jobs in parallel.')
210 if not os.access(process_dir,os.R_OK):
211 raise RuntimeError(
'No process directory found at '+process_dir)
212 if not os.access(process_dir+
'/bin/generate_events',os.R_OK):
213 raise RuntimeError(
'No generate_events module found in '+process_dir)
215 mglog.info(
'For your information, the libraries available are (should include LHAPDF):')
216 ls_dir(process_dir+
'/lib')
219 if bias_module
is not None:
222 mglog.info(
'Now I will hack the make files a bit. Apologies, but there seems to be no good way around this.')
223 shutil.copyfile(process_dir+
'/Source/make_opts',process_dir+
'/Source/make_opts_old')
224 old_opts = open(process_dir+
'/Source/make_opts_old',
'r')
225 new_opts = open(process_dir+
'/Source/make_opts',
'w')
226 for aline
in old_opts:
228 mglog.info(
'Configuring the fancy gfortran compiler instead of g77 / f77')
229 new_opts.write(
' FC=gfortran\n')
231 new_opts.write(aline)
234 mglog.info(
'Make file hacking complete.')
238 os.chdir(process_dir)
240 MADGRAPH_COMMAND_STACK += [
'cd ${MGaMC_PROCESS_DIR}' ]
243 my_MGC_instance.run_card_consistency_check()
247 code = check_PMG_updates(my_MGC_instance.paramCard)
248 if requirePMGSettings
and code!=0:
249 raise RuntimeError(
'Settings are not compliant with PMG defaults! Please use do_PMG_updates function to get PMG default params.')
253 command = [python,
'bin/generate_events']
255 command += [
'--name='+MADGRAPH_RUN_NAME]
256 mglog.info(
'Removing Cards/shower_card.dat to ensure we get parton level events only')
258 os.unlink(
'Cards/shower_card.dat')
259 except FileNotFoundError:
260 mglog.info(
'Cannot find Cards/shower_card.dat.')
262 command += [MADGRAPH_RUN_NAME]
264 setNCores(process_dir=os.getcwd(), Ncores=njobs)
267 mglog.info(
'Setting up cluster running')
268 my_MGC_instance.configCardDict[
'run_mode'] = 1
270 if cluster_type==
'pbs':
271 mglog.info(
'Modifying bin/internal/cluster.py for PBS cluster running')
272 os.system(
"sed -i \"s:text += prog:text += './'+prog:g\" bin/internal/cluster.py")
274 mglog.info(
'Setting up multi-core running on '+os.environ[
'ATHENA_CORE_NUMBER']+
' cores')
276 mglog.info(
'Setting up serial generation.')
279 my_MGC_instance.write_configCard()
281 my_MGC_instance.paramCard.write_paramCard()
289 (out,err) = generate.communicate()
290 error_check(err,generate.returncode)
294 MADGRAPH_COMMAND_STACK += [
'cd -' ]
299 energy =
'%1.1f'%(beamEnergy*2./1000.)
300 energy = energy.replace(
'.0',
'').
replace(
'.',
'p')
301 gridpack_name=
'mc_'+energy+
'TeV.'+get_physics_short()+
'.MG'+get_mg5_version().
replace(
'.',
'')+
'.GRID.tar.gz'
302 mglog.info(
'Tidying up gridpack '+gridpack_name)
307 MADGRAPH_COMMAND_STACK += [f
'mv {process_dir}/Cards/madspin_card.tmp.dat {process_dir}/Cards/madspin_card.dat']
308 os.rename(f
'{process_dir}/Cards/madspin_card.tmp.dat',f
'{process_dir}/Cards/madspin_card.dat')
311 MADGRAPH_COMMAND_STACK += [
'cp '+glob.glob(process_dir+
'/'+MADGRAPH_RUN_NAME+
'_*gridpack.tar.gz')[0]+
' '+gridpack_name]
312 shutil.copy(glob.glob(process_dir+
'/'+MADGRAPH_RUN_NAME+
'_*gridpack.tar.gz')[0],gridpack_name)
315 MADGRAPH_COMMAND_STACK += [
'mkdir tmp%i/'%os.getpid(),
'cd tmp%i/'%os.getpid()]
316 os.mkdir(
'tmp%i/'%os.getpid())
317 os.chdir(
'tmp%i/'%os.getpid())
318 mglog.info(
'untar gridpack')
321 mglog.info(
'compile and clean up')
322 MADGRAPH_COMMAND_STACK += [
'cd madevent']
323 os.chdir(
'madevent/')
325 (out,err) = compilep.communicate()
326 error_check(err,compilep.returncode)
328 (out,err) = clean.communicate()
329 error_check(err,clean.returncode)
331 MADGRAPH_COMMAND_STACK += [
'cd ..',
'rm ../'+gridpack_name]
333 mglog.info(
'remove old tarball')
334 os.unlink(
'../'+gridpack_name)
335 mglog.info(
'Package up new tarball')
336 tar =
stack_subprocess([
'tar',
'--exclude=SubProcesses/P*/G*/*_results.dat',
'--exclude=SubProcesses/P*/G*/*.log',
'--exclude=SubProcesses/P*/G*/*.txt',
'-cvsf',
'../'+gridpack_name,
'.'])
338 MADGRAPH_COMMAND_STACK += [
'cd ..',
'rm -r tmp%i/'%os.getpid()]
340 mglog.info(
'Remove temporary directory')
341 shutil.rmtree(
'tmp%i/'%os.getpid())
342 mglog.info(
'Tidying up complete!')
347 mglog.info(
'Package up process_dir')
348 MADGRAPH_COMMAND_STACK += [
'mv '+process_dir+
' '+MADGRAPH_GRIDPACK_LOCATION]
349 os.rename(process_dir,MADGRAPH_GRIDPACK_LOCATION)
350 tar =
stack_subprocess([
'tar',
'--exclude=Events/*/*events*gz',
'--exclude=SubProcesses/P*/G*/log*txt',
'--exclude=SubProcesses/P*/G*/events.lhe*',
'--exclude=*/*.o',
'--exclude=*/*/*.o',
'--exclude=*/*/*/*.o',
'--exclude=*/*/*/*/*.o',
'-czf',gridpack_name,MADGRAPH_GRIDPACK_LOCATION])
352 MADGRAPH_COMMAND_STACK += [
'mv '+MADGRAPH_GRIDPACK_LOCATION+
' '+process_dir]
353 os.rename(MADGRAPH_GRIDPACK_LOCATION,process_dir)
355 mglog.info(
'Gridpack sucessfully created, exiting the transform')
358 mglog.info(
'Touching output TXT (LHE) file for the transform')
359 open(output_txt_file,
'w').close()
360 from AthenaCommon.AppMgr
import theApp
364 mglog.info(
'Finished at '+str(time.asctime()))
368def generate_from_gridpack(runArgs=None, flags=None, extlhapath=None, gridpack_compile=None, requirePMGSettings=False, pdf_setting=None):
369 global my_MGC_instance
375 my_MGC_instance.setup_path_protection()
377 isNLO = my_MGC_instance.isNLO
382 gridpack_run_name =
'GridRun_'+str(random_seed)
385 if os.access(MADGRAPH_GRIDPACK_LOCATION+
'/Cards/madspin_card.dat',os.R_OK):
386 os.rename(MADGRAPH_GRIDPACK_LOCATION+
'/Cards/madspin_card.dat',MADGRAPH_GRIDPACK_LOCATION+
'/Cards/backup_madspin_card.dat')
395 code = check_PMG_updates(my_MGC_instance.paramCard)
396 if requirePMGSettings
and code!=0:
397 raise RuntimeError(
'Settings are not compliant with PMG defaults! Please use do_PMG_updates function to get PMG default params.')
400 settings={
'iseed':str(random_seed)}
402 settings[
'python_seed']=str(random_seed)
403 my_MGC_instance.runCardDict.update(settings)
405 mglog.info(
'Generating events from gridpack')
408 if not os.path.exists(MADGRAPH_GRIDPACK_LOCATION):
409 raise RuntimeError(
'Gridpack directory not found at '+MADGRAPH_GRIDPACK_LOCATION)
411 nevents = my_MGC_instance.runCardDict[
'nevents']
412 mglog.info(
'>>>> FOUND GRIDPACK <<<< <- This will be used for generation')
413 mglog.info(
'Generation of '+str(int(nevents))+
' events will be performed using the supplied gridpack with random seed '+str(random_seed))
414 mglog.info(
'Started generating events at '+str(time.asctime()))
417 if os.access(MADGRAPH_GRIDPACK_LOCATION+
'/bin/internal/addmasses.py',os.R_OK):
418 os.remove(MADGRAPH_GRIDPACK_LOCATION+
'/bin/internal/addmasses.py')
423 setNCores(process_dir=MADGRAPH_GRIDPACK_LOCATION)
427 ls_dir(MADGRAPH_GRIDPACK_LOCATION)
430 my_MGC_instance.run_card_consistency_check()
437 systematics_settings=
None
438 if my_MGC_instance.runCardDict.get(
'systematics_program',
None) ==
'systematics':
439 if not my_MGC_instance.runCardDict.get(
'store_rwgt_info',
None):
440 raise RuntimeError(
'Trying to run NLO systematics but reweight info not stored')
441 if 'systematics_arguments' in my_MGC_instance.runCardDict:
442 systematics_settings=MadGraphControl.MadGraphSystematicsUtils.parse_systematics_arguments(my_MGC_instance.runCardDict[
'systematics_arguments'])
444 systematics_settings={}
445 mglog.info(
'Turning off systematics for now, running standalone later')
449 my_MGC_instance.write_configCard()
450 my_MGC_instance.paramCard.write_paramCard()
452 global MADGRAPH_COMMAND_STACK
455 if not os.access(MADGRAPH_GRIDPACK_LOCATION+
'/bin/gridrun',os.R_OK):
456 mglog.error(
'/bin/gridrun not found at '+MADGRAPH_GRIDPACK_LOCATION)
457 raise RuntimeError(
'Could not find gridrun executable')
459 mglog.info(
'Found '+MADGRAPH_GRIDPACK_LOCATION+
'/bin/gridrun, starting generation.')
462 mg5_version_tuple = tuple(
map(int, get_mg5_version().
split(
".")))
463 if mg5_version_tuple[:3] > (3, 6, 2):
464 nprocs = int(os.environ[
'ATHENA_CORE_NUMBER'])
467 mglog.info(
"Now generating {} events with random seed {}, granularity {}, nprocs {}, and maxevts {}".format(int(nevents),int(random_seed),granularity,nprocs,maxevts))
469 mglog.info(
"Now generating {} events with random seed {} and granularity {}".format(int(nevents),int(random_seed),granularity))
471 new_ld_path=
":".join([os.environ[
'LD_LIBRARY_PATH'],os.getcwd()+
'/'+MADGRAPH_GRIDPACK_LOCATION+
'/madevent/lib',os.getcwd()+
'/'+MADGRAPH_GRIDPACK_LOCATION+
'/HELAS/lib'])
472 os.environ[
'LD_LIBRARY_PATH']=new_ld_path
473 MADGRAPH_COMMAND_STACK+=[
"export LD_LIBRARY_PATH="+
":".join([
'${LD_LIBRARY_PATH}',new_ld_path])]
474 if mg5_version_tuple[:3] > (3, 6, 2):
475 generate =
stack_subprocess([python,MADGRAPH_GRIDPACK_LOCATION+
'/bin/gridrun',str(int(nevents)),str(int(random_seed)),str(granularity),str(nprocs),str(maxevts)],stdin=subprocess.PIPE,stderr=subprocess.PIPE
if _should_catch_errors()
else None)
477 generate =
stack_subprocess([python,MADGRAPH_GRIDPACK_LOCATION+
'/bin/gridrun',str(int(nevents)),str(int(random_seed)),str(granularity)],stdin=subprocess.PIPE,stderr=subprocess.PIPE
if _should_catch_errors()
else None)
478 (out,err) = generate.communicate()
479 error_check(err,generate.returncode)
480 gp_events=MADGRAPH_GRIDPACK_LOCATION+
"/Events/GridRun_{}/unweighted_events.lhe.gz".format(int(random_seed))
481 if not os.path.exists(gp_events):
482 mglog.error(
'Error in gp generation, did not find events at '+gp_events)
486 if reweight_card
is not None:
487 pythonpath_backup=os.environ[
'PYTHONPATH']
489 os.environ[
'PYTHONPATH']=
':'.join([p
for p
in pythonpath_backup.split(
':')
if 'madgraph5amc' not in p])
491 os.environ[
'PYTHONPATH']=pythonpath_backup
493 shutil.move(gp_events,
'events.lhe.gz')
497 if not os.access(MADGRAPH_GRIDPACK_LOCATION+
'/bin/generate_events',os.R_OK):
498 raise RuntimeError(
'Could not find generate_events executable at '+MADGRAPH_GRIDPACK_LOCATION)
500 mglog.info(
'Found '+MADGRAPH_GRIDPACK_LOCATION+
'/bin/generate_events, starting generation.')
502 ls_dir(MADGRAPH_GRIDPACK_LOCATION+
'/Events/')
503 if os.access(MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name, os.F_OK):
504 mglog.info(
'Removing '+MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name+
' directory from gridpack generation')
505 MADGRAPH_COMMAND_STACK += [
'rm -rf '+MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name]
506 shutil.rmtree(MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name)
509 if os.access(MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name+
'_decayed_1', os.F_OK):
510 mglog.info(
'Removing '+MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name+
'_decayed_1 directory from gridpack generation')
511 MADGRAPH_COMMAND_STACK += [
'rm -rf '+MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name+
'_decayed_1']
512 shutil.rmtree(MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name+
'_decayed_1')
514 ls_dir(MADGRAPH_GRIDPACK_LOCATION+
'/Events/')
516 if not gridpack_compile:
517 mglog.info(
'Copying make_opts from Template')
518 shutil.copy(os.environ[
'MADPATH']+
'/Template/LO/Source/make_opts',MADGRAPH_GRIDPACK_LOCATION+
'/Source/')
521 generate =
stack_subprocess([python,MADGRAPH_GRIDPACK_LOCATION+
'/bin/generate_events',
'--parton',
'--nocompile',
'--only_generation',
'-f',
'--name='+gridpack_run_name],stdin=subprocess.PIPE,stderr=subprocess.PIPE
if _should_catch_errors()
else None)
522 (out,err) = generate.communicate()
523 error_check(err,generate.returncode)
525 mglog.info(
'Allowing recompilation of gridpack')
526 if os.path.islink(MADGRAPH_GRIDPACK_LOCATION+
'/lib/libLHAPDF.a'):
527 mglog.info(
'Unlinking '+MADGRAPH_GRIDPACK_LOCATION+
'/lib/libLHAPDF.a')
528 os.unlink(MADGRAPH_GRIDPACK_LOCATION+
'/lib/libLHAPDF.a')
531 generate =
stack_subprocess([python,MADGRAPH_GRIDPACK_LOCATION+
'/bin/generate_events',
'--parton',
'--only_generation',
'-f',
'--name='+gridpack_run_name],stdin=subprocess.PIPE,stderr=subprocess.PIPE
if _should_catch_errors()
else None)
532 (out,err) = generate.communicate()
533 error_check(err,generate.returncode)
534 if isNLO
and systematics_settings
is not None:
536 mglog.info(
'Running systematics standalone')
537 systematics_path=MADGRAPH_GRIDPACK_LOCATION+
'/bin/internal/systematics.py'
538 events_location=MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name+
'/events.lhe.gz'
540 syst_cmd=[python,systematics_path]+[events_location]*2+[
"--"+k+
"="+systematics_settings[k]
for k
in systematics_settings]
541 mglog.info(
'running: '+
' '.join(syst_cmd))
546 if not os.access(
'events.lhe.gz',os.R_OK):
547 mglog.info(
'Copying generated events to '+currdir)
548 if not os.path.exists(MADGRAPH_GRIDPACK_LOCATION+
'Events/'+gridpack_run_name):
549 shutil.copy(MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name+
'/events.lhe.gz',
'events.lhe.gz')
551 mglog.info(
'Events were already in place')
555 mglog.info(
'Moving generated events to be in correct format for arrange_output().')
556 mglog.info(
'Unzipping generated events.')
560 mglog.info(
'Moving file over to '+MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name+
'/unweighted_events.lhe')
561 mkdir =
stack_subprocess([
'mkdir',
'-p',(MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name)])
563 shutil.move(
'events.lhe',MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name+
'/unweighted_events.lhe')
565 mglog.info(
'Re-zipping into dataset name '+MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name+
'/unweighted_events.lhe.gz')
566 rezip =
stack_subprocess([
'gzip',MADGRAPH_GRIDPACK_LOCATION+
'/Events/'+gridpack_run_name+
'/unweighted_events.lhe'])
574 os.rename(MADGRAPH_GRIDPACK_LOCATION+
'/Cards/backup_madspin_card.dat',MADGRAPH_GRIDPACK_LOCATION+
'/Cards/madspin_card.dat')
575 mglog.info(
'Decaying with MadSpin.')
578 mglog.info(
'Finished at '+str(time.asctime()))
644def setupLHAPDF(process_dir=None, extlhapath=None, allow_links=True):
645 global my_MGC_instance
647 isNLO = my_MGC_instance.isNLO
649 origLHAPATH=os.environ[
'LHAPATH']
650 origLHAPDF_DATA_PATH=os.environ[
'LHAPDF_DATA_PATH']
652 LHAPATH,LHADATAPATH=get_LHAPDF_PATHS()
658 mydict= my_MGC_instance.runCardDict
659 if mydict[
"pdlabel"].
replace(
"'",
"") ==
'lhapdf':
661 mglog.info(
'creating local LHAPDF dir: MGC_LHAPDF/')
662 if os.path.islink(
'MGC_LHAPDF/'):
663 os.unlink(
'MGC_LHAPDF/')
664 elif os.path.isdir(
'MGC_LHAPDF/'):
665 shutil.rmtree(
'MGC_LHAPDF/')
667 newMGCLHA=
'MGC_LHAPDF/'
669 mkdir = subprocess.Popen([
'mkdir',
'-p',newMGCLHA])
672 pdfs_used=[ int(x)
for x
in mydict[
'lhaid'].
replace(
' ',
',').
split(
',') ]
674 if 'sys_pdf' in mydict:
680 pdfs_used.append(idx)
683 if 'systematics_arguments' in mydict:
684 systematics_arguments=MadGraphControl.MadGraphSystematicsUtils.parse_systematics_arguments(mydict[
'systematics_arguments'])
685 if 'pdf' in systematics_arguments:
691 pdfs_used.append(idx)
694 for pdf
in pdfs_used:
695 if isinstance(pdf,str)
and (pdf.lower()==
'errorset' or pdf.lower()==
'central'):
698 pdfid,pdfname=get_lhapdf_id_and_name(pdf)
699 mglog.info(
"Found LHAPDF ID="+str(pdfid)+
", name="+pdfname)
701 if not os.path.exists(newMGCLHA+pdfname)
and not os.path.lexists(newMGCLHA+pdfname):
702 if not os.path.exists(LHADATAPATH+
'/'+pdfname):
703 mglog.warning(
'PDF not installed at '+LHADATAPATH+
'/'+pdfname)
705 mglog.info(
'linking '+LHADATAPATH+
'/'+pdfname+
' --> '+newMGCLHA+pdfname)
706 os.symlink(LHADATAPATH+
'/'+pdfname,newMGCLHA+pdfname)
708 mglog.info(
'copying '+LHADATAPATH+
'/'+pdfname+
' --> '+newMGCLHA+pdfname)
709 shutil.copytree(LHADATAPATH+
'/'+pdfname,newMGCLHA+pdfname)
712 mglog.info(
'linking '+LHADATAPATH+
'/pdfsets.index --> '+newMGCLHA+
'pdfsets.index')
713 os.symlink(LHADATAPATH+
'/pdfsets.index',newMGCLHA+
'pdfsets.index')
715 atlasLHADATAPATH=LHADATAPATH.replace(
'sft.cern.ch/lcg/external/lhapdfsets/current',
'atlas.cern.ch/repo/sw/Generators/lhapdfsets/current')
716 mglog.info(
'linking '+atlasLHADATAPATH+
'/lhapdf.conf --> '+newMGCLHA+
'lhapdf.conf')
717 os.symlink(atlasLHADATAPATH+
'/lhapdf.conf',newMGCLHA+
'lhapdf.conf')
719 mglog.info(
'copying '+LHADATAPATH+
'/pdfsets.index --> '+newMGCLHA+
'pdfsets.index')
720 shutil.copy2(LHADATAPATH+
'/pdfsets.index',newMGCLHA+
'pdfsets.index')
722 atlasLHADATAPATH=LHADATAPATH.replace(
'sft.cern.ch/lcg/external/lhapdfsets/current',
'atlas.cern.ch/repo/sw/Generators/lhapdfsets/current')
723 mglog.info(
'copying '+atlasLHADATAPATH+
'/lhapdf.conf -->'+newMGCLHA+
'lhapdf.conf')
724 shutil.copy2(atlasLHADATAPATH+
'/lhapdf.conf',newMGCLHA+
'lhapdf.conf')
727 LHADATAPATH=os.getcwd()+
'/MGC_LHAPDF'
730 mglog.info(
'Not using LHAPDF')
731 return (LHAPATH,origLHAPATH,origLHAPDF_DATA_PATH)
735 os.environ[
'LHAPDF_DATA_PATH']=LHADATAPATH
737 mglog.info(
'Path to LHAPDF install dir: '+LHAPATH)
738 mglog.info(
'Path to LHAPDF data dir: '+LHADATAPATH)
739 if not os.path.isdir(LHADATAPATH):
740 raise RuntimeError(
'LHAPDF data dir not accesible: '+LHADATAPATH)
741 if not os.path.isdir(LHAPATH):
742 raise RuntimeError(
'LHAPDF path dir not accesible: '+LHAPATH)
746 lhapdfconfig=extlhapath
747 if not os.access(lhapdfconfig,os.X_OK):
748 raise RuntimeError(
'Failed to find valid external lhapdf-config at '+lhapdfconfig)
749 LHADATAPATH=subprocess.Popen([lhapdfconfig,
'--datadir'],stdout = subprocess.PIPE).stdout.read().
strip()
750 mglog.info(
'Changing LHAPDF_DATA_PATH to '+LHADATAPATH)
751 os.environ[
'LHAPDF_DATA_PATH']=LHADATAPATH
753 getlhaconfig = subprocess.Popen([
'get_files',
'-data',
'lhapdf-config'])
756 if not os.access(os.getcwd()+
'/lhapdf-config',os.X_OK):
757 mglog.error(
'Failed to get lhapdf-config from MadGraphControl')
759 lhapdfconfig = os.getcwd()+
'/lhapdf-config'
761 mglog.info(
'lhapdf-config --version: '+str(subprocess.Popen([lhapdfconfig,
'--version'],stdout = subprocess.PIPE).stdout.read().
strip()))
762 mglog.info(
'lhapdf-config --prefix: '+str(subprocess.Popen([lhapdfconfig,
'--prefix'],stdout = subprocess.PIPE).stdout.read().
strip()))
763 mglog.info(
'lhapdf-config --libdir: '+str(subprocess.Popen([lhapdfconfig,
'--libdir'],stdout = subprocess.PIPE).stdout.read().
strip()))
764 mglog.info(
'lhapdf-config --datadir: '+str(subprocess.Popen([lhapdfconfig,
'--datadir'],stdout = subprocess.PIPE).stdout.read().
strip()))
765 mglog.info(
'lhapdf-config --pdfsets-path: '+str(subprocess.Popen([lhapdfconfig,
'--pdfsets-path'],stdout = subprocess.PIPE).stdout.read().
strip()))
768 my_MGC_instance.configCardDict.update({
'lhapdf':lhapdfconfig,
'lhapdf_py3':lhapdfconfig})
770 mglog.info(
'Creating links for LHAPDF')
771 if os.path.islink(process_dir+
'/lib/PDFsets'):
772 os.unlink(process_dir+
'/lib/PDFsets')
773 elif os.path.isdir(process_dir+
'/lib/PDFsets'):
774 shutil.rmtree(process_dir+
'/lib/PDFsets')
776 os.symlink(LHADATAPATH,process_dir+
'/lib/PDFsets')
778 shutil.copytree(LHADATAPATH,process_dir+
'/lib/PDFsets')
779 mglog.info(
'Available PDFs are:')
780 mglog.info( sorted( [ x
for x
in os.listdir(process_dir+
'/lib/PDFsets')
if ".tar.gz" not in x ] ) )
782 global MADGRAPH_COMMAND_STACK
783 MADGRAPH_COMMAND_STACK += [
'# Copy the LHAPDF files locally' ]
784 MADGRAPH_COMMAND_STACK += [
'cp -r '+os.getcwd()+
'/MGC_LHAPDF .' ]
785 MADGRAPH_COMMAND_STACK += [
'cp -r '+process_dir+
'/lib/PDFsets ${MGaMC_PROCESS_DIR}/lib/' ]
787 return (LHAPATH,origLHAPATH,origLHAPDF_DATA_PATH)
913 """ Run MadSpin on an input LHE file. Takes the process
914 from the LHE file, so you don't need to have a process directory
915 set up in advance. Runs MadSpin and packs the LHE file up appropriately
916 Needs runArgs for the file handling"""
917 if not os.access(input_LHE,os.R_OK):
918 raise RuntimeError(
'Could not find LHE file '+input_LHE)
919 if not os.access(madspin_card,os.R_OK):
920 raise RuntimeError(
'Could not find input MadSpin card '+madspin_card)
922 shutil.copy(input_LHE,input_LHE+
'.original')
923 mglog.info(
'Put backup copy of LHE file at '+input_LHE+
'.original')
925 madspin_exec_card = open(
'madspin_exec_card',
'w')
926 madspin_exec_card.write(
'import '+input_LHE+
'\n')
928 input_madspin_card = open(madspin_card,
'r')
930 for l
in input_madspin_card.readlines():
931 commands = l.split(
'#')[0].
split()
933 if len(commands)>1
and 'import'==commands[0]
and not 'model'==commands[1]:
936 if len(commands)>0
and 'launch' == commands[0]:
938 madspin_exec_card.write(l.strip()+
'\n')
940 madspin_exec_card.write(
'launch\n')
941 madspin_exec_card.close()
942 input_madspin_card.close()
944 madpath=os.environ[
'MADPATH']
945 if not os.access(madpath+
'/MadSpin/madspin',os.R_OK):
946 raise RuntimeError(
'madspin executable not found in '+madpath)
947 mglog.info(
'Starting madspin at '+str(time.asctime()))
949 (out,err) = generate.communicate()
950 error_check(err,generate.returncode)
951 mglog.info(
'Done with madspin at '+str(time.asctime()))
955 if os.path.exists(os.getcwd()+
'/events.lhe'):
956 os.remove(os.getcwd()+
'/events.lhe')
958 mglog.info(
'Unzipping generated events.')
962 mglog.info(
'Putting a copy in place for the transform.')
963 mod_output = open(os.getcwd()+
'/events.lhe',
'w')
967 with open(input_LHE,
'r')
as fileobject:
968 for line
in fileobject:
970 mod_output.write(line)
975 mglog.info(
'Removed '+str(nEmpty)+
' empty lines from LHEF')
979 raise RuntimeError(
'Must provide runArgs to madspin_on_lhe')
981 outputDS = runArgs.outputTXTFile
if hasattr(runArgs,
'outputTXTFile')
else 'tmp_LHE_events.tar.gz'
983 mglog.info(
'Moving file over to '+outputDS.split(
'.tar.gz')[0]+
'.events')
984 shutil.move(os.getcwd()+
'/events.lhe',outputDS.split(
'.tar.gz')[0]+
'.events')
986 mglog.info(
'Re-zipping into dataset name '+outputDS)
987 rezip =
stack_subprocess([
'tar',
'cvzf',outputDS,outputDS.split(
'.tar.gz')[0]+
'.events'])
991 if hasattr(runArgs,
'outputTXTFile')
and runArgs.outputTXTFile
is not None:
992 outputDS = outputDS.split(
'.TXT')[0]
994 if runArgs
is not None:
995 mglog.debug(
'Setting inputGenerator file to '+outputDS)
996 runArgs.inputGeneratorFile=outputDS
999def arrange_output(process_dir=MADGRAPH_GRIDPACK_LOCATION,lhe_version=None,saveProcDir=False,runArgs=None,flags=None,fixEventWeightsForBridgeMode=False,pdf_setting=None):
1004 if len(glob.glob(os.path.join(process_dir,
'Events',
'*')))<1:
1005 mglog.error(
'Process dir '+process_dir+
' does not contain events?')
1006 proc_dir_list = glob.glob(os.path.join(process_dir,
'Events',
'*'))
1009 for adir
in proc_dir_list:
1010 if 'decayed' in adir:
1013 if 'GridRun_' in adir:
1016 elif os.path.join(process_dir,
'Events',MADGRAPH_RUN_NAME)
in adir:
1018 if not os.access(this_run_name,os.R_OK):
1019 raise RuntimeError(
'Unable to locate run directory')
1021 hasUnweighted = os.access(this_run_name+
'/unweighted_events.lhe.gz',os.R_OK)
1024 madspinDirs=sorted(glob.glob(this_run_name+
'_decayed_*/'))
1025 if len(madspinDirs):
1027 if hasRunMadSpin
and not hasUnweighted:
1029 hasUnweighted = os.access(madspinDirs[-1]+
'/unweighted_events.lhe.gz',os.R_OK)
1031 global MADGRAPH_COMMAND_STACK
1033 if len(madspinDirs):
1040 if os.path.exists(madspinDirs[-1]+
'/unweighted_events.lhe.gz'):
1041 MADGRAPH_COMMAND_STACK += [
'mv '+madspinDirs[-1]+
'/unweighted_events.lhe.gz'+
' '+this_run_name+
'/unweighted_events.lhe.gz']
1042 shutil.move(madspinDirs[-1]+
'/unweighted_events.lhe.gz',this_run_name+
'/unweighted_events.lhe.gz')
1043 mglog.info(
'Moving MadSpin events from '+madspinDirs[-1]+
'/unweighted_events.lhe.gz to '+this_run_name+
'/unweighted_events.lhe.gz')
1044 elif os.path.exists(madspinDirs[-1]+
'/events.lhe.gz'):
1045 MADGRAPH_COMMAND_STACK += [
'mv '+madspinDirs[-1]+
'/events.lhe.gz'+
' '+this_run_name+
'/unweighted_events.lhe.gz']
1046 shutil.move(madspinDirs[-1]+
'/events.lhe.gz',this_run_name+
'/unweighted_events.lhe.gz')
1047 mglog.info(
'Moving MadSpin events from '+madspinDirs[-1]+
'/events.lhe.gz to '+this_run_name+
'/unweighted_events.lhe.gz')
1049 raise RuntimeError(
'MadSpin was run but can\'t find files :(')
1052 MADGRAPH_COMMAND_STACK += [
'mv '+madspinDirs[-1]+
'/events.lhe.gz '+this_run_name+
'/events.lhe.gz']
1053 shutil.move(madspinDirs[-1]+
'/events.lhe.gz',this_run_name+
'/events.lhe.gz')
1054 mglog.info(
'Moving MadSpin events from '+madspinDirs[-1]+
'/events.lhe.gz to '+this_run_name+
'/events.lhe.gz')
1057 mglog.error(
'MadSpin was run but can\'t find output folder '+(this_run_name+
'_decayed_1/'))
1058 raise RuntimeError(
'MadSpin was run but can\'t find output folder '+(this_run_name+
'_decayed_1/'))
1060 if fixEventWeightsForBridgeMode:
1061 mglog.info(
"Fixing event weights after MadSpin... initial checks.")
1069 eventsfilename=
"unweighted_events"
1071 eventsfilename=
"events"
1072 unzip =
stack_subprocess([
'gunzip',
'-f',this_run_name+
'/%s.lhe.gz' % eventsfilename])
1075 for line
in open(process_dir+
'/Events/'+MADGRAPH_RUN_NAME+
'/%s.lhe'%eventsfilename):
1076 if "Number of Events" in line:
1078 MGnumevents=int(sline[-1])
1079 elif "Integrated weight (pb)" in line:
1081 MGintweight=float(sline[-1])
1082 elif "set spinmode none" in line:
1084 elif "</header>" in line:
1087 if spinmodenone
and MGnumevents>0
and MGintweight>0:
1088 mglog.info(
"Fixing event weights after MadSpin... modifying LHE file.")
1089 newlhe=open(this_run_name+
'/%s_fixXS.lhe'%eventsfilename,
'w')
1097 event_norm_setting=
"average"
1099 for line
in open(this_run_name+
'/%s.lhe'%eventsfilename):
1102 if "<init>" in line:
1105 elif "</init>" in line:
1107 elif inInit
and initlinecount==0:
1111 if abs(int(sline[-2])) == 3:
1112 event_norm_setting=
"sum"
1113 elif abs(int(sline[-2])) == 4:
1114 event_norm_setting=
"average"
1115 elif inInit
and initlinecount==1:
1118 relunc=float(sline[1])/float(sline[0])
1119 sline[0]=str(MGintweight)
1120 sline[1]=str(float(sline[0])*relunc)
1121 if event_norm_setting==
"sum":
1122 sline[2]=str(MGintweight/MGnumevents)
1123 elif event_norm_setting==
"average":
1124 sline[2]=str(MGintweight)
1125 newline=
' '.join(sline)
1128 elif inInit
and initlinecount>1:
1130 elif "<event>" in line:
1133 elif "</event>" in line:
1135 elif inEvent
and eventlinecount==0:
1138 if event_norm_setting==
"sum":
1139 sline[2]=str(MGintweight/MGnumevents)
1140 elif event_norm_setting==
"average":
1141 sline[2]=str(MGintweight)
1142 newline=
' '.join(sline)
1145 newlhe.write(newline)
1148 mglog.info(
"Fixing event weights after MadSpin... cleaning up.")
1149 shutil.copyfile(this_run_name+
'/%s.lhe' % eventsfilename,
1150 this_run_name+
'/%s_badXS.lhe' % eventsfilename)
1152 shutil.move(this_run_name+
'/%s_fixXS.lhe' % eventsfilename,
1153 this_run_name+
'/%s.lhe' % eventsfilename)
1155 rezip =
stack_subprocess([
'gzip',this_run_name+
'/%s.lhe' % eventsfilename])
1158 rezip =
stack_subprocess([
'gzip',this_run_name+
'/%s_badXS.lhe' % eventsfilename])
1162 if os.path.exists(os.getcwd()+
'/events.lhe'):
1163 os.remove(os.getcwd()+
'/events.lhe')
1165 mglog.info(
'Unzipping generated events.')
1167 unzip =
stack_subprocess([
'gunzip',
'-f',this_run_name+
'/unweighted_events.lhe.gz'])
1173 mglog.info(
'Putting a copy in place for the transform.')
1175 orig_input = this_run_name+
'/unweighted_events.lhe'
1176 mod_output = open(os.getcwd()+
'/events.lhe',
'w')
1178 orig_input = this_run_name+
'/events.lhe'
1179 mod_output = open(os.getcwd()+
'/events.lhe',
'w')
1186 with open(orig_input,
'r')
as fileobject:
1187 for line
in fileobject:
1191 if '#' not in newline:
1193 elif '>' not in newline[ newline.find(
'#'): ]:
1196 mglog.info(
'Found bad LHE line with an XML mark in a comment: "'+newline.strip()+
'"')
1197 newline=newline[:newline.find(
'#')]+
'#'+ (newline[newline.find(
'#'):].
replace(
'>',
'-'))
1199 if initrwgt
is False:
1201 elif "</initrwgt>" in newline:
1203 elif "<initrwgt>" in newline:
1205 elif initrwgt
is not None:
1206 newline=newline.replace(
'_DYNSCALE-1',
'')
1207 if '</weight>' in newline:
1208 iend=newline.find(
'</weight>')
1209 istart=newline[:iend].rfind(
'>')
1210 lhe_weights+=[newline[istart+1:iend].
strip()]
1211 mod_output.write(newline)
1215 mglog.info(
'Removed '+str(nEmpty)+
' empty lines from LHEF')
1217 mglog.info(
"The following "+str(len(lhe_weights))+
" weights have been written to the LHE file: "+
",".join(lhe_weights))
1220 mglog.info(
"Checking whether the following expected weights are in LHE file: "+
",".join(expected_weights))
1221 for w
in expected_weights:
1222 if w
not in lhe_weights:
1223 raise RuntimeError(
"Did not find expected weight "+w+
" in lhe file. Did the reweight or systematics module crash?")
1224 mglog.info(
"Found all required weights!")
1227 mod_output2 = open(os.getcwd()+
'/events.lhe',
'r')
1228 test=mod_output2.readline()
1229 if 'version="' in test:
1230 mglog.info(
'Applying LHE version hack')
1231 final_file = open(os.getcwd()+
'/events.lhe.copy',
'w')
1232 final_file.write(
'<LesHouchesEvents version="%i.0">\n'%lhe_version)
1233 shutil.copyfileobj(mod_output2, final_file)
1235 shutil.copy(os.getcwd()+
'/events.lhe.copy',os.getcwd()+
'/events.lhe')
1237 os.remove(os.getcwd()+
'/events.lhe.copy')
1242 if outputDS
is None:
1243 outputDS =
'tmp_LHE_events.tar.gz'
1244 if flags
is not None and hasattr(flags,
'Generator')
and hasattr(flags.Generator,
'avoidExtracting')
and flags.Generator.avoidExtracting:
1245 outputDS =
'tmp_LHE_events.events.gz'
1246 elif runArgs
is not None and hasattr(runArgs,
'avoidExtracting')
and runArgs.avoidExtracting:
1247 outputDS =
'tmp_LHE_events.events.gz'
1249 outputStem = outputDS
1250 if '.tar.gz' in outputDS:
1251 outputStem = outputDS.split(
'.tar.gz')[0]
1252 elif '.tgz' in outputDS:
1253 outputStem = outputDS.split(
'.tgz')[0]
1254 elif '.gz' in outputDS:
1255 outputStem = outputDS.split(
'.gz')[0]
1257 mglog.warning(f
'Could not figure out what output file type {outputDS} refers to')
1258 outputStem = outputDS.split(
'.')[0]
1259 if not outputStem.endswith(
'.events'):
1260 outputStem +=
'.events'
1262 mglog.info(
'Moving file over to '+outputStem)
1263 shutil.move(os.getcwd()+
'/events.lhe',outputStem)
1265 if '.tar.gz' in outputDS
or '.tgz' in outputDS:
1266 mglog.info(
'Re-zipping + tarring into dataset name '+outputDS)
1269 elif '.gz' in outputDS:
1270 mglog.info(
'Re-zipping into dataset name '+outputDS)
1273 compressedOutput = outputStem+
'.gz'
1274 if compressedOutput != outputDS:
1275 shutil.move(compressedOutput,outputDS)
1277 mglog.info(f
'Could not understand output type for {outputDS} - will leave uncompressed')
1280 mglog.info(
'Removing the process directory')
1281 shutil.rmtree(process_dir,ignore_errors=
True)
1283 if os.path.isdir(
'MGC_LHAPDF/'):
1284 shutil.rmtree(
'MGC_LHAPDF/',ignore_errors=
True)
1287 if runArgs
is not None and hasattr(runArgs,
'outputTXTFile')
and runArgs.outputTXTFile
is not None:
1288 outputDS = outputDS.split(
'.TXT')[0]
1290 if runArgs
is not None:
1291 mglog.debug(
'Setting inputGenerator file to '+outputDS)
1292 runArgs.inputGeneratorFile=outputDS
1294 mglog.info(
'All done with output arranging!')
1332 run_card = process_dir+
'/Cards/run_card.dat'
1333 if isinstance(bias_module,tuple):
1334 mglog.info(
'Using bias module '+bias_module[0])
1335 the_run_card = open(run_card,
'r')
1336 for line
in the_run_card:
1337 if 'bias_module' in line
and not bias_module[0]
in line:
1338 raise RuntimeError(
'You need to add the bias module '+bias_module[0]+
' to the run card to actually run it')
1339 the_run_card.close()
1340 if len(bias_module)!=3:
1341 raise RuntimeError(
'Please give a 3-tuple of strings containing bias module name, bias module, and makefile. Alternatively, give path to bias module tarball.')
1342 bias_module_newpath=process_dir+
'/Source/BIAS/'+bias_module[0]
1343 os.makedirs(bias_module_newpath)
1344 bias_module_file=open(bias_module_newpath+
'/'+bias_module[0]+
'.f',
'w')
1345 bias_module_file.write(bias_module[1])
1346 bias_module_file.close()
1347 bias_module_make_file=open(bias_module_newpath+
'/Makefile',
'w')
1348 bias_module_make_file.write(bias_module[2])
1349 bias_module_make_file.close()
1351 mglog.info(
'Using bias module '+bias_module)
1352 bias_module_name=bias_module.split(
'/')[-1].
replace(
'.gz',
'')
1353 bias_module_name=bias_module_name.replace(
'.tar',
'')
1354 the_run_card = open(run_card,
'r')
1355 for line
in the_run_card:
1356 if 'bias_module' in line
and bias_module_name
not in line:
1357 raise RuntimeError(
'You need to add the bias module '+bias_module_name+
' to the run card to actually run it')
1358 the_run_card.close()
1360 if os.path.exists(bias_module+
'.tar.gz'):
1361 bias_module_path=bias_module+
'.tar.gz'
1362 elif os.path.exists(bias_module+
'.gz'):
1363 bias_module_path=bias_module+
'.gz'
1364 elif os.path.exists(bias_module):
1365 bias_module_path=bias_module
1367 mglog.error(
'Did not find bias module '+bias_module+
' , this path should point to folder or tarball. Alternatively give a tuple of strings containing module name, module, and makefile')
1369 bias_module_newpath=process_dir+
'/Source/BIAS/'+bias_module_path.split(
'/')[-1]
1370 mglog.info(
'Copying bias module into place: '+bias_module_newpath)
1371 shutil.copy(bias_module_path,bias_module_newpath)
1372 mglog.info(
'Unpacking bias module')
1373 if bias_module_newpath.endswith(
'.tar.gz'):
1374 untar =
stack_subprocess([
'tar',
'xvzf',bias_module_newpath,
'--directory='+process_dir+
'/Source/BIAS/'])
1376 elif bias_module_path.endswith(
'.gz'):