ATLAS Offline Software
Loading...
Searching...
No Matches
MadGraphUtils.py
Go to the documentation of this file.
1# Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
2
3# Pythonized version of MadGraph steering executables
4# written by Zach Marshall <zach.marshall@cern.ch>
5# updates for aMC@NLO by Josh McFayden <mcfayden@cern.ch>
6# updates to LHE handling and SUSY functionality by Emma Kuwertz <ekuwertz@cern.ch>
7# Attempts to remove path-dependence of MadGraph
8
9import os,time,subprocess,glob,re
10# These Import lines are temporary for backwards compatibility of clients.
11from MCJobOptionUtils.JOsupport import check_reset_proc_number # noqa: F401
12from MCJobOptionUtils.LHAPDFsupport import get_LHAPDF_DATA_PATH # noqa: F401
13from MCJobOptionUtils.LHEsupport import remap_lhe_pdgids # noqa: F401
14from MCJobOptionUtils.LHAPDFsupport import get_lhapdf_id_and_name # noqa: F401
15from MCJobOptionUtils.LHAPDFsupport import get_LHAPDF_PATHS # noqa: F401
16from MCJobOptionUtils.JOsupport import get_physics_short
17from AthenaCommon import Logging
18from MadGraphControl.MGC import MGControl,MADGRAPH_PDFSETTING
19mglog = Logging.logging.getLogger('MadGraphUtils')
20my_MGC_instance = None
21
22# Import that allows transparent migration for current users
23
24# Name of python executable
25python='python'
26# Magic name of gridpack directory
27MADGRAPH_GRIDPACK_LOCATION='madevent'
28# Name for the run (since we only have 1, just needs consistency)
29MADGRAPH_RUN_NAME='run_01'
30# For error handling
31MADGRAPH_CATCH_ERRORS=True
32MADGRAPH_COMMAND_STACK = []
33
34
35import shutil
36
37
38from MadGraphControl.MadGraphUtilsHelpers import error_check,get_mg5_version # noqa: F401
39from MadGraphControl.MadGraphSystematicsUtils import systematics_run_card_options,get_pdf_and_systematic_settings
40from MadGraphControl.MGClassParamHelpers import check_PMG_updates
41
42def stack_subprocess(command,**kwargs):
43 global MADGRAPH_COMMAND_STACK
44 MADGRAPH_COMMAND_STACK += [' '.join(command)]
45 return subprocess.Popen(command,**kwargs)
46
47def generate_prep(process_dir):
48 global MADGRAPH_COMMAND_STACK
49 if not os.access('Cards_bkup',os.R_OK):
50 shutil.copytree(process_dir+'/Cards','Cards_bkup')
51 shutil.copyfile(process_dir+'/Source/make_opts','Cards_bkup/make_opts_bkup')
52 MADGRAPH_COMMAND_STACK += ['# In case this fails, Cards_bkup should be in your original run directory']
53 MADGRAPH_COMMAND_STACK += ['# And ${MGaMC_PROCESS_DIR} can be replaced with whatever process directory exists in your stand-alone test']
54 MADGRAPH_COMMAND_STACK += ['cp '+os.getcwd()+'/Cards_bkup/*dat ${MGaMC_PROCESS_DIR}/Cards/']
55 MADGRAPH_COMMAND_STACK += ['cp '+os.getcwd()+'/Cards_bkup/make_opts_bkup ${MGaMC_PROCESS_DIR}/Source/make_opts']
56 else:
57 mglog.warning('Found Cards_bkup directory existing. Suggests you are either running generation twice (a little funny) or are not using a clean directory.')
58 bkup_v = 1
59 while os.access('Cards_bkup_'+str(bkup_v),os.R_OK) and bkup_v<100:
60 bkup_v += 1
61 if bkup_v<100:
62 shutil.copytree(process_dir+'/Cards','Cards_bkup_'+str(bkup_v))
63 shutil.copyfile(process_dir+'/Source/make_opts','Cards_bkup_'+str(bkup_v)+'/make_opts_bkup')
64 MADGRAPH_COMMAND_STACK += ['# In case this fails, Cards_bkup should be in your original run directory']
65 MADGRAPH_COMMAND_STACK += ['# And ${MGaMC_PROCESS_DIR} can be replaced with whatever process directory exists in your stand-alone test']
66 MADGRAPH_COMMAND_STACK += ['cp '+os.getcwd()+'/Cards_bkup_'+str(bkup_v)+'/*dat ${MGaMC_PROCESS_DIR}/Cards/']
67 MADGRAPH_COMMAND_STACK += ['cp '+os.getcwd()+'/Cards_bkup_'+str(bkup_v)+'/make_opts_bkup ${MGaMC_PROCESS_DIR}/Source/make_opts']
68 else:
69 mglog.warning('Way too many Cards_bkup* directories found. Giving up -- standalone script may not work.')
70
71
72def new_process(process='generate p p > t t~\noutput -f', plugin=None, keepJpegs=False, usePMGSettings=False, pdf_setting=None, devices=None, catch_errors=MADGRAPH_CATCH_ERRORS):
73 global my_MGC_instance
74 my_MGC_instance = MGControl(
75 process,
76 plugin,
77 keepJpegs,
78 usePMGSettings,
79 pdf_setting=pdf_setting,
80 devices=devices,
81 catch_errors=catch_errors,
82 )
83 return my_MGC_instance.process_dir
84
85
87 if my_MGC_instance is not None and hasattr(my_MGC_instance, 'catch_errors'):
88 return bool(my_MGC_instance.catch_errors)
89 return MADGRAPH_CATCH_ERRORS
90
91
92def _write_run_card(runArgs=None, flags=None):
93 global my_MGC_instance # noqa: F824
94 if flags is not None:
95 my_MGC_instance.write_runCard(flags=flags)
96 elif runArgs is not None:
97 my_MGC_instance.write_runCard(runArgs=runArgs)
98 else:
99 my_MGC_instance.write_runCard()
100
101
102def get_pdf_setting(pdf_setting=None):
103 if pdf_setting is not None:
104 return pdf_setting
105 if my_MGC_instance is not None and hasattr(my_MGC_instance, 'pdf_setting'):
106 return my_MGC_instance.pdf_setting
107 return MADGRAPH_PDFSETTING
108
109def get_default_runcard(process_dir=MADGRAPH_GRIDPACK_LOCATION):
110 """ Copy the default runcard from one of several locations
111 to a local file with name run_card.tmp.dat"""
112 output_name = 'run_card.tmp.dat'
113
114 # Get the run card from the installation
115 run_card=process_dir+'/Cards/run_card.dat'
116 if os.access(run_card,os.R_OK):
117 mglog.info('Copying default run_card.dat from '+str(run_card))
118 shutil.copy(run_card,output_name)
119 return output_name
120 else:
121 run_card=process_dir+'/Cards/run_card_default.dat'
122 mglog.info('Fetching default run_card.dat from '+str(run_card))
123 if os.access(run_card,os.R_OK):
124 shutil.copy(run_card,output_name)
125 return output_name
126 else:
127 raise RuntimeError('Cannot find default run_card.dat or run_card_default.dat! I was looking here: %s'%run_card)
128
129
130def generate(process_dir='PROC_mssm_0', grid_pack=False, gridpack_compile=False, extlhapath=None, required_accuracy=0.01, runArgs=None, flags=None, bias_module=None, requirePMGSettings=False, pdf_setting=None):
131 global my_MGC_instance # noqa: F824
132
133
134 # Just in case
135 my_MGC_instance.setup_path_protection()
136
137 # Set consistent mode and number of jobs
138 mode = 0
139 njobs = 1
140 if 'ATHENA_CORE_NUMBER' in os.environ and int(os.environ['ATHENA_CORE_NUMBER'])>0:
141 njobs = int(os.environ['ATHENA_CORE_NUMBER'])
142 mglog.info('Lucky you - you are running on a full node queue. Will re-configure for '+str(njobs)+' jobs.')
143 mode = 2
144
145 cluster_type = get_cluster_type()
146 if cluster_type is not None:
147 mode = 1
148
150 mglog.info('Running event generation from gridpack (using smarter mode from generate() function)')
151 if flags is not None:
152 generate_from_gridpack(flags=flags,extlhapath=extlhapath,gridpack_compile=gridpack_compile,requirePMGSettings=requirePMGSettings,pdf_setting=pdf_setting)
153 else:
154 generate_from_gridpack(runArgs=runArgs,extlhapath=extlhapath,gridpack_compile=gridpack_compile,requirePMGSettings=requirePMGSettings,pdf_setting=pdf_setting)
155 return
156 else:
157 mglog.info('Did not identify an input gridpack.')
158 if grid_pack:
159 mglog.info('The grid_pack flag is set, so I am expecting to create a gridpack in this job')
160
161 # Now get beam energy and random seed out of runArgs or flags
162 beamEnergy,random_seed = get_runArgs_info(runArgs=runArgs, flags=flags)
163
164 # Check if process is NLO or LO
165 isNLO = my_MGC_instance.isNLO
166
167 # Setup PDF and systematics
168 setup_pdf_and_systematic_weights(get_pdf_setting(pdf_setting),my_MGC_instance.runCardDict,isNLO)
169
170 # temporary fix of makefile, needed for 3.3.1., remove in future
171 if isNLO:
172 fix_fks_makefile(process_dir=process_dir)
173
174 # if f2py not available
175 if get_reweight_card(process_dir=process_dir) is not None:
176 if shutil.which('f2py') is not None:
177 mglog.info('Found f2py, will use it for reweighting')
178 else:
179 raise RuntimeError('Could not find f2py, needed for reweighting')
180 check_reweight_card(process_dir)
181
182 global MADGRAPH_COMMAND_STACK
183
184 if grid_pack:
185 #Running in gridpack mode
186 mglog.info('Started generating gridpack at '+str(time.asctime()))
187 mglog.warning(' >>>>>> THIS KIND OF JOB SHOULD ONLY BE RUN LOCALLY - NOT IN GRID JOBS <<<<<<')
188
189 # Some events required if we specify MadSpin usage!
190 my_settings = {'nevents':'1000'}
191
192 if isNLO:
193 my_settings['req_acc']=str(required_accuracy)
194 else:
195 # At LO, no events are generated. That means we need to move the MS card aside and back.
196 LO_has_madspin = False
197 if os.access(f'{process_dir}/Cards/madspin_card.dat',os.R_OK):
198 MADGRAPH_COMMAND_STACK += [f'mv {process_dir}/Cards/madspin_card.dat {process_dir}/Cards/madspin_card.tmp.dat']
199 os.rename(f'{process_dir}/Cards/madspin_card.dat',f'{process_dir}/Cards/madspin_card.tmp.dat')
200 LO_has_madspin = True
201 my_settings = {'gridpack':'true'}
202 my_MGC_instance.runCardDict.update(my_settings)
203 else:
204 #Running in on-the-fly mode
205 mglog.info('Started generating at '+str(time.asctime()))
206
207 mglog.info('Run '+MADGRAPH_RUN_NAME+' will be performed in mode '+str(mode)+' with '+str(njobs)+' jobs in parallel.')
208
209 # Ensure that things are set up normally
210 if not os.access(process_dir,os.R_OK):
211 raise RuntimeError('No process directory found at '+process_dir)
212 if not os.access(process_dir+'/bin/generate_events',os.R_OK):
213 raise RuntimeError('No generate_events module found in '+process_dir)
214
215 mglog.info('For your information, the libraries available are (should include LHAPDF):')
216 ls_dir(process_dir+'/lib')
217
218 setupFastjet(process_dir=process_dir)
219 if bias_module is not None:
220 setup_bias_module(bias_module,process_dir)
221
222 mglog.info('Now I will hack the make files a bit. Apologies, but there seems to be no good way around this.')
223 shutil.copyfile(process_dir+'/Source/make_opts',process_dir+'/Source/make_opts_old')
224 old_opts = open(process_dir+'/Source/make_opts_old','r')
225 new_opts = open(process_dir+'/Source/make_opts','w')
226 for aline in old_opts:
227 if 'FC=g' in aline:
228 mglog.info('Configuring the fancy gfortran compiler instead of g77 / f77')
229 new_opts.write(' FC=gfortran\n')
230 else:
231 new_opts.write(aline)
232 old_opts.close()
233 new_opts.close()
234 mglog.info('Make file hacking complete.')
235
236 # Change directories
237 currdir=os.getcwd()
238 os.chdir(process_dir)
239 # Record the change
240 MADGRAPH_COMMAND_STACK += [ 'cd ${MGaMC_PROCESS_DIR}' ]
241
242 # Check the run card
243 my_MGC_instance.run_card_consistency_check()
244
245
246 # Check the param card
247 code = check_PMG_updates(my_MGC_instance.paramCard)
248 if requirePMGSettings and code!=0:
249 raise RuntimeError('Settings are not compliant with PMG defaults! Please use do_PMG_updates function to get PMG default params.')
250
251 # Build up the generate command
252 # Use the new-style way of passing things: just --name, everything else in config
253 command = [python,'bin/generate_events']
254 if isNLO:
255 command += ['--name='+MADGRAPH_RUN_NAME]
256 mglog.info('Removing Cards/shower_card.dat to ensure we get parton level events only')
257 try:
258 os.unlink('Cards/shower_card.dat')
259 except FileNotFoundError:
260 mglog.info('Cannot find Cards/shower_card.dat.')
261 else:
262 command += [MADGRAPH_RUN_NAME]
263 # Set the number of cores to be used
264 setNCores(process_dir=os.getcwd(), Ncores=njobs)
265 # Special handling for mode 1
266 if mode==1:
267 mglog.info('Setting up cluster running')
268 my_MGC_instance.configCardDict['run_mode'] = 1
269
270 if cluster_type=='pbs':
271 mglog.info('Modifying bin/internal/cluster.py for PBS cluster running')
272 os.system("sed -i \"s:text += prog:text += './'+prog:g\" bin/internal/cluster.py")
273 elif mode==2:
274 mglog.info('Setting up multi-core running on '+os.environ['ATHENA_CORE_NUMBER']+' cores')
275 elif mode==0:
276 mglog.info('Setting up serial generation.')
277
278 # Writing cards to disk
279 my_MGC_instance.write_configCard()
280
281 my_MGC_instance.paramCard.write_paramCard()
282 _write_run_card(runArgs=runArgs, flags=flags)
283
284 print_cards_from_dir(process_dir=my_MGC_instance.process_dir)
285
286
287 generate_prep(process_dir=os.getcwd())
288 generate = stack_subprocess(command,stdin=subprocess.PIPE, stderr=subprocess.PIPE if _should_catch_errors() else None)
289 (out,err) = generate.communicate()
290 error_check(err,generate.returncode)
291
292 # Get back to where we came from
293 os.chdir(currdir)
294 MADGRAPH_COMMAND_STACK += [ 'cd -' ]
295
296 if grid_pack:
297 # Name dictacted by https://twiki.cern.ch/twiki/bin/viewauth/AtlasProtected/PmgMcSoftware
298 # MG version included for traceability (e.g. MG351 for version 3.5.1)
299 energy = '%1.1f'%(beamEnergy*2./1000.)
300 energy = energy.replace('.0','').replace('.','p')
301 gridpack_name='mc_'+energy+'TeV.'+get_physics_short()+'.MG'+get_mg5_version().replace('.','')+'.GRID.tar.gz'
302 mglog.info('Tidying up gridpack '+gridpack_name)
303
304 if not isNLO:
305 # At LO, no events are generated. That means we need to move the MS card aside and back.
306 if LO_has_madspin:
307 MADGRAPH_COMMAND_STACK += [f'mv {process_dir}/Cards/madspin_card.tmp.dat {process_dir}/Cards/madspin_card.dat']
308 os.rename(f'{process_dir}/Cards/madspin_card.tmp.dat',f'{process_dir}/Cards/madspin_card.dat')
309
310
311 MADGRAPH_COMMAND_STACK += ['cp '+glob.glob(process_dir+'/'+MADGRAPH_RUN_NAME+'_*gridpack.tar.gz')[0]+' '+gridpack_name]
312 shutil.copy(glob.glob(process_dir+'/'+MADGRAPH_RUN_NAME+'_*gridpack.tar.gz')[0],gridpack_name)
313
314 if gridpack_compile:
315 MADGRAPH_COMMAND_STACK += ['mkdir tmp%i/'%os.getpid(),'cd tmp%i/'%os.getpid()]
316 os.mkdir('tmp%i/'%os.getpid())
317 os.chdir('tmp%i/'%os.getpid())
318 mglog.info('untar gridpack')
319 untar = stack_subprocess(['tar','xvzf',('../'+gridpack_name)])
320 untar.wait()
321 mglog.info('compile and clean up')
322 MADGRAPH_COMMAND_STACK += ['cd madevent']
323 os.chdir('madevent/')
324 compilep = stack_subprocess(['./bin/compile'],stderr=subprocess.PIPE if _should_catch_errors() else None)
325 (out,err) = compilep.communicate()
326 error_check(err,compilep.returncode)
327 clean = stack_subprocess(['./bin/clean4grid'],stderr=subprocess.PIPE if _should_catch_errors() else None)
328 (out,err) = clean.communicate()
329 error_check(err,clean.returncode)
330 clean.wait()
331 MADGRAPH_COMMAND_STACK += ['cd ..','rm ../'+gridpack_name]
332 os.chdir('../')
333 mglog.info('remove old tarball')
334 os.unlink('../'+gridpack_name)
335 mglog.info('Package up new tarball')
336 tar = stack_subprocess(['tar','--exclude=SubProcesses/P*/G*/*_results.dat','--exclude=SubProcesses/P*/G*/*.log','--exclude=SubProcesses/P*/G*/*.txt','-cvsf','../'+gridpack_name,'.'])
337 tar.wait()
338 MADGRAPH_COMMAND_STACK += ['cd ..','rm -r tmp%i/'%os.getpid()]
339 os.chdir('../')
340 mglog.info('Remove temporary directory')
341 shutil.rmtree('tmp%i/'%os.getpid())
342 mglog.info('Tidying up complete!')
343
344 else:
345 _write_run_card(runArgs=runArgs, flags=flags)
346
347 mglog.info('Package up process_dir')
348 MADGRAPH_COMMAND_STACK += ['mv '+process_dir+' '+MADGRAPH_GRIDPACK_LOCATION]
349 os.rename(process_dir,MADGRAPH_GRIDPACK_LOCATION)
350 tar = stack_subprocess(['tar','--exclude=Events/*/*events*gz','--exclude=SubProcesses/P*/G*/log*txt','--exclude=SubProcesses/P*/G*/events.lhe*','--exclude=*/*.o','--exclude=*/*/*.o','--exclude=*/*/*/*.o','--exclude=*/*/*/*/*.o','-czf',gridpack_name,MADGRAPH_GRIDPACK_LOCATION])
351 tar.wait()
352 MADGRAPH_COMMAND_STACK += ['mv '+MADGRAPH_GRIDPACK_LOCATION+' '+process_dir]
353 os.rename(MADGRAPH_GRIDPACK_LOCATION,process_dir)
354
355 mglog.info('Gridpack sucessfully created, exiting the transform')
356 output_txt_file = get_output_txt_file(runArgs=runArgs, flags=flags)
357 if output_txt_file:
358 mglog.info('Touching output TXT (LHE) file for the transform')
359 open(output_txt_file, 'w').close()
360 from AthenaCommon.AppMgr import theApp
361 theApp.finalize()
362 theApp.exit()
363
364 mglog.info('Finished at '+str(time.asctime()))
365 return 0
366
367
368def generate_from_gridpack(runArgs=None, flags=None, extlhapath=None, gridpack_compile=None, requirePMGSettings=False, pdf_setting=None):
369 global my_MGC_instance # noqa: F824
370
371 # Get beam energy and random seed out of runArgs or flags
372 beamEnergy,random_seed = get_runArgs_info(runArgs=runArgs, flags=flags)
373
374 # Just in case
375 my_MGC_instance.setup_path_protection()
376
377 isNLO = my_MGC_instance.isNLO
378
379 setupFastjet(process_dir=MADGRAPH_GRIDPACK_LOCATION)
380
381 # This is hard-coded as a part of MG5_aMC :'(
382 gridpack_run_name = 'GridRun_'+str(random_seed)
383
384 # Ensure that we only do madspin at the end
385 if os.access(MADGRAPH_GRIDPACK_LOCATION+'/Cards/madspin_card.dat',os.R_OK):
386 os.rename(MADGRAPH_GRIDPACK_LOCATION+'/Cards/madspin_card.dat',MADGRAPH_GRIDPACK_LOCATION+'/Cards/backup_madspin_card.dat')
387 do_madspin=True
388 else:
389 do_madspin=False
390
391 if get_reweight_card(process_dir=MADGRAPH_GRIDPACK_LOCATION) is not None:
392 check_reweight_card(MADGRAPH_GRIDPACK_LOCATION)
393
394 # Check the param card
395 code = check_PMG_updates(my_MGC_instance.paramCard)
396 if requirePMGSettings and code!=0:
397 raise RuntimeError('Settings are not compliant with PMG defaults! Please use do_PMG_updates function to get PMG default params.')
398
399 # Modify run card, then print
400 settings={'iseed':str(random_seed)}
401 if not isNLO:
402 settings['python_seed']=str(random_seed)
403 my_MGC_instance.runCardDict.update(settings)
404
405 mglog.info('Generating events from gridpack')
406
407 # Ensure that things are set up normally
408 if not os.path.exists(MADGRAPH_GRIDPACK_LOCATION):
409 raise RuntimeError('Gridpack directory not found at '+MADGRAPH_GRIDPACK_LOCATION)
410
411 nevents = my_MGC_instance.runCardDict['nevents']
412 mglog.info('>>>> FOUND GRIDPACK <<<< <- This will be used for generation')
413 mglog.info('Generation of '+str(int(nevents))+' events will be performed using the supplied gridpack with random seed '+str(random_seed))
414 mglog.info('Started generating events at '+str(time.asctime()))
415
416 #Remove addmasses if it's there
417 if os.access(MADGRAPH_GRIDPACK_LOCATION+'/bin/internal/addmasses.py',os.R_OK):
418 os.remove(MADGRAPH_GRIDPACK_LOCATION+'/bin/internal/addmasses.py')
419
420 currdir=os.getcwd()
421
422 # Make sure we've set the number of processes appropriately
423 setNCores(process_dir=MADGRAPH_GRIDPACK_LOCATION)
424
425 # Run the consistency check, print some useful info
426 ls_dir(currdir)
427 ls_dir(MADGRAPH_GRIDPACK_LOCATION)
428
429 # Update the run card according to consistency checks
430 my_MGC_instance.run_card_consistency_check()
431
432 # Now all done with updates, so print the cards with the final settings
433 print_cards_from_dir(process_dir=MADGRAPH_GRIDPACK_LOCATION)
434
435 if isNLO:
436 #turn off systematics for gridpack generation and store settings for standalone run
437 systematics_settings=None
438 if my_MGC_instance.runCardDict.get('systematics_program',None) == 'systematics':
439 if not my_MGC_instance.runCardDict.get('store_rwgt_info',None):
440 raise RuntimeError('Trying to run NLO systematics but reweight info not stored')
441 if 'systematics_arguments' in my_MGC_instance.runCardDict:
442 systematics_settings=MadGraphControl.MadGraphSystematicsUtils.parse_systematics_arguments(my_MGC_instance.runCardDict['systematics_arguments'])
443 else:
444 systematics_settings={}
445 mglog.info('Turning off systematics for now, running standalone later')
446
447 # Writing run card to disk
448 _write_run_card(runArgs=runArgs, flags=flags)
449 my_MGC_instance.write_configCard()
450 my_MGC_instance.paramCard.write_paramCard()
451
452 global MADGRAPH_COMMAND_STACK
453 if not isNLO:
454
455 if not os.access(MADGRAPH_GRIDPACK_LOCATION+'/bin/gridrun',os.R_OK):
456 mglog.error('/bin/gridrun not found at '+MADGRAPH_GRIDPACK_LOCATION)
457 raise RuntimeError('Could not find gridrun executable')
458 else:
459 mglog.info('Found '+MADGRAPH_GRIDPACK_LOCATION+'/bin/gridrun, starting generation.')
460 generate_prep(MADGRAPH_GRIDPACK_LOCATION)
461 granularity=1
462 mg5_version_tuple = tuple(map(int, get_mg5_version().split(".")))
463 if mg5_version_tuple[:3] > (3, 6, 2):
464 nprocs = int(os.environ['ATHENA_CORE_NUMBER'])
465 # Hard-coded for now to match previous defaults; to be configured programmaticaly
466 maxevts = 2500
467 mglog.info("Now generating {} events with random seed {}, granularity {}, nprocs {}, and maxevts {}".format(int(nevents),int(random_seed),granularity,nprocs,maxevts))
468 else:
469 mglog.info("Now generating {} events with random seed {} and granularity {}".format(int(nevents),int(random_seed),granularity))
470 # not sure whether this is needed but it is done in the old "run.sh" script
471 new_ld_path=":".join([os.environ['LD_LIBRARY_PATH'],os.getcwd()+'/'+MADGRAPH_GRIDPACK_LOCATION+'/madevent/lib',os.getcwd()+'/'+MADGRAPH_GRIDPACK_LOCATION+'/HELAS/lib'])
472 os.environ['LD_LIBRARY_PATH']=new_ld_path
473 MADGRAPH_COMMAND_STACK+=["export LD_LIBRARY_PATH="+":".join(['${LD_LIBRARY_PATH}',new_ld_path])]
474 if mg5_version_tuple[:3] > (3, 6, 2):
475 generate = stack_subprocess([python,MADGRAPH_GRIDPACK_LOCATION+'/bin/gridrun',str(int(nevents)),str(int(random_seed)),str(granularity),str(nprocs),str(maxevts)],stdin=subprocess.PIPE,stderr=subprocess.PIPE if _should_catch_errors() else None)
476 else:
477 generate = stack_subprocess([python,MADGRAPH_GRIDPACK_LOCATION+'/bin/gridrun',str(int(nevents)),str(int(random_seed)),str(granularity)],stdin=subprocess.PIPE,stderr=subprocess.PIPE if _should_catch_errors() else None)
478 (out,err) = generate.communicate()
479 error_check(err,generate.returncode)
480 gp_events=MADGRAPH_GRIDPACK_LOCATION+"/Events/GridRun_{}/unweighted_events.lhe.gz".format(int(random_seed))
481 if not os.path.exists(gp_events):
482 mglog.error('Error in gp generation, did not find events at '+gp_events)
483
484 # add reweighting, which is not run automatically from LO GPs
485 reweight_card=get_reweight_card(MADGRAPH_GRIDPACK_LOCATION)
486 if reweight_card is not None:
487 pythonpath_backup=os.environ['PYTHONPATH']
488 # workaround as madevent crashes when path to mg in PYTHONPATH
489 os.environ['PYTHONPATH']=':'.join([p for p in pythonpath_backup.split(':') if 'madgraph5amc' not in p])
490 add_reweighting('GridRun_{}'.format(int(random_seed)))
491 os.environ['PYTHONPATH']=pythonpath_backup
492
493 shutil.move(gp_events,'events.lhe.gz')
494
495 else:
496
497 if not os.access(MADGRAPH_GRIDPACK_LOCATION+'/bin/generate_events',os.R_OK):
498 raise RuntimeError('Could not find generate_events executable at '+MADGRAPH_GRIDPACK_LOCATION)
499 else:
500 mglog.info('Found '+MADGRAPH_GRIDPACK_LOCATION+'/bin/generate_events, starting generation.')
501
502 ls_dir(MADGRAPH_GRIDPACK_LOCATION+'/Events/')
503 if os.access(MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name, os.F_OK):
504 mglog.info('Removing '+MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name+' directory from gridpack generation')
505 MADGRAPH_COMMAND_STACK += ['rm -rf '+MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name]
506 shutil.rmtree(MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name)
507
508 # Delete events generated when setting up MadSpin during gridpack generation
509 if os.access(MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name+'_decayed_1', os.F_OK):
510 mglog.info('Removing '+MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name+'_decayed_1 directory from gridpack generation')
511 MADGRAPH_COMMAND_STACK += ['rm -rf '+MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name+'_decayed_1']
512 shutil.rmtree(MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name+'_decayed_1')
513
514 ls_dir(MADGRAPH_GRIDPACK_LOCATION+'/Events/')
515
516 if not gridpack_compile:
517 mglog.info('Copying make_opts from Template')
518 shutil.copy(os.environ['MADPATH']+'/Template/LO/Source/make_opts',MADGRAPH_GRIDPACK_LOCATION+'/Source/')
519
520 generate_prep(MADGRAPH_GRIDPACK_LOCATION)
521 generate = stack_subprocess([python,MADGRAPH_GRIDPACK_LOCATION+'/bin/generate_events','--parton','--nocompile','--only_generation','-f','--name='+gridpack_run_name],stdin=subprocess.PIPE,stderr=subprocess.PIPE if _should_catch_errors() else None)
522 (out,err) = generate.communicate()
523 error_check(err,generate.returncode)
524 else:
525 mglog.info('Allowing recompilation of gridpack')
526 if os.path.islink(MADGRAPH_GRIDPACK_LOCATION+'/lib/libLHAPDF.a'):
527 mglog.info('Unlinking '+MADGRAPH_GRIDPACK_LOCATION+'/lib/libLHAPDF.a')
528 os.unlink(MADGRAPH_GRIDPACK_LOCATION+'/lib/libLHAPDF.a')
529
530 generate_prep(MADGRAPH_GRIDPACK_LOCATION)
531 generate = stack_subprocess([python,MADGRAPH_GRIDPACK_LOCATION+'/bin/generate_events','--parton','--only_generation','-f','--name='+gridpack_run_name],stdin=subprocess.PIPE,stderr=subprocess.PIPE if _should_catch_errors() else None)
532 (out,err) = generate.communicate()
533 error_check(err,generate.returncode)
534 if isNLO and systematics_settings is not None:
535 # run systematics
536 mglog.info('Running systematics standalone')
537 systematics_path=MADGRAPH_GRIDPACK_LOCATION+'/bin/internal/systematics.py'
538 events_location=MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name+'/events.lhe.gz'
539
540 syst_cmd=[python,systematics_path]+[events_location]*2+["--"+k+"="+systematics_settings[k] for k in systematics_settings]
541 mglog.info('running: '+' '.join(syst_cmd))
542 systematics = stack_subprocess(syst_cmd)
543 systematics.wait()
544
545 # See if MG5 did the job for us already
546 if not os.access('events.lhe.gz',os.R_OK):
547 mglog.info('Copying generated events to '+currdir)
548 if not os.path.exists(MADGRAPH_GRIDPACK_LOCATION+'Events/'+gridpack_run_name):
549 shutil.copy(MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name+'/events.lhe.gz','events.lhe.gz')
550 else:
551 mglog.info('Events were already in place')
552
553 ls_dir(currdir)
554
555 mglog.info('Moving generated events to be in correct format for arrange_output().')
556 mglog.info('Unzipping generated events.')
557 unzip = stack_subprocess(['gunzip','-f','events.lhe.gz'])
558 unzip.wait()
559
560 mglog.info('Moving file over to '+MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name+'/unweighted_events.lhe')
561 mkdir = stack_subprocess(['mkdir','-p',(MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name)])
562 mkdir.wait()
563 shutil.move('events.lhe',MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name+'/unweighted_events.lhe')
564
565 mglog.info('Re-zipping into dataset name '+MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name+'/unweighted_events.lhe.gz')
566 rezip = stack_subprocess(['gzip',MADGRAPH_GRIDPACK_LOCATION+'/Events/'+gridpack_run_name+'/unweighted_events.lhe'])
567 rezip.wait()
568
569 os.chdir(currdir)
570
571 # Now consider MadSpin:
572 if do_madspin:
573 # Move card back
574 os.rename(MADGRAPH_GRIDPACK_LOCATION+'/Cards/backup_madspin_card.dat',MADGRAPH_GRIDPACK_LOCATION+'/Cards/madspin_card.dat')
575 mglog.info('Decaying with MadSpin.')
577
578 mglog.info('Finished at '+str(time.asctime()))
579
580 return 0
581
582
583def setupFastjet(process_dir=None):
584 global my_MGC_instance # noqa: F824
585
586 isNLO = my_MGC_instance.isNLO
587
588 mglog.info('Path to fastjet install dir: '+os.environ['FASTJETPATH'])
589 fastjetconfig = os.environ['FASTJETPATH']+'/bin/fastjet-config'
590
591 mglog.info('fastjet-config --version: '+str(subprocess.Popen([fastjetconfig, '--version'],stdout = subprocess.PIPE).stdout.read().strip()))
592 mglog.info('fastjet-config --prefix: '+str(subprocess.Popen([fastjetconfig, '--prefix'],stdout = subprocess.PIPE).stdout.read().strip()))
593
594 if not isNLO:
595 config_card=process_dir+'/Cards/me5_configuration.txt'
596 else:
597 config_card=process_dir+'/Cards/amcatnlo_configuration.txt'
598
599 oldcard = open(config_card,'r')
600 newcard = open(config_card+'.tmp','w')
601
602 for line in oldcard:
603 if 'fastjet = ' in line:
604 newcard.write('fastjet = '+fastjetconfig+'\n')
605 mglog.info('Setting fastjet = '+fastjetconfig+' in '+config_card)
606 else:
607 newcard.write(line)
608 oldcard.close()
609 newcard.close()
610 shutil.move(config_card+'.tmp',config_card)
611
612 return
613
614def get_runArgs_info(runArgs=None, flags=None):
615 """Return beam energy and random seed from runArgs or flags.
616 Adding flags compatibility while clients migrate.
617 """
618 global my_MGC_instance # noqa: F824
619 if runArgs is not None:
620 if flags is not None:
621 mglog.warning('Both runArgs and flags were provided to get_runArgs_info. Using flags.')
622 my_MGC_instance.get_flags_info(flags)
623 else:
624 my_MGC_instance.get_runArgs_info(runArgs)
625 elif flags is not None:
626 my_MGC_instance.get_flags_info(flags)
627 else:
628 raise RuntimeError('Must provide runArgs or flags to get run information.')
629
630 return my_MGC_instance.beamEnergy, my_MGC_instance.random_seed
631
632
633def get_output_txt_file(runArgs=None, flags=None):
634 """Return output TXT file path from runArgs or flags if available."""
635 if flags is not None:
636 if flags.Output.TXTFileName:
637 return flags.Output.TXTFileName
638 if runArgs is not None and hasattr(runArgs, 'outputTXTFile'):
639 if runArgs.outputTXTFile:
640 return runArgs.outputTXTFile
641 return None
642
643
644def setupLHAPDF(process_dir=None, extlhapath=None, allow_links=True):
645 global my_MGC_instance # noqa: F824
646
647 isNLO = my_MGC_instance.isNLO
648
649 origLHAPATH=os.environ['LHAPATH']
650 origLHAPDF_DATA_PATH=os.environ['LHAPDF_DATA_PATH']
651
652 LHAPATH,LHADATAPATH=get_LHAPDF_PATHS()
653
654 pdfname=''
655 pdfid=-999
656
657
658 mydict= my_MGC_instance.runCardDict
659 if mydict["pdlabel"].replace("'","") == 'lhapdf':
660 #Make local LHAPDF dir
661 mglog.info('creating local LHAPDF dir: MGC_LHAPDF/')
662 if os.path.islink('MGC_LHAPDF/'):
663 os.unlink('MGC_LHAPDF/')
664 elif os.path.isdir('MGC_LHAPDF/'):
665 shutil.rmtree('MGC_LHAPDF/')
666
667 newMGCLHA='MGC_LHAPDF/'
668
669 mkdir = subprocess.Popen(['mkdir','-p',newMGCLHA])
670 mkdir.wait()
671
672 pdfs_used=[ int(x) for x in mydict['lhaid'].replace(' ',',').split(',') ]
673 # included systematics pdfs here
674 if 'sys_pdf' in mydict:
675 sys_pdf=mydict['sys_pdf'].replace('&&',' ').split()
676 for s in sys_pdf:
677 if s.isdigit():
678 idx=int(s)
679 if idx>1000: # the sys_pdf syntax is such that small numbers are used to specify the subpdf index
680 pdfs_used.append(idx)
681 else:
682 pdfs_used.append(s)
683 if 'systematics_arguments' in mydict:
684 systematics_arguments=MadGraphControl.MadGraphSystematicsUtils.parse_systematics_arguments(mydict['systematics_arguments'])
685 if 'pdf' in systematics_arguments:
686 sys_pdf=systematics_arguments['pdf'].replace(',',' ').replace('@',' ').split()
687 for s in sys_pdf:
688 if s.isdigit():
689 idx=int(s)
690 if idx>1000: # the sys_pdf syntax is such that small numbers are used to specify the subpdf index
691 pdfs_used.append(idx)
692 else:
693 pdfs_used.append(s)
694 for pdf in pdfs_used:
695 if isinstance(pdf,str) and (pdf.lower()=='errorset' or pdf.lower()=='central'):
696 continue
697 # new function to get both lhapdf id and name
698 pdfid,pdfname=get_lhapdf_id_and_name(pdf)
699 mglog.info("Found LHAPDF ID="+str(pdfid)+", name="+pdfname)
700
701 if not os.path.exists(newMGCLHA+pdfname) and not os.path.lexists(newMGCLHA+pdfname):
702 if not os.path.exists(LHADATAPATH+'/'+pdfname):
703 mglog.warning('PDF not installed at '+LHADATAPATH+'/'+pdfname)
704 if allow_links:
705 mglog.info('linking '+LHADATAPATH+'/'+pdfname+' --> '+newMGCLHA+pdfname)
706 os.symlink(LHADATAPATH+'/'+pdfname,newMGCLHA+pdfname)
707 else:
708 mglog.info('copying '+LHADATAPATH+'/'+pdfname+' --> '+newMGCLHA+pdfname)
709 shutil.copytree(LHADATAPATH+'/'+pdfname,newMGCLHA+pdfname)
710
711 if allow_links:
712 mglog.info('linking '+LHADATAPATH+'/pdfsets.index --> '+newMGCLHA+'pdfsets.index')
713 os.symlink(LHADATAPATH+'/pdfsets.index',newMGCLHA+'pdfsets.index')
714
715 atlasLHADATAPATH=LHADATAPATH.replace('sft.cern.ch/lcg/external/lhapdfsets/current','atlas.cern.ch/repo/sw/Generators/lhapdfsets/current')
716 mglog.info('linking '+atlasLHADATAPATH+'/lhapdf.conf --> '+newMGCLHA+'lhapdf.conf')
717 os.symlink(atlasLHADATAPATH+'/lhapdf.conf',newMGCLHA+'lhapdf.conf')
718 else:
719 mglog.info('copying '+LHADATAPATH+'/pdfsets.index --> '+newMGCLHA+'pdfsets.index')
720 shutil.copy2(LHADATAPATH+'/pdfsets.index',newMGCLHA+'pdfsets.index')
721
722 atlasLHADATAPATH=LHADATAPATH.replace('sft.cern.ch/lcg/external/lhapdfsets/current','atlas.cern.ch/repo/sw/Generators/lhapdfsets/current')
723 mglog.info('copying '+atlasLHADATAPATH+'/lhapdf.conf -->'+newMGCLHA+'lhapdf.conf')
724 shutil.copy2(atlasLHADATAPATH+'/lhapdf.conf',newMGCLHA+'lhapdf.conf')
725
726
727 LHADATAPATH=os.getcwd()+'/MGC_LHAPDF'
728
729 else:
730 mglog.info('Not using LHAPDF')
731 return (LHAPATH,origLHAPATH,origLHAPDF_DATA_PATH)
732
733
734 if isNLO:
735 os.environ['LHAPDF_DATA_PATH']=LHADATAPATH
736
737 mglog.info('Path to LHAPDF install dir: '+LHAPATH)
738 mglog.info('Path to LHAPDF data dir: '+LHADATAPATH)
739 if not os.path.isdir(LHADATAPATH):
740 raise RuntimeError('LHAPDF data dir not accesible: '+LHADATAPATH)
741 if not os.path.isdir(LHAPATH):
742 raise RuntimeError('LHAPDF path dir not accesible: '+LHAPATH)
743
744 # Dealing with LHAPDF
745 if extlhapath:
746 lhapdfconfig=extlhapath
747 if not os.access(lhapdfconfig,os.X_OK):
748 raise RuntimeError('Failed to find valid external lhapdf-config at '+lhapdfconfig)
749 LHADATAPATH=subprocess.Popen([lhapdfconfig, '--datadir'],stdout = subprocess.PIPE).stdout.read().strip()
750 mglog.info('Changing LHAPDF_DATA_PATH to '+LHADATAPATH)
751 os.environ['LHAPDF_DATA_PATH']=LHADATAPATH
752 else:
753 getlhaconfig = subprocess.Popen(['get_files','-data','lhapdf-config'])
754 getlhaconfig.wait()
755 #Get custom lhapdf-config
756 if not os.access(os.getcwd()+'/lhapdf-config',os.X_OK):
757 mglog.error('Failed to get lhapdf-config from MadGraphControl')
758 return 1
759 lhapdfconfig = os.getcwd()+'/lhapdf-config'
760
761 mglog.info('lhapdf-config --version: '+str(subprocess.Popen([lhapdfconfig, '--version'],stdout = subprocess.PIPE).stdout.read().strip()))
762 mglog.info('lhapdf-config --prefix: '+str(subprocess.Popen([lhapdfconfig, '--prefix'],stdout = subprocess.PIPE).stdout.read().strip()))
763 mglog.info('lhapdf-config --libdir: '+str(subprocess.Popen([lhapdfconfig, '--libdir'],stdout = subprocess.PIPE).stdout.read().strip()))
764 mglog.info('lhapdf-config --datadir: '+str(subprocess.Popen([lhapdfconfig, '--datadir'],stdout = subprocess.PIPE).stdout.read().strip()))
765 mglog.info('lhapdf-config --pdfsets-path: '+str(subprocess.Popen([lhapdfconfig, '--pdfsets-path'],stdout = subprocess.PIPE).stdout.read().strip()))
766
767
768 my_MGC_instance.configCardDict.update({'lhapdf':lhapdfconfig,'lhapdf_py3':lhapdfconfig})
769
770 mglog.info('Creating links for LHAPDF')
771 if os.path.islink(process_dir+'/lib/PDFsets'):
772 os.unlink(process_dir+'/lib/PDFsets')
773 elif os.path.isdir(process_dir+'/lib/PDFsets'):
774 shutil.rmtree(process_dir+'/lib/PDFsets')
775 if allow_links:
776 os.symlink(LHADATAPATH,process_dir+'/lib/PDFsets')
777 else:
778 shutil.copytree(LHADATAPATH,process_dir+'/lib/PDFsets')
779 mglog.info('Available PDFs are:')
780 mglog.info( sorted( [ x for x in os.listdir(process_dir+'/lib/PDFsets') if ".tar.gz" not in x ] ) )
781
782 global MADGRAPH_COMMAND_STACK
783 MADGRAPH_COMMAND_STACK += [ '# Copy the LHAPDF files locally' ]
784 MADGRAPH_COMMAND_STACK += [ 'cp -r '+os.getcwd()+'/MGC_LHAPDF .' ]
785 MADGRAPH_COMMAND_STACK += [ 'cp -r '+process_dir+'/lib/PDFsets ${MGaMC_PROCESS_DIR}/lib/' ]
786
787 return (LHAPATH,origLHAPATH,origLHAPDF_DATA_PATH)
788
789
790# Function to set the number of cores and the running mode in the run card
791def setNCores(process_dir, Ncores=None):
792 global my_MGC_instance # noqa: F824
793
794 my_Ncores = Ncores
795 my_runMode = 2 if 'ATHENA_CORE_NUMBER' in os.environ else 0
796 if Ncores is None and 'ATHENA_CORE_NUMBER' in os.environ and int(os.environ['ATHENA_CORE_NUMBER'])>0:
797 my_Ncores = int(os.environ['ATHENA_CORE_NUMBER'])
798 my_runMode = 2
799 if my_Ncores is None:
800 mglog.info('Setting up for serial run')
801 my_Ncores = 1
802 my_MGC_instance.configCardDict.update({'nb_core':my_Ncores,'run_mode':my_runMode,'automatic_html_opening':'False'})
803
804
805
806
807
808
810 madpath=os.environ['MADPATH']
811 if not os.access(madpath+'/bin/mg5_aMC',os.R_OK):
812 raise RuntimeError('mg5_aMC executable not found in '+madpath)
813 return madpath+'/bin/mg5_aMC'
814
815
816def add_lifetimes(process_dir,threshold=None):
817 """ Add lifetimes to the generated LHE file. Should be
818 called after generate_events is called.
819 """
820
821 me_exec=get_mg5_executable()
822
823 if len(glob.glob(process_dir+'/Events/*'))<1:
824 mglog.error('Process dir '+process_dir+' does not contain events?')
825 run = glob.glob(process_dir+'/Events/*')[0].split('/')[-1]
826
827 # Note : This slightly clunky implementation is needed for the time being
828 # See : https://answers.launchpad.net/mg5amcnlo/+question/267904
829
830 tof_c = open('time_of_flight_exec_card','w')
831 tof_c.write('launch '+process_dir+''' -i
832add_time_of_flight '''+run+((' --threshold='+str(threshold)) if threshold is not None else ''))
833 tof_c.close()
834
835 mglog.info('Started adding time of flight info '+str(time.asctime()))
836
837 generate = stack_subprocess([python,me_exec,'time_of_flight_exec_card'],stdin=subprocess.PIPE,stderr=subprocess.PIPE if _should_catch_errors() else None)
838 (out,err) = generate.communicate()
839 error_check(err,generate.returncode)
840
841 mglog.info('Finished adding time of flight information at '+str(time.asctime()))
842
843 # Re-zip the file if needed
844 lhe_gz = glob.glob(process_dir+'/Events/*/*lhe.gz')[0]
845 if not os.access(lhe_gz,os.R_OK):
846 mglog.info('LHE file needs to be zipped')
847 lhe = glob.glob(process_dir+'/Events/*/*lhe.gz')[0]
848 rezip = stack_subprocess(['gzip',lhe])
849 rezip.wait()
850 mglog.info('Zipped')
851 else:
852 mglog.info('LHE file zipped by MadGraph automatically. Nothing to do')
853
854 return True
855
856
857def add_madspin(madspin_card=None):
858 """ Run madspin on the generated LHE file. Should be
859 run when you have inputGeneratorFile set.
860 Only requires a simplified process with the same model that you are
861 interested in (needed to set up a process directory for MG5_aMC)
862 """
863 global my_MGC_instance # noqa: F824
864 me_exec=get_mg5_executable()
865 process_dir = my_MGC_instance.process_dir
866
867 if madspin_card is not None:
868 shutil.copyfile(madspin_card,process_dir+'/Cards/madspin_card.dat')
869
870 if len(glob.glob(process_dir+'/Events/*'))<1:
871 mglog.error('Process dir '+process_dir+' does not contain events?')
872 proc_dir_list = glob.glob(process_dir+'/Events/*')
873 run=None
874 for adir in proc_dir_list:
875 if 'GridRun_' in adir:
876 run=adir.split('/')[-1]
877 break
878 else:
879 run=proc_dir_list[0].split('/')[-1]
880
881 # Note : This slightly clunky implementation is needed for the time being
882 # See : https://answers.launchpad.net/mg5amcnlo/+question/267904
883
884 ms_c = open('madspin_exec_card','w')
885 ms_c.write('launch '+process_dir+''' -i
886decay_events '''+run)
887 ms_c.close()
888
889 mglog.info('Started running madspin at '+str(time.asctime()))
890
891 generate = stack_subprocess([python,me_exec,'madspin_exec_card'],stdin=subprocess.PIPE,stderr=subprocess.PIPE if _should_catch_errors() else None)
892 (out,err) = generate.communicate()
893 error_check(err,generate.returncode)
894 if len(glob.glob(process_dir+'/Events/'+run+'_decayed_*/')) == 0:
895 mglog.error('No '+process_dir+'/Events/'+run+'_decayed_*/ can be found')
896 raise RuntimeError('Problem while running MadSpin')
897
898 mglog.info('Finished running madspin at '+str(time.asctime()))
899
900 # Re-zip the file if needed
901 lhe_gz = glob.glob(process_dir+'/Events/*/*lhe.gz')[0]
902 if not os.access(lhe_gz,os.R_OK):
903 mglog.info('LHE file needs to be zipped')
904 lhe = glob.glob(process_dir+'/Events/*/*lhe.gz')[0]
905 rezip = stack_subprocess(['gzip',lhe])
906 rezip.wait()
907 mglog.info('Zipped')
908 else:
909 mglog.info('LHE file zipped by MadGraph automatically. Nothing to do')
910
911
912def madspin_on_lhe(input_LHE,madspin_card,runArgs=None,keep_original=False):
913 """ Run MadSpin on an input LHE file. Takes the process
914 from the LHE file, so you don't need to have a process directory
915 set up in advance. Runs MadSpin and packs the LHE file up appropriately
916 Needs runArgs for the file handling"""
917 if not os.access(input_LHE,os.R_OK):
918 raise RuntimeError('Could not find LHE file '+input_LHE)
919 if not os.access(madspin_card,os.R_OK):
920 raise RuntimeError('Could not find input MadSpin card '+madspin_card)
921 if keep_original:
922 shutil.copy(input_LHE,input_LHE+'.original')
923 mglog.info('Put backup copy of LHE file at '+input_LHE+'.original')
924 # Start writing the card for execution
925 madspin_exec_card = open('madspin_exec_card','w')
926 madspin_exec_card.write('import '+input_LHE+'\n')
927 # Based on the original card
928 input_madspin_card = open(madspin_card,'r')
929 has_launch = False
930 for l in input_madspin_card.readlines():
931 commands = l.split('#')[0].split()
932 # Skip import of a file name that isn't our file
933 if len(commands)>1 and 'import'==commands[0] and not 'model'==commands[1]:
934 continue
935 # Check for a launch command
936 if len(commands)>0 and 'launch' == commands[0]:
937 has_launch = True
938 madspin_exec_card.write(l.strip()+'\n')
939 if not has_launch:
940 madspin_exec_card.write('launch\n')
941 madspin_exec_card.close()
942 input_madspin_card.close()
943 # Now get the madspin executable
944 madpath=os.environ['MADPATH']
945 if not os.access(madpath+'/MadSpin/madspin',os.R_OK):
946 raise RuntimeError('madspin executable not found in '+madpath)
947 mglog.info('Starting madspin at '+str(time.asctime()))
948 generate = stack_subprocess([python,madpath+'/MadSpin/madspin','madspin_exec_card'],stdin=subprocess.PIPE,stderr=subprocess.PIPE if _should_catch_errors() else None)
949 (out,err) = generate.communicate()
950 error_check(err,generate.returncode)
951 mglog.info('Done with madspin at '+str(time.asctime()))
952 # Should now have a re-zipped LHE file
953 # We now have to do a shortened version of arrange_output below
954 # Clean up in case a link or file was already there
955 if os.path.exists(os.getcwd()+'/events.lhe'):
956 os.remove(os.getcwd()+'/events.lhe')
957
958 mglog.info('Unzipping generated events.')
959 unzip = stack_subprocess(['gunzip','-f',input_LHE+'.gz'])
960 unzip.wait()
961
962 mglog.info('Putting a copy in place for the transform.')
963 mod_output = open(os.getcwd()+'/events.lhe','w')
964
965 #Removing empty lines in LHE
966 nEmpty=0
967 with open(input_LHE,'r') as fileobject:
968 for line in fileobject:
969 if line.strip():
970 mod_output.write(line)
971 else:
972 nEmpty=nEmpty+1
973 mod_output.close()
974
975 mglog.info('Removed '+str(nEmpty)+' empty lines from LHEF')
976
977 # Actually move over the dataset - this first part is horrible...
978 if runArgs is None:
979 raise RuntimeError('Must provide runArgs to madspin_on_lhe')
980
981 outputDS = runArgs.outputTXTFile if hasattr(runArgs,'outputTXTFile') else 'tmp_LHE_events.tar.gz'
982
983 mglog.info('Moving file over to '+outputDS.split('.tar.gz')[0]+'.events')
984 shutil.move(os.getcwd()+'/events.lhe',outputDS.split('.tar.gz')[0]+'.events')
985
986 mglog.info('Re-zipping into dataset name '+outputDS)
987 rezip = stack_subprocess(['tar','cvzf',outputDS,outputDS.split('.tar.gz')[0]+'.events'])
988 rezip.wait()
989
990 # shortening the outputDS in the case of an output TXT file
991 if hasattr(runArgs,'outputTXTFile') and runArgs.outputTXTFile is not None:
992 outputDS = outputDS.split('.TXT')[0]
993 # Do some fixing up for them
994 if runArgs is not None:
995 mglog.debug('Setting inputGenerator file to '+outputDS)
996 runArgs.inputGeneratorFile=outputDS
997
998
999def arrange_output(process_dir=MADGRAPH_GRIDPACK_LOCATION,lhe_version=None,saveProcDir=False,runArgs=None,flags=None,fixEventWeightsForBridgeMode=False,pdf_setting=None):
1000
1001 # NLO is not *really* the question here, we need to know if we should look for weighted or
1002 # unweighted events in the output directory. MadSpin (above) only seems to give weighted
1003 # results for now?
1004 if len(glob.glob(os.path.join(process_dir, 'Events','*')))<1:
1005 mglog.error('Process dir '+process_dir+' does not contain events?')
1006 proc_dir_list = glob.glob(os.path.join(process_dir, 'Events', '*'))
1007 this_run_name=None
1008 # looping over possible directories to find the right one
1009 for adir in proc_dir_list:
1010 if 'decayed' in adir:# skipping '*decayed*' directories produced by MadSpin, will be picked later if they exist
1011 continue
1012 else:
1013 if 'GridRun_' in adir:
1014 this_run_name=adir
1015 break # GridRun_* directories have priority
1016 elif os.path.join(process_dir, 'Events',MADGRAPH_RUN_NAME) in adir:
1017 this_run_name=adir
1018 if not os.access(this_run_name,os.R_OK):
1019 raise RuntimeError('Unable to locate run directory')
1020
1021 hasUnweighted = os.access(this_run_name+'/unweighted_events.lhe.gz',os.R_OK)
1022
1023 hasRunMadSpin=False
1024 madspinDirs=sorted(glob.glob(this_run_name+'_decayed_*/'))
1025 if len(madspinDirs):
1026 hasRunMadSpin=True
1027 if hasRunMadSpin and not hasUnweighted:
1028 # check again:
1029 hasUnweighted = os.access(madspinDirs[-1]+'/unweighted_events.lhe.gz',os.R_OK)
1030
1031 global MADGRAPH_COMMAND_STACK
1032 if hasRunMadSpin:
1033 if len(madspinDirs):
1034 if hasUnweighted:
1035 # so this is a bit of a mess now...
1036 # if madspin is run from an NLO grid pack the correct lhe events are at both
1037 # madevent/Events/run_01/unweighted_events.lhe.gz
1038 # and madevent/Events/run_01_decayed_1/events.lhe.gz
1039 # so there are unweighted events but not in the madspinDir...
1040 if os.path.exists(madspinDirs[-1]+'/unweighted_events.lhe.gz'):
1041 MADGRAPH_COMMAND_STACK += ['mv '+madspinDirs[-1]+'/unweighted_events.lhe.gz'+' '+this_run_name+'/unweighted_events.lhe.gz']
1042 shutil.move(madspinDirs[-1]+'/unweighted_events.lhe.gz',this_run_name+'/unweighted_events.lhe.gz')
1043 mglog.info('Moving MadSpin events from '+madspinDirs[-1]+'/unweighted_events.lhe.gz to '+this_run_name+'/unweighted_events.lhe.gz')
1044 elif os.path.exists(madspinDirs[-1]+'/events.lhe.gz'):
1045 MADGRAPH_COMMAND_STACK += ['mv '+madspinDirs[-1]+'/events.lhe.gz'+' '+this_run_name+'/unweighted_events.lhe.gz']
1046 shutil.move(madspinDirs[-1]+'/events.lhe.gz',this_run_name+'/unweighted_events.lhe.gz')
1047 mglog.info('Moving MadSpin events from '+madspinDirs[-1]+'/events.lhe.gz to '+this_run_name+'/unweighted_events.lhe.gz')
1048 else:
1049 raise RuntimeError('MadSpin was run but can\'t find files :(')
1050
1051 else:
1052 MADGRAPH_COMMAND_STACK += ['mv '+madspinDirs[-1]+'/events.lhe.gz '+this_run_name+'/events.lhe.gz']
1053 shutil.move(madspinDirs[-1]+'/events.lhe.gz',this_run_name+'/events.lhe.gz')
1054 mglog.info('Moving MadSpin events from '+madspinDirs[-1]+'/events.lhe.gz to '+this_run_name+'/events.lhe.gz')
1055
1056 else:
1057 mglog.error('MadSpin was run but can\'t find output folder '+(this_run_name+'_decayed_1/'))
1058 raise RuntimeError('MadSpin was run but can\'t find output folder '+(this_run_name+'_decayed_1/'))
1059
1060 if fixEventWeightsForBridgeMode:
1061 mglog.info("Fixing event weights after MadSpin... initial checks.")
1062
1063 # get the cross section from the undecayed LHE file
1064 spinmodenone=False
1065 MGnumevents=-1
1066 MGintweight=-1
1067
1068 if hasUnweighted:
1069 eventsfilename="unweighted_events"
1070 else:
1071 eventsfilename="events"
1072 unzip = stack_subprocess(['gunzip','-f',this_run_name+'/%s.lhe.gz' % eventsfilename])
1073 unzip.wait()
1074
1075 for line in open(process_dir+'/Events/'+MADGRAPH_RUN_NAME+'/%s.lhe'%eventsfilename):
1076 if "Number of Events" in line:
1077 sline=line.split()
1078 MGnumevents=int(sline[-1])
1079 elif "Integrated weight (pb)" in line:
1080 sline=line.split()
1081 MGintweight=float(sline[-1])
1082 elif "set spinmode none" in line:
1083 spinmodenone=True
1084 elif "</header>" in line:
1085 break
1086
1087 if spinmodenone and MGnumevents>0 and MGintweight>0:
1088 mglog.info("Fixing event weights after MadSpin... modifying LHE file.")
1089 newlhe=open(this_run_name+'/%s_fixXS.lhe'%eventsfilename,'w')
1090 initlinecount=0
1091 eventlinecount=0
1092 inInit=False
1093 inEvent=False
1094
1095 # new default for MG 2.6.1+ (https://its.cern.ch/jira/browse/AGENE-1725)
1096 # but verified from LHE below.
1097 event_norm_setting="average"
1098
1099 for line in open(this_run_name+'/%s.lhe'%eventsfilename):
1100
1101 newline=line
1102 if "<init>" in line:
1103 inInit=True
1104 initlinecount=0
1105 elif "</init>" in line:
1106 inInit=False
1107 elif inInit and initlinecount==0:
1108 initlinecount=1
1109 # check event_norm setting in LHE file, deteremines how Pythia interprets event weights
1110 sline=line.split()
1111 if abs(int(sline[-2])) == 3:
1112 event_norm_setting="sum"
1113 elif abs(int(sline[-2])) == 4:
1114 event_norm_setting="average"
1115 elif inInit and initlinecount==1:
1116 sline=line.split()
1117 # update the global XS info
1118 relunc=float(sline[1])/float(sline[0])
1119 sline[0]=str(MGintweight)
1120 sline[1]=str(float(sline[0])*relunc)
1121 if event_norm_setting=="sum":
1122 sline[2]=str(MGintweight/MGnumevents)
1123 elif event_norm_setting=="average":
1124 sline[2]=str(MGintweight)
1125 newline=' '.join(sline)
1126 newline+="\n"
1127 initlinecount+=1
1128 elif inInit and initlinecount>1:
1129 initlinecount+=1
1130 elif "<event>" in line:
1131 inEvent=True
1132 eventlinecount=0
1133 elif "</event>" in line:
1134 inEvent=False
1135 elif inEvent and eventlinecount==0:
1136 sline=line.split()
1137 # next change the per-event weights
1138 if event_norm_setting=="sum":
1139 sline[2]=str(MGintweight/MGnumevents)
1140 elif event_norm_setting=="average":
1141 sline[2]=str(MGintweight)
1142 newline=' '.join(sline)
1143 newline+="\n"
1144 eventlinecount+=1
1145 newlhe.write(newline)
1146 newlhe.close()
1147
1148 mglog.info("Fixing event weights after MadSpin... cleaning up.")
1149 shutil.copyfile(this_run_name+'/%s.lhe' % eventsfilename,
1150 this_run_name+'/%s_badXS.lhe' % eventsfilename)
1151
1152 shutil.move(this_run_name+'/%s_fixXS.lhe' % eventsfilename,
1153 this_run_name+'/%s.lhe' % eventsfilename)
1154
1155 rezip = stack_subprocess(['gzip',this_run_name+'/%s.lhe' % eventsfilename])
1156 rezip.wait()
1157
1158 rezip = stack_subprocess(['gzip',this_run_name+'/%s_badXS.lhe' % eventsfilename])
1159 rezip.wait()
1160
1161 # Clean up in case a link or file was already there
1162 if os.path.exists(os.getcwd()+'/events.lhe'):
1163 os.remove(os.getcwd()+'/events.lhe')
1164
1165 mglog.info('Unzipping generated events.')
1166 if hasUnweighted:
1167 unzip = stack_subprocess(['gunzip','-f',this_run_name+'/unweighted_events.lhe.gz'])
1168 unzip.wait()
1169 else:
1170 unzip = stack_subprocess(['gunzip','-f',this_run_name+'/events.lhe.gz'])
1171 unzip.wait()
1172
1173 mglog.info('Putting a copy in place for the transform.')
1174 if hasUnweighted:
1175 orig_input = this_run_name+'/unweighted_events.lhe'
1176 mod_output = open(os.getcwd()+'/events.lhe','w')
1177 else:
1178 orig_input = this_run_name+'/events.lhe'
1179 mod_output = open(os.getcwd()+'/events.lhe','w')
1180
1181 #Removing empty lines and bad comments in LHE
1182 #and check for existence of weights
1183 initrwgt=None
1184 nEmpty=0
1185 lhe_weights=[]
1186 with open(orig_input,'r') as fileobject:
1187 for line in fileobject:
1188 if line.strip():
1189 # search for bad characters (neccessary until at least MG5 2.8.1)
1190 newline=line
1191 if '#' not in newline:
1192 newline=newline
1193 elif '>' not in newline[ newline.find('#'): ]:
1194 newline=newline
1195 else:
1196 mglog.info('Found bad LHE line with an XML mark in a comment: "'+newline.strip()+'"')
1197 newline=newline[:newline.find('#')]+'#'+ (newline[newline.find('#'):].replace('>','-'))
1198 # check for weightnames that should exist, simplify nominal weight names
1199 if initrwgt is False:
1200 pass
1201 elif "</initrwgt>" in newline:
1202 initrwgt=False
1203 elif "<initrwgt>" in newline:
1204 initrwgt=True
1205 elif initrwgt is not None:
1206 newline=newline.replace('_DYNSCALE-1','')
1207 if '</weight>' in newline:
1208 iend=newline.find('</weight>')
1209 istart=newline[:iend].rfind('>')
1210 lhe_weights+=[newline[istart+1:iend].strip()]
1211 mod_output.write(newline)
1212 else:
1213 nEmpty=nEmpty+1
1214 mod_output.close()
1215 mglog.info('Removed '+str(nEmpty)+' empty lines from LHEF')
1216
1217 mglog.info("The following "+str(len(lhe_weights))+" weights have been written to the LHE file: "+",".join(lhe_weights))
1218 expected_weights=get_expected_reweight_names(get_reweight_card(process_dir))
1219 expected_weights+=get_expected_systematic_names(get_pdf_setting(pdf_setting))
1220 mglog.info("Checking whether the following expected weights are in LHE file: "+",".join(expected_weights))
1221 for w in expected_weights:
1222 if w not in lhe_weights:
1223 raise RuntimeError("Did not find expected weight "+w+" in lhe file. Did the reweight or systematics module crash?")
1224 mglog.info("Found all required weights!")
1225
1226 if lhe_version:
1227 mod_output2 = open(os.getcwd()+'/events.lhe','r')
1228 test=mod_output2.readline()
1229 if 'version="' in test:
1230 mglog.info('Applying LHE version hack')
1231 final_file = open(os.getcwd()+'/events.lhe.copy','w')
1232 final_file.write('<LesHouchesEvents version="%i.0">\n'%lhe_version)
1233 shutil.copyfileobj(mod_output2, final_file)
1234 final_file.close()
1235 shutil.copy(os.getcwd()+'/events.lhe.copy',os.getcwd()+'/events.lhe')
1236 # Clean up after ourselves
1237 os.remove(os.getcwd()+'/events.lhe.copy')
1238 mod_output2.close()
1239
1240 # Actually move over the dataset
1241 outputDS = get_output_txt_file(runArgs=runArgs, flags=flags)
1242 if outputDS is None:
1243 outputDS = 'tmp_LHE_events.tar.gz'
1244 if flags is not None and hasattr(flags, 'Generator') and hasattr(flags.Generator, 'avoidExtracting') and flags.Generator.avoidExtracting:
1245 outputDS = 'tmp_LHE_events.events.gz'
1246 elif runArgs is not None and hasattr(runArgs, 'avoidExtracting') and runArgs.avoidExtracting:
1247 outputDS = 'tmp_LHE_events.events.gz'
1248
1249 outputStem = outputDS
1250 if '.tar.gz' in outputDS:
1251 outputStem = outputDS.split('.tar.gz')[0]
1252 elif '.tgz' in outputDS:
1253 outputStem = outputDS.split('.tgz')[0]
1254 elif '.gz' in outputDS:
1255 outputStem = outputDS.split('.gz')[0]
1256 else:
1257 mglog.warning(f'Could not figure out what output file type {outputDS} refers to')
1258 outputStem = outputDS.split('.')[0]
1259 if not outputStem.endswith('.events'):
1260 outputStem += '.events'
1261
1262 mglog.info('Moving file over to '+outputStem)
1263 shutil.move(os.getcwd()+'/events.lhe',outputStem)
1264
1265 if '.tar.gz' in outputDS or '.tgz' in outputDS:
1266 mglog.info('Re-zipping + tarring into dataset name '+outputDS)
1267 rezip = stack_subprocess(['tar','cvzf',outputDS,outputStem])
1268 rezip.wait()
1269 elif '.gz' in outputDS:
1270 mglog.info('Re-zipping into dataset name '+outputDS)
1271 rezip = stack_subprocess(['gzip',outputStem])
1272 rezip.wait()
1273 compressedOutput = outputStem+'.gz'
1274 if compressedOutput != outputDS:
1275 shutil.move(compressedOutput,outputDS)
1276 else:
1277 mglog.info(f'Could not understand output type for {outputDS} - will leave uncompressed')
1278
1279 if not saveProcDir:
1280 mglog.info('Removing the process directory')
1281 shutil.rmtree(process_dir,ignore_errors=True)
1282
1283 if os.path.isdir('MGC_LHAPDF/'):
1284 shutil.rmtree('MGC_LHAPDF/',ignore_errors=True)
1285
1286 # shortening the outputDS in the case of an output LHE file
1287 if runArgs is not None and hasattr(runArgs,'outputTXTFile') and runArgs.outputTXTFile is not None:
1288 outputDS = outputDS.split('.TXT')[0]
1289 # Do some fixing up for them
1290 if runArgs is not None:
1291 mglog.debug('Setting inputGenerator file to '+outputDS)
1292 runArgs.inputGeneratorFile=outputDS
1293
1294 mglog.info('All done with output arranging!')
1295 return outputDS
1296
1297def get_expected_reweight_names(reweight_card_loc):
1298 if reweight_card_loc is None:
1299 return []
1300 names=[]
1301 f_rw=open(reweight_card_loc)
1302 for line in f_rw:
1303 if 'launch' not in line:
1304 continue
1305 match=re.match(r'launch.*--rwgt_info\s*=\s*(\S+).*',line.strip())
1306 if len(match.groups())!=1:
1307 raise RuntimeError('Unexpected format of reweight card in line'+line)
1308 else:
1309 names+=[match.group(1)]
1310 f_rw.close()
1311 return names
1312
1314 names=[]
1315 if syst_setting is None or 'central_pdf' not in syst_setting:
1316 mglog.warning("Systematics have not been defined via base fragment or explicit PDF settings; cannot check for expected weights")
1317 return []
1318 if 'pdf_variations' in syst_setting and isinstance(syst_setting['pdf_variations'],list):
1319 names+=[MadGraphControl.MadGraphSystematicsUtils.SYSTEMATICS_WEIGHT_INFO%{'mur':1.0,'muf':1.0,'pdf':syst_setting['central_pdf']}]
1320 for pdf in syst_setting['pdf_variations']:
1321 names+=[MadGraphControl.MadGraphSystematicsUtils.SYSTEMATICS_WEIGHT_INFO%{'mur':1.0,'muf':1.0,'pdf':pdf+1}]
1322 if 'alternative_pdfs' in syst_setting and isinstance(syst_setting['alternative_pdfs'],list):
1323 for pdf in syst_setting['alternative_pdfs']:
1324 names+=[MadGraphControl.MadGraphSystematicsUtils.SYSTEMATICS_WEIGHT_INFO%{'mur':1.0,'muf':1.0,'pdf':pdf}]
1325 if 'scale_variations' in syst_setting and isinstance(syst_setting['scale_variations'],list):
1326 for mur in syst_setting['scale_variations']:
1327 for muf in syst_setting['scale_variations']:
1328 names+=[MadGraphControl.MadGraphSystematicsUtils.SYSTEMATICS_WEIGHT_INFO%{'mur':mur,'muf':muf,'pdf':syst_setting['central_pdf']}]
1329 return names
1330
1331def setup_bias_module(bias_module,process_dir):
1332 run_card = process_dir+'/Cards/run_card.dat'
1333 if isinstance(bias_module,tuple):
1334 mglog.info('Using bias module '+bias_module[0])
1335 the_run_card = open(run_card,'r')
1336 for line in the_run_card:
1337 if 'bias_module' in line and not bias_module[0] in line:
1338 raise RuntimeError('You need to add the bias module '+bias_module[0]+' to the run card to actually run it')
1339 the_run_card.close()
1340 if len(bias_module)!=3:
1341 raise RuntimeError('Please give a 3-tuple of strings containing bias module name, bias module, and makefile. Alternatively, give path to bias module tarball.')
1342 bias_module_newpath=process_dir+'/Source/BIAS/'+bias_module[0]
1343 os.makedirs(bias_module_newpath)
1344 bias_module_file=open(bias_module_newpath+'/'+bias_module[0]+'.f','w')
1345 bias_module_file.write(bias_module[1])
1346 bias_module_file.close()
1347 bias_module_make_file=open(bias_module_newpath+'/Makefile','w')
1348 bias_module_make_file.write(bias_module[2])
1349 bias_module_make_file.close()
1350 else:
1351 mglog.info('Using bias module '+bias_module)
1352 bias_module_name=bias_module.split('/')[-1].replace('.gz','')
1353 bias_module_name=bias_module_name.replace('.tar','')
1354 the_run_card = open(run_card,'r')
1355 for line in the_run_card:
1356 if 'bias_module' in line and bias_module_name not in line:
1357 raise RuntimeError('You need to add the bias module '+bias_module_name+' to the run card to actually run it')
1358 the_run_card.close()
1359
1360 if os.path.exists(bias_module+'.tar.gz'):
1361 bias_module_path=bias_module+'.tar.gz'
1362 elif os.path.exists(bias_module+'.gz'):
1363 bias_module_path=bias_module+'.gz'
1364 elif os.path.exists(bias_module):
1365 bias_module_path=bias_module
1366 else:
1367 mglog.error('Did not find bias module '+bias_module+' , this path should point to folder or tarball. Alternatively give a tuple of strings containing module name, module, and makefile')
1368 return 1
1369 bias_module_newpath=process_dir+'/Source/BIAS/'+bias_module_path.split('/')[-1]
1370 mglog.info('Copying bias module into place: '+bias_module_newpath)
1371 shutil.copy(bias_module_path,bias_module_newpath)
1372 mglog.info('Unpacking bias module')
1373 if bias_module_newpath.endswith('.tar.gz'):
1374 untar = stack_subprocess(['tar','xvzf',bias_module_newpath,'--directory='+process_dir+'/Source/BIAS/'])
1375 untar.wait()
1376 elif bias_module_path.endswith('.gz'):
1377 gunzip = stack_subprocess(['gunzip',bias_module_newpath])
1378 gunzip.wait()
1379
1380
1381def get_reweight_card(process_dir=MADGRAPH_GRIDPACK_LOCATION):
1382 if os.access(process_dir+'/Cards/reweight_card.dat',os.R_OK):
1383 return process_dir+'/Cards/reweight_card.dat'
1384 return None
1385
1386
1387def check_reweight_card(process_dir=MADGRAPH_GRIDPACK_LOCATION):
1388 reweight_card=get_reweight_card(process_dir=process_dir)
1389 shutil.move(reweight_card,reweight_card+'.old')
1390 oldcard = open(reweight_card+'.old','r')
1391 newcard = open(reweight_card,'w')
1392 changed = False
1393 info_expression=r'launch.*--rwgt_info\s*=\s*(\S+).*'
1394 name_expression=info_expression.replace('info','name')
1395 goodname_expression=r'^[A-Za-z0-9_\-.]+$'
1396 for line in oldcard:
1397 # we are only interested in the 'launch' line
1398 if not line.strip().startswith('launch') :
1399 newcard.write(line)
1400 else:
1401 rwgt_name_match=re.match(name_expression,line.strip())
1402 rwgt_info_match=re.match(info_expression,line.strip())
1403 if rwgt_name_match is None and rwgt_info_match is None:
1404 raise RuntimeError('Every reweighting should have a --rwgt_info (see https://cp3.irmp.ucl.ac.be/projects/madgraph/wiki/Reweight), please update your reweight_card accordingly. Line to fix: '+line)
1405 for match in [rwgt_info_match,rwgt_name_match]:
1406 if match is None:
1407 continue
1408 if len(match.groups())!=1:
1409 raise RuntimeError('Unexpected format of reweight card in line: '+line)
1410 if not re.match(goodname_expression,match.group(1)):
1411 raise RuntimeError('No special character in reweighting info/name, only allowing '+goodname_expression)
1412 if rwgt_info_match is not None:
1413 newcard.write(line)
1414 elif rwgt_name_match is not None:
1415 newcard.write(line.strip()+' --rwgt_info={0}\n'.format(rwgt_name_match.group(1)))
1416 changed=True
1417 if changed:
1418 mglog.info('Updated reweight_card')
1419 newcard.close()
1420 oldcard.close()
1421
1422
1423def update_lhe_file(lhe_file_old,param_card_old=None,lhe_file_new=None,masses={},delete_old_lhe=True):
1424 """Build a new LHE file from an old one and an updated param card.
1425 The masses of some particles can be changed via the masses dictionary. No particles that appear in the events
1426 may have their masses changed.
1427 If the param card is provided, the decay block in the LHE file will be replaced with the one in the param card.
1428 By default, the old LHE file is removed.
1429 If None is provided as a new LHE file name, the new file will replace the old one."""
1430 # If we want to just use a temp file, then put in a little temp holder
1431 lhe_file_new_tmp = lhe_file_new if lhe_file_new is not None else lhe_file_old+'.tmp'
1432 # Make sure the LHE file is there
1433 if not os.access(lhe_file_old,os.R_OK):
1434 raise RuntimeError('Could not access old LHE file at '+str(lhe_file_old)+'. Please check the file location.')
1435 # Grab the old param card
1436 if param_card_old is not None:
1437 paramcard = subprocess.Popen(['get_files','-data',param_card_old])
1438 paramcard.wait()
1439 if not os.access(param_card_old,os.R_OK):
1440 raise RuntimeError('Could not get param card '+param_card_old)
1441 # Don't overwrite old param cards
1442 if os.access(lhe_file_new_tmp,os.R_OK):
1443 raise RuntimeError('Old file at'+str(lhe_file_new_tmp)+' in the current directory. Dont want to clobber it. Please move it first.')
1444
1445 newlhe = open(lhe_file_new_tmp,'w')
1446 blockName = None
1447 decayEdit = False
1448 eventRead = False
1449 particles_in_events = []
1450 # Decay block ends with </slha>
1451
1452 with open(lhe_file_old,'r') as fileobject:
1453 for line in fileobject:
1454 if decayEdit and '</slha>' not in line:
1455 continue
1456 if decayEdit and '</slha>' in line:
1457 decayEdit = False
1458 if line.strip().upper().startswith('BLOCK') or line.strip().upper().startswith('DECAY')\
1459 and len(line.strip().split()) > 1:
1460 pos = 0 if line.strip().startswith('DECAY') else 1
1461 blockName = line.strip().upper().split()[pos]
1462
1463 akey = None
1464 if blockName != 'DECAY' and len(line.strip().split()) > 0:
1465 akey = line.strip().split()[0]
1466 elif blockName == 'DECAY' and len(line.strip().split()) > 1:
1467 akey = line.strip().split()[1]
1468
1469 # Replace the masses with those in the dictionary
1470 if akey is not None and blockName == 'MASS' and akey in masses:
1471 newlhe.write(' '+akey+' '+str(masses[akey])+' # \n')
1472 mglog.info(' '+akey+' '+str(masses[akey])+' #')
1473 decayEdit = False
1474 continue
1475
1476 # Replace the entire decay section of the LHE file with the one from the param card
1477 if blockName == 'DECAY' and param_card_old is not None:
1478 # We are now reading the decay blocks! Take them from the param card
1479 oldparam = open(param_card_old,'r')
1480 newDecays = False
1481 for old_line in oldparam.readlines():
1482 newBlockName = None
1483 if old_line.strip().upper().startswith('DECAY') and len(old_line.strip().split()) > 1:
1484 newBlockName = line.strip().upper().split()[pos]
1485 if newDecays:
1486 newlhe.write(old_line)
1487 elif newBlockName == 'DECAY':
1488 newDecays = True
1489 newlhe.write(old_line)
1490 oldparam.close()
1491 # Done adding the decays
1492 decayEdit = True
1493 blockName = None
1494 continue
1495
1496 # Keep a record of the particles that are in the events
1497 if not eventRead and '<event>' in line:
1498 eventRead = True
1499 if eventRead:
1500 if len(line.split())==11:
1501 aparticle = line.split()[0]
1502 if aparticle not in particles_in_events:
1503 particles_in_events += [aparticle]
1504
1505 # Otherwise write the line again
1506 newlhe.write(line)
1507
1508 # Check that none of the particles that we were setting the masses of appear in the LHE events
1509 for akey in masses:
1510 if akey in particles_in_events:
1511 mglog.error('Attempted to change mass of a particle that was in an LHE event! This is not allowed!')
1512 return -1
1513
1514 # Close up and return
1515 newlhe.close()
1516
1517 # Move the new file to the old file location
1518 if lhe_file_new is None:
1519 os.remove(lhe_file_old)
1520 shutil.move(lhe_file_new_tmp,lhe_file_old)
1521 lhe_file_new_tmp = lhe_file_old
1522 # Delete the old file if requested
1523 elif delete_old_lhe:
1524 os.remove(lhe_file_old)
1525
1526 return lhe_file_new_tmp
1527
1528
1529
1530def print_cards_from_dir(process_dir=MADGRAPH_GRIDPACK_LOCATION):
1531 card_dir=process_dir+'/Cards/'
1532 print_cards(proc_card=card_dir+'proc_card_mg5.dat',run_card=card_dir+'run_card.dat',param_card=card_dir+'param_card.dat',\
1533 madspin_card=card_dir+'madspin_card.dat',reweight_card=card_dir+'reweight_card.dat',warn_on_missing=False)
1534
1535
1536def print_cards(proc_card='proc_card_mg5.dat',run_card=None,param_card=None,madspin_card=None,reweight_card=None,warn_on_missing=True):
1537 if os.access(proc_card,os.R_OK):
1538 mglog.info("proc_card:")
1539 procCard = subprocess.Popen(['cat',proc_card])
1540 procCard.wait()
1541 elif warn_on_missing:
1542 mglog.warning('No proc_card: '+proc_card+' found')
1543
1544 if run_card is not None and os.access(run_card,os.R_OK):
1545 mglog.info("run_card:")
1546 runCard = subprocess.Popen(['cat',run_card])
1547 runCard.wait()
1548 elif run_card is not None and warn_on_missing:
1549 mglog.warning('No run_card: '+run_card+' found')
1550 else:
1551 mglog.info('Default run card in use')
1552
1553 if param_card is not None and os.access(param_card,os.R_OK):
1554 mglog.info("param_card:")
1555 paramCard = subprocess.Popen(['cat',param_card])
1556 paramCard.wait()
1557 elif param_card is not None and warn_on_missing:
1558 mglog.warning('No param_card: '+param_card+' found')
1559 else:
1560 mglog.info('Default param card in use')
1561
1562 if madspin_card is not None and os.access(madspin_card,os.R_OK):
1563 mglog.info("madspin_card:")
1564 madspinCard = subprocess.Popen(['cat',madspin_card])
1565 madspinCard.wait()
1566 elif madspin_card is not None and warn_on_missing:
1567 mglog.warning('No madspin_card: '+madspin_card+' found')
1568 else:
1569 mglog.info('No madspin card in use')
1570
1571 if reweight_card is not None and os.access(reweight_card,os.R_OK):
1572 mglog.info("reweight_card:")
1573 madspinCard = subprocess.Popen(['cat',reweight_card])
1574 madspinCard.wait()
1575 elif reweight_card is not None and warn_on_missing:
1576 mglog.warning('No reweight_card: '+reweight_card+' found')
1577 else:
1578 mglog.info('No reweight card in use')
1579
1580
1582 """ Simple function for checking if there is a grid pack.
1583 Relies on the specific location of the unpacked gridpack (madevent)
1584 which is here set as a global variable. The gridpack is untarred by
1585 the transform (Gen_tf.py) and no sign is sent to the job itself
1586 that there is a gridpack in use except the file's existence"""
1587 if os.access(MADGRAPH_GRIDPACK_LOCATION,os.R_OK):
1588 mglog.info('Located input grid pack area')
1589 return True
1590 return False
1591
1592
1593
1594def modify_run_card(run_card_input=None,run_card_backup=None,process_dir=MADGRAPH_GRIDPACK_LOCATION,runArgs=None,flags=None,settings={},skipBaseFragment=False,pdf_setting=None):
1595 """This is a legacy function, rather use the functions outlined in MGC.py
1596 Build a new run_card.dat from an existing one.
1597 This function can get a fresh runcard from DATAPATH or start from the process directory.
1598 Settings is a dictionary of keys (no spaces needed) and values to replace.
1599 """
1600
1601 global my_MGC_instance # noqa: F824
1602 if my_MGC_instance is None:
1603 my_MGC_instance = MGControl()
1604
1605 #update the run card dictionary
1606 my_MGC_instance.runCardDict.update(settings)
1607
1608
1609
1610
1611def modify_config_card(config_card_backup=None,process_dir=MADGRAPH_GRIDPACK_LOCATION,settings={},set_commented=True):
1612 """This is a legacy function, rather use the functions outlined in MGC.py
1613 Build a new configuration from an existing one.
1614 This function can get a fresh runcard from DATAPATH or start from the process directory.
1615 Settings is a dictionary of keys (no spaces needed) and values to replace.
1616 """
1617 global my_MGC_instance # noqa: F824
1618
1619 my_MGC_instance.configCardDict.update(settings)
1620
1621def modify_param_card(param_card_input=None,param_card_backup=None,process_dir=MADGRAPH_GRIDPACK_LOCATION,params={},output_location=None):
1622 """This is a legacy function, rather use the functions outlined in MGC.py
1623 Build a new param_card.dat from an existing one.
1624 Params should be a dictionary of dictionaries. The first key is the block name, and the second in the param name.
1625 Keys can include MASS (for masses) and DECAY X (for decays of particle X)"""
1626
1627 global my_MGC_instance # noqa: F824
1628 #update the paramCardDict that is a part of the MGC class
1629 my_MGC_instance.paramCard.modify_paramCardDict(params = params)
1630 # If they asked for a copy, give them a copy
1631 if output_location is not None:
1632 my_MGC_instance.paramCard.write_paramCard(output_location)
1633
1634
1636 global my_MGC_instance # noqa: F824
1637 if 'cluster_type' in my_MGC_instance.configCardDict:
1638 return my_MGC_instance.configCardDict['cluster_type']
1639 else:
1640 return None
1641
1642
1643
1644
1645def add_reweighting(run_name,reweight_card=None,process_dir=MADGRAPH_GRIDPACK_LOCATION):
1646 mglog.info('Running reweighting module on existing events')
1647 if reweight_card is not None:
1648 mglog.info('Copying new reweight card from '+reweight_card)
1649 shutil.move(reweight_card,process_dir+'/Cards/reweight_card.dat')
1650 reweight_cmd='{}/bin/madevent reweight {} -f'.format(process_dir,run_name)
1651 reweight = stack_subprocess([python]+reweight_cmd.split(),stdin=subprocess.PIPE,stderr=subprocess.PIPE if _should_catch_errors() else None)
1652 (out,err) = reweight.communicate()
1653 error_check(err,reweight.returncode)
1654 mglog.info('Finished reweighting')
1655
1656
1657
1658
1659def ls_dir(directory):
1660 mglog.info('For your information, ls of '+directory+':')
1661 mglog.info( sorted( os.listdir( directory ) ) )
1662
1663# Final import of some code used in these functions
1664import MadGraphControl.MadGraphSystematicsUtils
1665
1666# To be removed once we moved past MG5 3.3.1
1667def fix_fks_makefile(process_dir):
1668 makefile_fks=process_dir+'/SubProcesses/makefile_fks_dir'
1669 mglog.info('Fixing '+makefile_fks)
1670 shutil.move(makefile_fks,makefile_fks+'_orig')
1671 fin=open(makefile_fks+'_orig')
1672 fout=open(makefile_fks,'w')
1673 edit=False
1674 for line in fin:
1675 if 'FKSParams.mod' in line:
1676 fout.write(line.replace('FKSParams.mod','FKSParams.o'))
1677 edit=True
1678 elif edit and 'driver_mintFO' in line:
1679 fout.write('driver_mintFO.o: weight_lines.o mint_module.o FKSParams.o\n')
1680 elif edit and 'genps_fks.o' in line:
1681 fout.write('genps_fks.o: mint_module.o FKSParams.o\n')
1682 elif edit and 'test_soft_col_limits' in line:
1683 fout.write(line)
1684 fout.write('madfks_plot.o: mint_module.o\n')
1685 fout.write('cluster.o: weight_lines.o\n')
1686 else:
1687 fout.write(line)
1688 fin.close()
1689 fout.close()
1690
1691#==================================================================================
1692# this function is called during build_run card to check the consistency of user-provided arguments with the inlude
1693# and throw errors, warnings, or corrects the input as is appropriate
1694def setup_pdf_and_systematic_weights(the_base_fragment,extras,isNLO):
1695
1696 global my_MGC_instance # noqa: F824
1697
1698
1699 list = []
1700 tmp_dict = {}
1701 for k in extras:
1702 k_clean=k.lower().replace("'",'').replace('"','')
1703 if k_clean!=k and k_clean in systematics_run_card_options(isNLO):
1704 list.append(k)
1705 tmp_dict[k_clean] = extras[k]
1706 # Removing systematics with incorrect formatting
1707 for o in list:
1708 if o in extras:
1709 extras.pop(o,None)
1710 # Adding cleaned up systematics into dictionary
1711 extras.update(tmp_dict)
1712
1713 if my_MGC_instance.base_fragment_setup_check(the_base_fragment,extras,isNLO):
1714 return
1715
1716 new_settings=get_pdf_and_systematic_settings(the_base_fragment,isNLO)
1717
1718 user_set_extras=dict(extras)
1719 for s in new_settings:
1720 if s is not None:
1721 extras[s]=new_settings[s]
1722
1723
1724 mglog.info('PDF and scale settings were set as follows:')
1725 for p in systematics_run_card_options(isNLO):
1726 user_set='not set'
1727 if p in user_set_extras:
1728 user_set=str(user_set_extras[p])
1729 new_value='not set'
1730 if p in extras:
1731 new_value=str(extras[p])
1732 mglog.info('MadGraphUtils set '+str(p)+' to "'+new_value+'", was set to "'+user_set+'"')
int upper(int c)
STL class.
std::string replace(std::string s, const std::string &s2, const std::string &s3)
Definition hcg.cxx:312
std::vector< std::string > split(const std::string &s, const std::string &t=":")
Definition hcg.cxx:179
get_pdf_setting(pdf_setting=None)
setupLHAPDF(process_dir=None, extlhapath=None, allow_links=True)
add_madspin(madspin_card=None)
get_expected_systematic_names(syst_setting)
new_process(process='generate p p > t t~\noutput -f', plugin=None, keepJpegs=False, usePMGSettings=False, pdf_setting=None, devices=None, catch_errors=MADGRAPH_CATCH_ERRORS)
get_runArgs_info(runArgs=None, flags=None)
update_lhe_file(lhe_file_old, param_card_old=None, lhe_file_new=None, masses={}, delete_old_lhe=True)
print_cards(proc_card='proc_card_mg5.dat', run_card=None, param_card=None, madspin_card=None, reweight_card=None, warn_on_missing=True)
modify_param_card(param_card_input=None, param_card_backup=None, process_dir=MADGRAPH_GRIDPACK_LOCATION, params={}, output_location=None)
arrange_output(process_dir=MADGRAPH_GRIDPACK_LOCATION, lhe_version=None, saveProcDir=False, runArgs=None, flags=None, fixEventWeightsForBridgeMode=False, pdf_setting=None)
modify_run_card(run_card_input=None, run_card_backup=None, process_dir=MADGRAPH_GRIDPACK_LOCATION, runArgs=None, flags=None, settings={}, skipBaseFragment=False, pdf_setting=None)
print_cards_from_dir(process_dir=MADGRAPH_GRIDPACK_LOCATION)
get_output_txt_file(runArgs=None, flags=None)
generate(process_dir='PROC_mssm_0', grid_pack=False, gridpack_compile=False, extlhapath=None, required_accuracy=0.01, runArgs=None, flags=None, bias_module=None, requirePMGSettings=False, pdf_setting=None)
check_reweight_card(process_dir=MADGRAPH_GRIDPACK_LOCATION)
modify_config_card(config_card_backup=None, process_dir=MADGRAPH_GRIDPACK_LOCATION, settings={}, set_commented=True)
setupFastjet(process_dir=None)
add_reweighting(run_name, reweight_card=None, process_dir=MADGRAPH_GRIDPACK_LOCATION)
_write_run_card(runArgs=None, flags=None)
get_default_runcard(process_dir=MADGRAPH_GRIDPACK_LOCATION)
add_lifetimes(process_dir, threshold=None)
stack_subprocess(command, **kwargs)
setup_bias_module(bias_module, process_dir)
madspin_on_lhe(input_LHE, madspin_card, runArgs=None, keep_original=False)
generate_from_gridpack(runArgs=None, flags=None, extlhapath=None, gridpack_compile=None, requirePMGSettings=False, pdf_setting=None)
fix_fks_makefile(process_dir)
generate_prep(process_dir)
setNCores(process_dir, Ncores=None)
get_reweight_card(process_dir=MADGRAPH_GRIDPACK_LOCATION)
setup_pdf_and_systematic_weights(the_base_fragment, extras, isNLO)
get_expected_reweight_names(reweight_card_loc)