ATLAS Offline Software
Herwig7Control.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
2 # \file Herwig7Control.py
3 # \brief Main python interface for %Herwig7 for preparing the event generation
4 # \author Daniel Rauch (daniel.rauch@desy.de)
5 # \author Lukas Kretschmann (lukas.kretschmann@cern.ch)
6 #
7 # This part of the interface provides functionality for running all the tasks
8 # necessary to initialize and prepare the event generation.
9 # More concretely, it handles the read or alternatively the build/integrate/
10 # mergegrids steps in order to produce the Herwig runfile and all other
11 # ingredients for a run, possibly also creating a gridpack.
12 # The event generation itself starting from reading the runfile is handled
13 # in Herwig7_i/Herwig7.h and src/Herwig7.cxx.
14 
15 import os, shutil, subprocess, sys
16 import six
17 
18 from . import Herwig7Utils as hw7Utils
19 from . import Herwig7JOChecker as JOChecker
20 from . import Herwig7ConfigDecoder as ConfigDecoder
21 
22 from AthenaCommon import Logging
23 athMsgLog = Logging.logging.getLogger('Herwig7Control')
24 
25 
26 # \brief Get path to the `share/Herwig` folder
27 #
28 # Try to get it from the `InstallArea` first.
29 # If this fails fall back to `$HERWIG7_PATH/share/Herwig`
30 #
32 
33  cmt_paths = os.environ.get("CMAKE_PREFIX_PATH")
34  cmt_config = os.environ.get("BINARY_TAG")
35 
36  # trying to get it from the `InstallArea`
37  for path in cmt_paths.split(':'):
38  path = os.path.join(path, "InstallArea", cmt_config, "share")
39  try:
40  filelist = os.listdir(path)
41  except Exception:
42  filelist = []
43  if "HerwigDefaults.rpo" in filelist: return(path)
44 
45  # falling back to `$HERWIG7_PATH`
46  path = os.path.join(os.environ['HERWIG7_PATH'], 'share/Herwig')
47  if os.path.isfile(os.path.join(path, 'HerwigDefaults.rpo')):
48  return(path)
49 
50  # raise exception if none of the two methods work out
51  raise RuntimeError(hw7Utils.ansi_format_error('Could not find a valid share/Herwig folder'))
52 
53 
54 # proper handling with path set in External/Herwig7/cmt/requirements
55 herwig7_path = os.environ['HERWIG7_PATH']
56 herwig7_bin_path = os.path.join(herwig7_path, 'bin')
57 herwig7_share_path = get_share_path()
58 
59 herwig7_binary = os.path.join(herwig7_bin_path, 'Herwig')
60 
61 
62 # Do the read/run sequence.
63 #
64 # This function should provide the read and run step in one go
65 def run(gen_config):
66 
67  # perform the read step
68  do_read(gen_config)
69 
70  # start the event generation
71  do_run(gen_config, cleanup_herwig_scratch=False)
72 
73 
74 # Do the build, integrate, mergegrids and run step in one go
75 # without creating a gridpack
76 #
77 # \param[in] cleanup_herwig_scratch Remove `Herwig-scratch` or 'Herwig-cache' folder after event generation to save disk space
78 def matchbox_run(gen_config, integration_jobs, cleanup_herwig_scratch):
79 
80  # perform build/integrate/mergegrids sequence
81  do_build_integrate_mergegrids(gen_config, integration_jobs)
82 
83  # start the event generation
84  do_run(gen_config, cleanup_herwig_scratch)
85 
86 
87 # Either do the build, integrate and mergegrids steps and create a gridpack
88 # or extract it and generate events from it
89 #
90 # \param[in] cleanup_herwig_scratch Remove `Herwig-scratch` or 'Herwig-cache' folder after event generation to save disk space
91 def matchbox_run_gridpack(gen_config, integration_jobs, gridpack_name, cleanup_herwig_scratch, integrate):
92 
93  # print start banner including version numbers
94  log(message=start_banner())
95 
96  if not gridpack_name or integrate:
97 
98  # create infile from jobOption commands
99  write_infile(gen_config)
100 
101  # do build/integrate/mergegrids sequence
102  xsec, err = do_build_integrate_mergegrids(gen_config, integration_jobs)
103 
104  # compress infile, runfile and process folder to gridpack tarball
105  do_compress_gridpack(gen_config.run_name, gridpack_name)
106 
107  # display banner and exit
108  log(message=exit_banner(gridpack_name, xsec, err))
109  sys.exit(0)
110 
111  else:
112 
113  # unpack the gridpack
114  DSIS_dir = gen_config.runArgs.jobConfig[0]+"/"
115  do_uncompress_gridpack(DSIS_dir+gridpack_name)
116  athMsgLog.info("Finished unpacking the gridpack")
117 
118  # start the event generation
119  do_run(gen_config, cleanup_herwig_scratch)
120 
121 
122 
123 def do_step(step, command, logfile_name=None):
124 
125  athMsgLog.info(hw7Utils.ansi_format_info("Starting Herwig7 '{}' step with command '{}'".format(step, ' '.join(command))))
126 
127  logfile = open(logfile_name, 'w') if logfile_name else None
128  do = subprocess.Popen(command, stdout=logfile, stderr=logfile)
129  do.wait()
130  if not do.returncode == 0:
131  raise RuntimeError(hw7Utils.ansi_format_error("Some error occured during the '{}' step.".format(step)))
132 
133  if logfile:
134  athMsgLog.info("Content of %s log file '%s':", step, logfile_name)
135  athMsgLog.info("")
136  with open(logfile_name, 'r') as logfile:
137  for line in logfile:
138  athMsgLog.info(' %s', line.rstrip('\n'))
139  athMsgLog.info("")
140 
141 
142 def do_abort():
143  athMsgLog.info(hw7Utils.ansi_format_info("Doing abort"))
144  sys.exit(0)
145 
146 
147 # Do the read step
148 def do_read(gen_config):
149 
150  # print start banner including version numbers
151  log(message=start_banner())
152 
153  # create infile from JobOption object
154  write_infile(gen_config)
155 
156  # copy HerwigDefaults.rpo to the current working directory
158 
159  # call Herwig7 binary to do the read step
160  share_path = get_share_path()
161  do_step('read', [herwig7_binary, 'read', get_infile_name(gen_config.run_name), '-I', share_path])
162 
163 # Do the read step and re-use an already existing infile
164 def do_read_existing_infile(gen_config):
165 
166  # print start banner including version numbers
167  log(message=start_banner())
168 
169  # copy HerwigDefaults.rpo to the current working directory
171 
172  # call Herwig7 binary to do the read step
173  share_path = get_share_path()
174  do_step('read', [herwig7_binary, 'read', gen_config.infile_name, '-I', share_path])
175 
176 
177 # Do the build step
178 def do_build(gen_config, integration_jobs):
179 
180  # print start banner including version numbers
181  log(message=start_banner())
182 
183  # create infile from JobOption object
184  write_infile(gen_config)
185 
186  # copy HerwigDefaults.rpo to the current working directory
188 
189  # call the Herwig7 binary to do the build step
190  share_path = get_share_path()
191  do_step('build', [herwig7_binary, 'build', get_infile_name(gen_config.run_name), '-I', share_path, '-y '+str(integration_jobs)])
192 
193 
194 # Do the integrate step for one specific integration job
195 # \todo provide info about the range
196 def do_integrate(run_name, integration_job):
197 
198  runfile_name = get_runfile_name(run_name)
199 
200  integrate_log = run_name+'.integrate'+str(integration_job)+'.log'
201  integrate_command = [herwig7_binary,'integrate',runfile_name,'--jobid='+str(integration_job)]
202 
203  do_step('integrate', integrate_command, integrate_log)
204 
205 
206 # This function provides the mergegrids step
207 def do_mergegrids(run_name, integration_jobs):
208 
209  runfile_name = get_runfile_name(run_name)
210  mergegrids_command = [herwig7_binary, 'mergegrids', runfile_name]
211 
212  do_step('mergegrids', mergegrids_command)
213 
214  # calculate the cross section from the integration logfiles and possibly warn about low accuracy
215  xsec, err = hw7Utils.get_cross_section(run_name, integration_jobs)
216 
217  return(xsec, err)
218 
219 
220 # Subsequent build, integrate and mergegrid steps
221 def do_build_integrate_mergegrids(gen_config, integration_jobs):
222 
223  # run build step
224  do_build(gen_config, integration_jobs)
225 
226  # run integration jobs in parallel subprocesses
227  runfile_name = get_runfile_name(gen_config.run_name)
228  athMsgLog.info(hw7Utils.ansi_format_info('Starting integration with {} jobs'.format(integration_jobs)))
229 
230  integration_procs = []
231  for integration_job in range(integration_jobs):
232  integrate_log = gen_config.run_name+'.integrate'+str(integration_job)+'.log'
233  integrate_command = [herwig7_binary,'integrate',runfile_name,'--jobid='+str(integration_job)]
234  integration_procs.append(hw7Utils.Process(integration_job, integrate_command, integrate_log))
235 
236  integration_handler = hw7Utils.ProcessHandler(integration_procs, athMsgLog)
237  if not integration_handler.success():
238  raise RuntimeError(hw7Utils.ansi_format_error('Not all of the integration jobs finished successfully'))
239 
240  athMsgLog.info(hw7Utils.ansi_format_ok('All integration jobs finished successfully'))
241 
242  # combine the different integration grids
243  xsec, err = do_mergegrids(gen_config.run_name, integration_jobs)
244 
245  return(xsec, err)
246 
247 
248 def do_compress_gridpack(run_name, gridpack_name):
249 
250  if not (gridpack_name.endswith('.tar.gz') or gridpack_name.endswith('.tgz')): gridpack_name += '.tar.gz'
251  infile_name = get_infile_name(run_name)
252  runfile_name = get_runfile_name(run_name)
253  version = herwig_version()
254  athMsgLog.debug("Scratch area, this is Herwig version '{}'".format(version))
255  if "7.1" in version or "7.0" in version:
256  do_step('compress', ['tar', 'czf', gridpack_name, infile_name, runfile_name, 'Herwig-scratch'])
257  else:
258  do_step('compress', ['tar', 'czf', gridpack_name, infile_name, runfile_name, 'Herwig-cache'])
259 
260 
261 def do_uncompress_gridpack(gridpack_name):
262 
263  athMsgLog.info("unpacking gridpack '%s'", gridpack_name)
264  do_step('uncompress', ['tar', 'xzf', gridpack_name])
265 
266 
267 # \param[in] cleanup_herwig_scratch Remove `Herwig-scratch` folder after event generation to save disk space
268 def do_run(gen_config, cleanup_herwig_scratch=True):
269 
270  # this is necessary to make Herwig aware of the name of the run file
271  gen_config.genSeq.Herwig7.RunFile = get_runfile_name(gen_config.run_name)
272 
273  # check the options in the .in file
274  JOChecker.check_file()
275 
276  # decode the run file to get list of all parameters
277  ConfigDecoder.DecodeRunCard(input_file = gen_config.genSeq.Herwig7.RunFile)
278 
279  # overwrite athena's seed for the random number generator
280  if gen_config.runArgs.randomSeed is None:
281  gen_config.genSeq.Herwig7.UseRandomSeedFromGeneratetf = False
282  else:
283  gen_config.genSeq.Herwig7.UseRandomSeedFromGeneratetf = True
284  gen_config.genSeq.Herwig7.RandomSeedFromGeneratetf = gen_config.runArgs.randomSeed
285 
286  # set matrix element PDF name in the Herwig7 C++ class
287  gen_config.genSeq.Herwig7.PDFNameME = gen_config.me_pdf_name
288 
289  # set underlying event PDF name in the Herwig7 C++ class
290  gen_config.genSeq.Herwig7.PDFNameMPI = gen_config.mpi_pdf_name
291 
292  # possibly delete Herwig-scratch folder after finishing the event generation
293  gen_config.genSeq.Herwig7.CleanupHerwigScratch = cleanup_herwig_scratch
294 
295  # don't break out here so that the job options can be finished and the C++
296  # part of the interface can take over and generate the events
297  athMsgLog.info(hw7Utils.ansi_format_info("Returning to the job options and starting the event generation afterwards"))
298 
299 
300 # Do the run step and re-use an already existing runfile
301 def do_run_existing_runfile(gen_config):
302 
303  # this is necessary to make Herwig aware of the name of the run file
304  gen_config.genSeq.Herwig7.RunFile = gen_config.runfile_name
305 
306  # overwrite athena's seed for the random number generator
307  if gen_config.runArgs.randomSeed is None:
308  gen_config.genSeq.Herwig7.UseRandomSeedFromGeneratetf = False
309  else:
310  gen_config.genSeq.Herwig7.UseRandomSeedFromGeneratetf = True
311  gen_config.genSeq.Herwig7.RandomSeedFromGeneratetf = gen_config.runArgs.randomSeed
312 
313  # don't break out here so that the job options can be finished and the C++
314  # part of the interface can take over and generate the events
315  athMsgLog.info(hw7Utils.ansi_format_info("Returning to the job options and starting the event generation afterwards"))
316 
317 
318 # utility functions -----------------------------------------------------------
319 
320 
322 
323  versions = get_software_versions()
324  return(' '.join(versions[0].split()[1:]))
325 
327 
328  versions = get_software_versions()
329  return(' '.join(versions[1].split()[1:]))
330 
332 
333  herwig_version_number = herwig_version()
334  thepeg_version_number = thepeg_version()
335  herwig_version_space = ' '.join(['' for i in range(14-len(herwig_version_number))])
336  thepeg_version_space = ' '.join(['' for i in range(14-len(thepeg_version_number))])
337 
338  banner = ''
339  banner += "#####################################\n"
340  banner += "## {} ##\n".format(hw7Utils.ansi_format_ok("---------------------------"))
341  banner += "## {} ##\n".format(hw7Utils.ansi_format_ok("Starting HERWIG 7 in ATHENA"))
342  banner += "## {} ##\n".format(hw7Utils.ansi_format_ok("---------------------------"))
343  banner += "## ##\n"
344  banner += "## with software versions: ##\n"
345  banner += "## - Herwig7: {}{} ##\n".format(herwig_version_number, herwig_version_space)
346  banner += "## - ThePEG: {}{} ##\n".format(thepeg_version_number, thepeg_version_space)
347  banner += "## ##\n"
348  banner += "#####################################\n"
349  return(banner)
350 
351 
353 
354  return(six.ensure_str(subprocess.check_output([herwig7_binary,'--version'])).splitlines())
355 
356 
357 def get_infile_name(run_name="Herwig-Matchbox"):
358 
359  return('{}.in'.format(run_name))
360 
361 
362 def get_setupfile_name(run_name="Herwig-Matchbox"):
363 
364  return('{}.setupfile.in'.format(run_name))
365 
366 
367 def get_runfile_name(run_name="Herwig-Matchbox"):
368 
369  return('{}.run'.format(run_name) if not run_name.endswith('.run') else run_name)
370 
371 
372 def write_infile(gen_config, print_infile=True):
373 
374  # lock settings to prevent modification from within the job options after infile was written to disk
375  gen_config.default_commands.lock()
376  gen_config.commands.lock()
377 
378  infile_name = get_infile_name(gen_config.run_name)
379  if print_infile: athMsgLog.info("")
380  athMsgLog.info(hw7Utils.ansi_format_info("Writing infile '{}'".format(infile_name)))
381  commands = \
382  gen_config.global_pre_commands().splitlines() \
383  + gen_config.local_pre_commands().splitlines() \
384  + ["",
385  "## ================",
386  "## Default Commands",
387  "## ================"] \
388  + str(gen_config.default_commands.commands).splitlines() \
389  + ["",
390  "## ========================",
391  "## Commands from jobOptions",
392  "## ========================"] \
393  + str(gen_config.commands.commands).splitlines() \
394  + gen_config.local_post_commands().splitlines()
395  try:
396  with open(infile_name, 'w') as infile:
397  for command in commands:
398  infile.write(command+'\n')
399  except Exception:
400  raise RuntimeError('Could not write Herwig/Matchbox infile')
401 
402  if print_infile:
403  athMsgLog.info("")
404  for command in commands:
405  athMsgLog.info(' %s', command)
406  athMsgLog.info("")
407 
408 
409 def write_setupfile(run_name, commands, print_setupfile=True):
410 
411  setupfile_name = get_setupfile_name(run_name)
412 
413  if len(commands) > 0:
414  if print_setupfile: athMsgLog.info("")
415  athMsgLog.info("Writing setupfile '%s'", setupfile_name)
416  try:
417  with open(setupfile_name, 'w') as setupfile:
418  for command in commands: setupfile.write(command+'\n')
419  except Exception:
420  raise RuntimeError('Could not write Herwig/Matchbox setupfile')
421 
422  if print_setupfile:
423  athMsgLog.info("")
424  for command in commands: athMsgLog.info(' %s', command)
425  athMsgLog.info("")
426 
427  else:
428  athMsgLog.info("No setupfile commands given.")
429 
430 
431 # \brief Copy default repository `HerwigDefaults.rpo` to current working directory
432 #
434 
435  shutil.copy(os.path.join(get_share_path(), 'HerwigDefaults.rpo'), 'HerwigDefaults.rpo')
436 
437 
438 def log(level='info', message=''):
439 
440  if level in ['info', 'warn', 'error']:
441  logger = getattr(athMsgLog, level)
442  for line in message.splitlines(): logger(line)
443  else:
444  raise ValueError("Unknown logging level'{}' specified. Possible values are 'info', 'warn' or 'error'".format(level))
445 
446 
447 def exit_banner(gridpack, cross_section, cross_section_error):
448 
449  size = hw7Utils.humanize_bytes(hw7Utils.get_size(gridpack))
450  space_size = hw7Utils.get_repeated_pattern(' ', 31-len(size))
451 
452  xsec = '{:f}'.format(cross_section)
453  err = '{:f}'.format(cross_section_error)
454  rel_err = '{:.2f}'.format(cross_section_error / cross_section * 100.0)
455  space_xsec = hw7Utils.get_repeated_pattern(' ', 37-len(xsec)-len(err)-len(rel_err))
456 
457  banner = ''
458  space = ' '.join(['' for i in range(70+4+1-len(gridpack))])
459  banner += "##########################################################################################\n"
460  banner += "## ------------------------------------------------------------------------------- ##\n"
461  banner += "## {} (size: {}){} ##\n".format(hw7Utils.ansi_format_ok("HERWIG 7 successfully created the gridpack"), size, space_size)
462  banner += "## ##\n"
463  banner += "## {}{} ##\n".format(hw7Utils.ansi_format_info(gridpack), space)
464  banner += "## ##\n"
465  banner += "## cross section from integration: {} +/- {} ({}%) nb {} ##\n".format(xsec, err, rel_err, space_xsec)
466 
467  if cross_section_error / cross_section > hw7Utils.integration_grids_precision_threshold:
468  threshold = '{}%'.format(hw7Utils.integration_grids_precision_threshold*100.0)
469  space_threshold = hw7Utils.get_repeated_pattern(' ', 6-len(threshold))
470  banner += "## ##\n"
471  banner += "## {} ##\n".format(hw7Utils.ansi_format_warning("! WARNING: The integration grids only have a low precision (worse than {}){}!".format(threshold, space_threshold)))
472 
473  banner += "## ##\n"
474  banner += "## ------------------------------------------------------------------------------- ##\n"
475  banner += "## ##\n"
476  banner += "## Please ignore the error ##\n"
477  banner += "## ##\n"
478  banner += "## No such file or directory: 'evgen.root' raised while stating file evgen.root ##\n"
479  banner += "## ##\n"
480  banner += "##########################################################################################\n"
481  return(banner)
482 
Herwig7Control.do_integrate
def do_integrate(run_name, integration_job)
Definition: Herwig7Control.py:196
Herwig7Control.exit_banner
def exit_banner(gridpack, cross_section, cross_section_error)
Definition: Herwig7Control.py:447
Herwig7Control.do_build
def do_build(gen_config, integration_jobs)
Definition: Herwig7Control.py:178
vtune_athena.format
format
Definition: vtune_athena.py:14
Herwig7Control.do_build_integrate_mergegrids
def do_build_integrate_mergegrids(gen_config, integration_jobs)
Definition: Herwig7Control.py:221
Herwig7Control.do_mergegrids
def do_mergegrids(run_name, integration_jobs)
Definition: Herwig7Control.py:207
Herwig7Control.matchbox_run
def matchbox_run(gen_config, integration_jobs, cleanup_herwig_scratch)
Definition: Herwig7Control.py:78
Herwig7Control.do_run_existing_runfile
def do_run_existing_runfile(gen_config)
Definition: Herwig7Control.py:301
Herwig7Control.do_abort
def do_abort()
Definition: Herwig7Control.py:142
Herwig7Control.get_default_repository
def get_default_repository()
Definition: Herwig7Control.py:433
Herwig7Control.write_setupfile
def write_setupfile(run_name, commands, print_setupfile=True)
Definition: Herwig7Control.py:409
Herwig7Control.do_step
def do_step(step, command, logfile_name=None)
Definition: Herwig7Control.py:123
Herwig7Control.matchbox_run_gridpack
def matchbox_run_gridpack(gen_config, integration_jobs, gridpack_name, cleanup_herwig_scratch, integrate)
Definition: Herwig7Control.py:91
Herwig7Control.run
def run(gen_config)
Definition: Herwig7Control.py:65
Herwig7Control.start_banner
def start_banner()
Definition: Herwig7Control.py:331
Herwig7Control.do_uncompress_gridpack
def do_uncompress_gridpack(gridpack_name)
Definition: Herwig7Control.py:261
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:194
Herwig7Control.get_infile_name
def get_infile_name(run_name="Herwig-Matchbox")
Definition: Herwig7Control.py:357
Herwig7Control.get_share_path
def get_share_path()
Definition: Herwig7Control.py:31
Herwig7Control.do_read_existing_infile
def do_read_existing_infile(gen_config)
Definition: Herwig7Control.py:164
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
Herwig7Control.do_read
def do_read(gen_config)
Definition: Herwig7Control.py:148
Herwig7Control.do_compress_gridpack
def do_compress_gridpack(run_name, gridpack_name)
Definition: Herwig7Control.py:248
Herwig7Control.log
def log(level='info', message='')
Definition: Herwig7Control.py:438
Trk::open
@ open
Definition: BinningType.h:40
Herwig7Control.get_software_versions
def get_software_versions()
Definition: Herwig7Control.py:352
Herwig7Control.get_runfile_name
def get_runfile_name(run_name="Herwig-Matchbox")
Definition: Herwig7Control.py:367
Herwig7Control.herwig_version
def herwig_version()
Definition: Herwig7Control.py:321
str
Definition: BTagTrackIpAccessor.cxx:11
Herwig7Control.do_run
def do_run(gen_config, cleanup_herwig_scratch=True)
Definition: Herwig7Control.py:268
Trk::split
@ split
Definition: LayerMaterialProperties.h:38
python.iconfTool.gui.pad.logger
logger
Definition: pad.py:14
Herwig7Control.get_setupfile_name
def get_setupfile_name(run_name="Herwig-Matchbox")
Definition: Herwig7Control.py:362
Herwig7Control.write_infile
def write_infile(gen_config, print_infile=True)
Definition: Herwig7Control.py:372
Herwig7Control.thepeg_version
def thepeg_version()
Definition: Herwig7Control.py:326