ATLAS Offline Software
Job.h
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
3 */
4 
5 #ifndef EVENT_LOOP_JOB_HH
6 #define EVENT_LOOP_JOB_HH
7 
8 //
9 // Distributed under the Boost Software License, Version 1.0.
10 // (See accompanying file LICENSE_1_0.txt or copy at
11 // http://www.boost.org/LICENSE_1_0.txt)
12 
13 // Please feel free to contact me (krumnack@iastate.edu) for bug
14 // reports, feature suggestions, praise and complaints.
15 
16 
21 
22 
23 
24 #include <EventLoop/Global.h>
25 
26 
27 #include <AnaAlgorithm/Global.h>
28 #include <EventLoop/JobConfig.h>
31 #include <vector>
32 #include <memory>
33 class AnaReentrantAlgorithmConfig;
34 
35 namespace asg
36 {
37  class AsgServiceConfig;
38 }
39 
40 namespace EL
41 {
42  class PythonConfigBase;
43 
44 
47  void swap (Job& a, Job& b);
48 
49 
50  class Job
51  {
52  //
53  // public interface
54  //
55 
58  public:
59  void testInvariant () const;
60 
61 
65  public:
66  Job ();
67 
68 
72  public:
73  Job (const Job& that);
74 
75 
78  public:
79  ~Job ();
80 
81 
86  public:
87  Job& operator = (const Job& that);
88 
89 
93  public:
94  const SH::SampleHandler& sampleHandler () const;
95  void sampleHandler (const SH::SampleHandler& val_sampleHandler);
96 
97 
102  public:
103  void algsAdd (std::unique_ptr<IAlgorithmWrapper> val_algorithm);
104  void algsAdd (std::unique_ptr<Algorithm> val_algorithm);
105  void algsAdd (Algorithm *alg_swallow);
106  void algsAdd (const AnaAlgorithmConfig& config);
108  void algsAdd (const asg::AsgServiceConfig& config);
109  void algsAdd (const EL::PythonConfigBase& config);
110 
111 
122  public:
123  void algsAddClone (const Algorithm& alg);
124 
125 
128  public:
129  bool algsHas (const std::string& name) const;
130 
131 
141  public:
143  typedef const OutputStream* outputIter;
145  outputIter outputBegin () const;
147  outputIter outputEnd () const;
148  void outputAdd (const OutputStream& val_output);
149 
150 
153  public:
154  bool outputHas (const std::string& name) const;
155 
156 
161  public:
162  void useXAOD ();
163 
164 
168  public:
170  const SH::MetaObject *options () const;
171 
172 
176  public:
177  const JobConfig& jobConfig () const noexcept;
178 
179 
188  public:
189  static const std::string optRemoveSubmitDir;
190 
191 
194  public:
195  static const std::string optSubmitDirMode;
196 
199  public:
200  static const std::string optUniqueDateFormat;
201 
202 
205  static const std::string optAlgorithmTimer;
206 
207 
210  static const std::string optAlgorithmMemoryMonitor;
211 
212 
214  static const std::string optFactoryPreload;
215 
217  static const std::string optStreamAliases;
218 
219 
224  public:
225  static const std::string optMaxEvents;
226 
227 
232  public:
233  static const std::string optSkipEvents;
234 
235 
245  public:
246  static const std::string optWorkerConfigFile;
247 
248 
252 
258  public:
259  static const std::string optFilesPerWorker;
260 
261 
273  public:
274  static const std::string optEventsPerWorker;
275 
276  public:
287 
288 
295  public:
296  static const std::string optSubmitFlags;
297 
298 
301  public:
302  static const std::string optCondorConf;
303 
304 
311  public:
312  static const std::string optCacheSize;
313 
314 
326  public:
327  static const std::string optCacheLearnEntries;
328 
329 
336  public:
337  static const std::string optD3PDPerfStats;
338 
339 
345  public:
346  static const std::string optD3PDReadStats;
347 
348 
355  public:
356  static const std::string optXAODPerfStats;
357 
358 
364  public:
365  static const std::string optXAODReadStats;
366 
367 
375  public:
376  static const std::string optD3PDCacheMinEvent;
378  static const std::string optD3PDCacheMinByte;
379  static const std::string optD3PDCacheMinByteFraction;
380 
381 
387  public:
388  static const std::string optPerfTree;
389 
390 
392  public:
393  static const std::string optXAODInput;
394 
399  public:
400  static const std::string optXaodAccessMode;
401  static const std::string optXaodAccessMode_branch;
402  static const std::string optXaodAccessMode_class;
403  static const std::string optXaodAccessMode_athena;
404 
405 
408  public:
409  static const std::string optXAODSummaryReport;
410 
411 
418  public:
419  static const std::string optPrintPerFileStats;
420 
421 
423  public:
424  static const std::string optDisableMetrics;
425 
426 
430  public:
431  static const std::string optResetShell;
432 
435  public:
436  static const std::string optLocalNoUnsetup;
437 
445  public:
446  static const std::string optNumParallelProcs;
447 
448 
450  public:
451  static const std::string optBackgroundProcess;
452 
453 
455  public:
456  static const std::string optOutputSampleName;
457 
458 
462  public:
463  static const std::string optGridDestSE;
464  static const std::string optGridSite;
465  static const std::string optGridExcludedSite;
466  static const std::string optGridNGBPerJob;
467  static const std::string optGridMemory;
468  static const std::string optGridMaxCpuCount;
469  static const std::string optGridNFiles;
470  static const std::string optGridNFilesPerJob;
471  static const std::string optGridNJobs;
472  static const std::string optGridMaxFileSize;
473  static const std::string optGridMaxNFilesPerJob;
474  static const std::string optGridNEventsPerJob;
475  static const std::string optGridExpress;
476  static const std::string optGridNoSubmit;
477  static const std::string optGridMergeOutput;
479  static const std::string optGridWorkingGroup;
480  static const std::string optGridShowCmd;
481  static const std::string optGridCpuTimePerEvent;
482  static const std::string optGridMaxWalltime;
483  static const std::string optGridAvoidVP;
489  static const std::string optTmpDir;
490  static const std::string optRootVer;
491  static const std::string optCmtConfig;
492  static const std::string optGridDisableAutoRetry;
493  static const std::string optOfficial;
494  static const std::string optVoms;
495 
497  static const std::string optGridReporting;
498 
501 
502  static const std::string optRetries;
503  static const std::string optRetriesWait;
504 
506 
507 
509  static const std::string optUserFiles;
510 
511 
512 
515  public:
516  static const std::string optBatchSharedFileSystem;
522  static const std::string optBatchSlurmWrapperExec;
524  static const std::string optBatchSetupCommand;
525 
528  static const std::string optDockerImage;
529 
531  static const std::string optDockerOptions;
532 
535  static const std::string optBatchConfigFile;
536 
540  static const std::string optBatchSetupFile;
541 
542 
543  public:
546 
555 
565 
572  static const std::string optMemResidentIncreaseLimit;
573 
580  static const std::string optMemVirtualIncreaseLimit;
581 
594  static const std::string optMemFailOnLeak;
595 
597 
598 
605  static const std::string histogramStreamName;
606 
607 
608 
609  //
610  // private interface
611  //
612 
613  friend void swap (Job& a, Job& b);
614 
616  private:
617  SH::SampleHandler m_sampleHandler;
618  private:
620  private:
621  SH::MetaObject m_options;
622 
624  private:
626  };
627 }
628 
629 #endif
EL::Job::optFilesPerWorker
static const std::string optFilesPerWorker
description: the name of the option for selecting the number of files per batch job.
Definition: Job.h:259
EL::Job::optCondorConf
static const std::string optCondorConf
description: the name of the option for supplying extra parameters for condor systems
Definition: Job.h:302
EL::Job::optMaxEvents
static const std::string optMaxEvents
description: the name of the option used for setting the maximum number of events to process per samp...
Definition: Job.h:225
EL::Job::optPerfTree
static const std::string optPerfTree
description: the option to turn on the performance tree in PROOF.
Definition: Job.h:388
EL::Job::optGridAddNthFieldOfInDSToLFN
static const std::string optGridAddNthFieldOfInDSToLFN
Definition: Job.h:478
EL::Job::jobConfig
const JobConfig & jobConfig() const noexcept
the JobConfig object we are wrapping
Definition: Job.cxx:429
EL::Job::options
SH::MetaObject * options()
description: the list of options to the job guarantee: no-fail postcondition: result !...
Definition: Job.cxx:411
EL::Job::optGridExcludedSite
static const std::string optGridExcludedSite
Definition: Job.h:465
EL::Job::optBatchSharedFileSystem
static const std::string optBatchSharedFileSystem
description: batch-specific options rationale: these options are for configuring batch drivers
Definition: Job.h:516
EL::Job::Job
Job()
effects: standard default constructor guarantee: strong failures: low level errors I
Definition: Job.cxx:151
EL::Job::optXaodAccessMode
static const std::string optXaodAccessMode
description: the option to select the access mode for xAODs.
Definition: Job.h:400
EL::Job::outputMIter
OutputStream * outputMIter
description: the list of output datasets used guarantee: no-fail / strong failures(3): out of memory ...
Definition: Job.h:142
EL::Job::swap
friend void swap(Job &a, Job &b)
effects: standard swap guarantee: no-fail
Definition: Job.cxx:133
EL::Job::optD3PDReadStats
static const std::string optD3PDReadStats
description: the name of the D3PDPerfStats object produced as I gather it, as well as the name of the...
Definition: Job.h:346
SH::MetaObject
A class that manages meta-data to be associated with an object.
Definition: MetaObject.h:56
EL::Job::optDockerImage
static const std::string optDockerImage
this is the name of the docker image, when using docker with a supported batch driver
Definition: Job.h:528
EL::Job::optPrintPerFileStats
static const std::string optPrintPerFileStats
description: the option to turn on printing of i/o statistics at the end of each file rationale: whil...
Definition: Job.h:419
EL::OutputStream
Definition: OutputStream.h:34
EL::Job::optGridNEventsPerJob
static const std::string optGridNEventsPerJob
Definition: Job.h:474
EL::Job::algsHas
bool algsHas(const std::string &name) const
returns: whether we have the algorithm with the given name guarantee: no-fail
Definition: Job.cxx:331
EL::Job::optRemoveSubmitDir
static const std::string optRemoveSubmitDir
description: the name of the option for overwriting the submission directory.
Definition: Job.h:189
EL::Job::m_jobConfig
EL::JobConfig m_jobConfig
the JobConfig object we use
Definition: Job.h:625
EL::Job::optD3PDCacheMinEventFraction
static const std::string optD3PDCacheMinEventFraction
Definition: Job.h:377
EL::Job::optGridMaxNFilesPerJob
static const std::string optGridMaxNFilesPerJob
Definition: Job.h:473
EL::Job::~Job
~Job()
effects: standard destructor guarantee: no-fail
Definition: Job.cxx:174
EL::Job::optMemResidentPerEventIncreaseLimit
static const std::string optMemResidentPerEventIncreaseLimit
The minimal per-event resident memory increase for triggering an error.
Definition: Job.h:554
asg
Definition: DataHandleTestTool.h:28
EL::swap
void swap(Job &a, Job &b)
effects: standard swap guarantee: no-fail
Definition: Job.cxx:133
EL::Job::optXaodAccessMode_class
static const std::string optXaodAccessMode_class
Definition: Job.h:402
SampleHandler.h
EL::Job::optUserFiles
static const std::string optUserFiles
a list of files that need to be available within the worker job
Definition: Job.h:509
EL::Job::optFactoryPreload
static const std::string optFactoryPreload
a boolean flag for whether to perform a component factory preload
Definition: Job.h:214
EL::AnaAlgorithmConfig
an object that can create a AnaAlgorithm
Definition: AnaAlgorithmConfig.h:29
EL::Job::m_output
std::vector< EL::OutputStream > m_output
Definition: Job.h:619
const
bool const RAWDATA *ch2 const
Definition: LArRodBlockPhysicsV0.cxx:560
EL::Job::optGridMemory
static const std::string optGridMemory
Definition: Job.h:467
EL::Job::optDisableMetrics
static const std::string optDisableMetrics
description: the option to turn off collection of performance data
Definition: Job.h:424
EL::Job::optGridNFiles
static const std::string optGridNFiles
Definition: Job.h:469
EL::Job::optD3PDCacheMinByteFraction
static const std::string optD3PDCacheMinByteFraction
Definition: Job.h:379
EL::Job::optXAODSummaryReport
static const std::string optXAODSummaryReport
the option to turn on/off the xAOD summary reporting at the end of the job
Definition: Job.h:409
EL::Job::optGridMaxFileSize
static const std::string optGridMaxFileSize
Definition: Job.h:472
EL::Job::optXAODInput
static const std::string optXAODInput
the option to select whether our input is xAODs
Definition: Job.h:393
config
Definition: PhysicsAnalysis/AnalysisCommon/AssociationUtils/python/config.py:1
EL::Job::m_options
SH::MetaObject m_options
Definition: Job.h:621
EL::Job::m_sampleHandler
SH::SampleHandler m_sampleHandler
description: members directly corresponding to accessors
Definition: Job.h:617
EL::Job::algsAddClone
void algsAddClone(const Algorithm &alg)
add a clone of the given algorithm
Definition: Job.cxx:322
EL::Job::optStreamAliases
static const std::string optStreamAliases
an option for stream aliases
Definition: Job.h:217
EL::Job::optBatchSetupCommand
static const std::string optBatchSetupCommand
This overrides the asetup command if you need to use a custom one.
Definition: Job.h:524
EL::Job::optRetriesWait
static const std::string optRetriesWait
Definition: Job.h:503
EL::Job::optGridShowCmd
static const std::string optGridShowCmd
Definition: Job.h:480
EL::Job::optGridDisableAutoRetry
static const std::string optGridDisableAutoRetry
Definition: Job.h:492
MetaObject.h
EL::Algorithm
Definition: Algorithm.h:22
EL::Job::optGridNoSubmit
static const std::string optGridNoSubmit
Definition: Job.h:476
EL::Job::optGridDestSE
static const std::string optGridDestSE
description: grid-specific options rationale: these are named so as to correspond to prun equivalents...
Definition: Job.h:463
EL::Job::optGridSite
static const std::string optGridSite
Definition: Job.h:464
EL::PythonConfigBase
the base class for the python configuration of any asg::AsgComponent
Definition: PythonConfigBase.h:36
EL::Job::optCacheSize
static const std::string optCacheSize
description: this option allows to configure the TTreeCache size for this job.
Definition: Job.h:312
vector
Definition: MultiHisto.h:13
EL::Job::optWorkerConfigFile
static const std::string optWorkerConfigFile
a python configuration file that will be executed on the worker
Definition: Job.h:246
EL::Job::optResetShell
static const std::string optResetShell
description: the option to reset the shell on the worker nodes rationale: this is currently only used...
Definition: Job.h:431
EL::Job::optD3PDCacheMinByte
static const std::string optD3PDCacheMinByte
Definition: Job.h:378
EL::Job::optGridNFilesPerJob
static const std::string optGridNFilesPerJob
Definition: Job.h:470
EL::Job::optBatchSlurmWrapperExec
static const std::string optBatchSlurmWrapperExec
Append a command before the main executable is called This is useful is you want to execute the comma...
Definition: Job.h:522
EL::Job::optXAODPerfStats
static const std::string optXAODPerfStats
description: the name of the option for turning on XAODPerfStats.
Definition: Job.h:356
EL::Job::optGridPrunShipAdditionalFilesOrDirs
static const std::string optGridPrunShipAdditionalFilesOrDirs
Enables to ship additional files to the tarbal sent to the grid Should be a list of comma separated p...
Definition: Job.h:488
EL::Job::optWorkerPostClosedOutputsExecutable
static const std::string optWorkerPostClosedOutputsExecutable
Optionnal executable to be provided by the user that will be called, by the worker,...
Definition: Job.h:286
EL::Job::optGridNGBPerJob
static const std::string optGridNGBPerJob
Definition: Job.h:466
EL::Job::optRetries
static const std::string optRetries
these options are defined in SH::MetaNames
Definition: Job.h:502
EL::Job::outputAdd
void outputAdd(const OutputStream &val_output)
Definition: Job.cxx:376
EL::Job::optD3PDCacheMinEvent
static const std::string optD3PDCacheMinEvent
description: these options configure the D3PDReader TTreeCache settings.
Definition: Job.h:376
EL::Job::optGridWorkingGroup
static const std::string optGridWorkingGroup
Definition: Job.h:479
JobConfig.h
EL::Job::optGridExpress
static const std::string optGridExpress
Definition: Job.h:475
EL
This module defines the arguments passed from the BATCH driver to the BATCH worker.
Definition: AsgComponentFactories.h:16
EL::Job::optNumParallelProcs
static const std::string optNumParallelProcs
the option to specify the number of parallel jobs in LocalDriver (0 = number of hardware cores) (defa...
Definition: Job.h:446
EL::Job::operator=
Job & operator=(const Job &that)
effects: standard assignment operator returns: *this guarantee: strong failures: out of memory II
Definition: Job.cxx:182
EL::Job::optVoms
static const std::string optVoms
Definition: Job.h:494
EL::Job::optCacheLearnEntries
static const std::string optCacheLearnEntries
description: this option allows to configure the number of tree entries used for learning cache behav...
Definition: Job.h:327
EL::Job::optMemFailOnLeak
static const std::string optMemFailOnLeak
Failure behaviour of the code when a "significant memory leak" is found.
Definition: Job.h:594
EL::Job::optSubmitFlags
static const std::string optSubmitFlags
description: the name of the option for supplying extra submit parameters to batch systems rationale:...
Definition: Job.h:296
EL::Job::outputHas
bool outputHas(const std::string &name) const
returns: whether we have an output with the given name guarantee: no-fail
Definition: Job.cxx:386
EL::Job::optD3PDPerfStats
static const std::string optD3PDPerfStats
description: the name of the option for turning on D3PDPerfStats.
Definition: Job.h:337
EL::Job::optDockerOptions
static const std::string optDockerOptions
any extra options we may want to pass to docker
Definition: Job.h:531
EL::Job::optAlgorithmTimer
static const std::string optAlgorithmTimer
a boolean flag for whether to add a timer for the algorithms
Definition: Job.h:205
EL::Job::optBatchSlurmExtraConfigLines
static const std::string optBatchSlurmExtraConfigLines
The content of this string will be executed in the job script on the worker node before the main exec...
Definition: Job.h:519
EL::Job::optOfficial
static const std::string optOfficial
Definition: Job.h:493
Global.h
EL::Job::optMemResidentIncreaseLimit
static const std::string optMemResidentIncreaseLimit
The minimal resident memory increase necessary to trigger an error.
Definition: Job.h:572
EL::Job::optGridMergeOutput
static const std::string optGridMergeOutput
Definition: Job.h:477
EL::Job::optBackgroundProcess
static const std::string optBackgroundProcess
the option to do processing in a background process in PROOF
Definition: Job.h:451
EL::Job::histogramStreamName
static const std::string histogramStreamName
the name of the histogram output stream
Definition: Job.h:605
EL::Job::optAlgorithmMemoryMonitor
static const std::string optAlgorithmMemoryMonitor
a boolean flag for whether to add a memory monitor for the algorithms
Definition: Job.h:210
EL::Job::optBatchConfigFile
static const std::string optBatchConfigFile
the job submission configuration file (used by some drivers that need more complex configuration)
Definition: Job.h:535
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:240
plotBeamSpotMon.b
b
Definition: plotBeamSpotMon.py:76
EL::JobConfig
the job configuration that is independent of driver and dataset
Definition: JobConfig.h:39
EL::Job::optXaodAccessMode_branch
static const std::string optXaodAccessMode_branch
Definition: Job.h:401
RegSelToolConfig.alg
alg
Definition: RegSelToolConfig.py:332
asg::AsgServiceConfig
an object that can create a AsgService
Definition: AsgServiceConfig.h:25
EL::Job::optCmtConfig
static const std::string optCmtConfig
Definition: Job.h:491
EL::Job::algsAdd
void algsAdd(std::unique_ptr< IAlgorithmWrapper > val_algorithm)
description: the list of algorithms used guarantee: no-fail / strong failures: out of memory II invar...
Definition: Job.cxx:211
EL::Job::optTmpDir
static const std::string optTmpDir
Definition: Job.h:489
EL::Job::optGridMaxWalltime
static const std::string optGridMaxWalltime
Definition: Job.h:482
EL::Job::outputBegin
outputMIter outputBegin()
Definition: Job.cxx:340
a
TList * a
Definition: liststreamerinfos.cxx:10
EL::Job::outputIter
const OutputStream * outputIter
Definition: Job.h:143
EL::Job::optXaodAccessMode_athena
static const std::string optXaodAccessMode_athena
Definition: Job.h:403
EL::Job::outputEnd
outputMIter outputEnd()
Definition: Job.cxx:358
EL::Job::testInvariant
void testInvariant() const
effects: test the invariant of this object guarantee: no-fail
Definition: Job.cxx:143
EL::Job::useXAOD
void useXAOD()
effects: register this job to use XAODs guarantee: strong failures: out of memory II failures: TEvent...
Definition: Job.cxx:401
EL::Job::sampleHandler
const SH::SampleHandler & sampleHandler() const
description: the sample handler used guarantee: no-fail / strong failures: out of memory II
Definition: Job.cxx:193
EL::Job::optGridCpuTimePerEvent
static const std::string optGridCpuTimePerEvent
Definition: Job.h:481
EL::Job::optGridAvoidVP
static const std::string optGridAvoidVP
Definition: Job.h:483
private
#define private
Definition: xAODTruthCnvAlg.h:20
EL::Job::optRootVer
static const std::string optRootVer
Definition: Job.h:490
EL::Job::optOutputSampleName
static const std::string optOutputSampleName
the output sample name
Definition: Job.h:456
EL::Job::optGridNJobs
static const std::string optGridNJobs
Definition: Job.h:471
SH::SampleHandler
A class that manages a list of Sample objects.
Definition: SampleHandler.h:60
EL::Job::optSubmitDirMode
static const std::string optSubmitDirMode
the submit-dir mode (allowed values: "no-clobber", "overwrite", "unique", "unique-link")
Definition: Job.h:195
SH
This module provides a lot of global definitions, forward declarations and includes that are used by ...
Definition: PrunDriver.h:15
EL::Job::optGridMaxCpuCount
static const std::string optGridMaxCpuCount
Definition: Job.h:468
EL::Job::optSkipEvents
static const std::string optSkipEvents
description: the name of the option used for skipping a certain number of events in the beginning rat...
Definition: Job.h:233
EL::Job
Definition: Job.h:51
EL::Job::optMemVirtualPerEventIncreaseLimit
static const std::string optMemVirtualPerEventIncreaseLimit
The minimal per-event virtual memory increase for triggering an error.
Definition: Job.h:564
EL::AnaReentrantAlgorithmConfig
an object that can create a AnaReentrantAlgorithm
Definition: AnaReentrantAlgorithmConfig.h:29
EL::Job::optUniqueDateFormat
static const std::string optUniqueDateFormat
the date-format to use when generating unique submission directory names
Definition: Job.h:200
EL::Job::optBatchSetupFile
static const std::string optBatchSetupFile
the job submission setup file.
Definition: Job.h:540
Global.h
EL::Job::optMemVirtualIncreaseLimit
static const std::string optMemVirtualIncreaseLimit
The minimal virtual memory increase necessary to trigger an error.
Definition: Job.h:580
EL::Job::optXAODReadStats
static const std::string optXAODReadStats
description: the name of the XAODPerfStats object produced as I gather it, as well as the name of the...
Definition: Job.h:365
EL::Job::optEventsPerWorker
static const std::string optEventsPerWorker
description: the name of the option for selecting the number of events per batch job.
Definition: Job.h:274
EL::Job::optGridReporting
static const std::string optGridReporting
whether to use grid reporting even when not running on the grid
Definition: Job.h:497
EL::Job::optLocalNoUnsetup
static const std::string optLocalNoUnsetup
the option not to unsetup the environment in LocalDriver
Definition: Job.h:436