ATLAS Offline Software
Functions
ELG_prun Namespace Reference

Functions

def ELG_prun (sample)
 

Function Documentation

◆ ELG_prun()

def ELG_prun.ELG_prun (   sample)

Definition at line 9 of file ELG_prun.py.

9 def ELG_prun(sample) :
10 
11  try:
12  from pandatools import PandaToolsPkgInfo # noqa: F401
13  except ImportError:
14  print ("prun needs additional setup, try:")
15  print (" lsetup panda")
16  return 99
17 
18  cmd = ["prun"]
19 
20  #These are options that can be set by the user
21  opts = ['destSE',
22  'site',
23  'rootVer',
24  'cmtConfig',
25  'excludedSite',
26  'nGBPerJob',
27  'memory',
28  'maxCpuCount',
29  'nFiles',
30  'nFilesPerJob',
31  'nEventsPerJob',
32  'nJobs',
33  'maxFileSize',
34  'maxNFilesPerJob',
35  'addNthFieldOfInDSToLFN',
36  'cpuTimePerEvent',
37  'maxWalltime',
38  'voms',
39  'workingGroup',
40  'tmpDir']
41 
42  #These are options that can be set by the user
43  switches = ['express',
44  'noSubmit',
45  'skipScout',
46  'disableAutoRetry',
47  'useNewCode',
48  'official',
49  'mergeOutput',
50  'useRootCore',
51  'useAthenaPackages',
52  'avoidVP']
53 
54  using_nEventsPerJob = False
55  from ROOT import SH
56  for opt in opts :
57  arg = sample.meta().castDouble('nc_' + opt, -1, SH.MetaObject.CAST_NOCAST_DEFAULT)
58  if abs(arg + 1) > 1e-6 :
59  cmd += ["--" + opt + "=" + str(int(round(arg)))]
60  if opt=="nEventsPerJob":
61  using_nEventsPerJob=True
62  else :
63  arg = sample.meta().castString('nc_' + opt)
64  if len(arg) :
65  cmd += ["--" + opt + "=" + arg]
66 
67  # nGBPerJob and nEventsPerJob are incompatible to prun
68  if using_nEventsPerJob:
69  cmd = [ x for x in cmd if "nGBPerJob" not in x ]
70  print(cmd)
71 
72  for switch in switches :
73  arg = sample.meta().castDouble('nc_' + switch, 0, SH.MetaObject.CAST_NOCAST_DEFAULT)
74  if arg != 0 :
75  cmd += ["--" + switch]
76  else :
77  arg = sample.meta().castString('nc_' + switch)
78  if len(arg) :
79  if arg != "False" and arg != "false" and arg != "FALSE" :
80  cmd += ["--" + switch]
81 
82  #These options should normally not be touched by the user
83  internalOpts = ['exec',
84  'inDS',
85  'outDS',
86  'outputs',
87  'writeInputToTxt',
88  'match',
89  'framework']
90 
91  for opt in internalOpts :
92  value = sample.meta().castString('nc_' + opt)
93  if opt == "exec" and using_nEventsPerJob:
94  value += " %SKIPEVENTS %MAXEVENTS"
95  cmd += ["--" + opt + "=" + value]
96 
97  if sample.meta().castDouble('nc_mergeOutput', 1, SH.MetaObject.CAST_NOCAST_DEFAULT) == 0 or sample.meta().castString('nc_mergeOutput').upper() == 'FALSE' :
98  #don't set merge script
99  pass
100  else :
101  cmd += ["--mergeScript=" + sample.meta().castString('nc_mergeScript')]
102 
103  if len(sample.meta().castString('nc_EventLoop_SubmitFlags')) :
104  cmd += shlex.split (sample.meta().castString('nc_EventLoop_SubmitFlags'))
105 
106  if sample.meta().castDouble('nc_showCmd', 0, SH.MetaObject.CAST_NOCAST_DEFAULT) != 0 :
107  print (cmd)
108 
109  if not os.path.isfile('jobcontents.tgz') :
110  import copy
111  dummycmd = copy.deepcopy(cmd)
112  dummycmd += ["--outTarBall=jobcontents.tgz"]
113  if len(sample.meta().castString('nc_EventLoop_UserFiles')) :
114  dummycmd += ["--extFile=jobdef.root,runjob.sh," + sample.meta().castString('nc_EventLoop_UserFiles').replace(" ",",")]
115  pass
116  else :
117  dummycmd += ["--extFile=jobdef.root,runjob.sh"]
118  pass
119  dummycmd += ["--noSubmit"]
120 
121  try:
122  out = subprocess.check_output(dummycmd, stderr=subprocess.STDOUT)
123  except subprocess.CalledProcessError as e:
124  # Handle a case where we couldn't get the grid nickname in advance
125  if b'Need to generate a grid proxy' in e.output and any( ['%nickname%' in x for x in cmd ] ):
126  print('Detected nickname still undefined. Trying to replace it.')
127  try:
128  from pandatools import PsubUtils
129  nickname = PsubUtils.getNickname()
130  dummycmd = [ x.replace('%nickname%',nickname) for x in dummycmd ]
131  cmd = [ x.replace('%nickname%',nickname) for x in cmd ]
132  except Exception as e_rep:
133  print(f'Nickname replacement failed with error {e_rep.returncode}: {e_rep.output}')
134  # Now try the job again
135  try:
136  out = subprocess.check_output(dummycmd, stderr=subprocess.STDOUT)
137  except subprocess.CalledProcessError as e_take2:
138  print ("Command:")
139  print (e_take2.cmd)
140  print ("failed with return code " , e_take2.returncode)
141  print ("output was:")
142  print (e_take2.output)
143  return 1
144  else:
145  print ("Command:")
146  print (e.cmd)
147  print ("failed with return code " , e.returncode)
148  print ("output was:")
149  print (e.output)
150  return 1
151 
152  cmd += ["--inTarBall=jobcontents.tgz"]
153 
154  out = ""
155  try:
156  out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
157  except subprocess.CalledProcessError as e:
158  print ("Command:")
159  print (e.cmd)
160  print ("failed with return code ", e.returncode)
161  print ("output was:")
162  print (e.output)
163  return 2
164 
165  jediTaskID = 0
166  try:
167  line = re.findall(r'TaskID=\d+', str(out))[0]
168  jediTaskID = int(re.findall(r'\d+', line)[0])
169  except IndexError:
170  print (out)
171  return 3
172 
173  return jediTaskID
ELG_prun.ELG_prun
def ELG_prun(sample)
Definition: ELG_prun.py:9
replace
std::string replace(std::string s, const std::string &s2, const std::string &s3)
Definition: hcg.cxx:307
MuonGM::round
float round(const float toRound, const unsigned int decimals)
Definition: Mdt.cxx:27
upper
int upper(int c)
Definition: LArBadChannelParser.cxx:49
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:26
python.CaloAddPedShiftConfig.int
int
Definition: CaloAddPedShiftConfig.py:45
str
Definition: BTagTrackIpAccessor.cxx:11