Gen_tf.EvgenExecutor Class Reference

Inheritance diagram for Gen_tf.EvgenExecutor:

Collaboration diagram for Gen_tf.EvgenExecutor:

Public Member Functions
	__init__ (self, name="generate", skeleton=None, skeletonCA=None, substep=None, inData=set(), outData=set())
	preExecute (self, input=set(), output=set())

Detailed Description

Definition at line 26 of file Gen_tf.py.

Constructor & Destructor Documentation

__init__()

Gen_tf.EvgenExecutor.__init__	(		self,
			name = "generate",
			skeleton = None,
			skeletonCA = None,
			substep = None,
			inData = set(),
			outData = set() )

Definition at line 28 of file Gen_tf.py.

  def __init__(self, name="generate", skeleton=None, skeletonCA=None, substep=None, inData=set(), outData=set()):
        athenaExecutor.__init__(self, name=name, skeletonFile=skeleton, skeletonCA=skeletonCA,
                                substep=substep, inputEventTest=False, tryDropAndReload=False,
                                inData=inData, outData=outData)
 

Member Function Documentation

preExecute()

Gen_tf.EvgenExecutor.preExecute	(		self,
			input = set(),
			output = set() )

Definition at line 33 of file Gen_tf.py.

  def preExecute(self, input=set(), output=set()):
        "Get input tarball, unpack and set up env if an evgenJobOpts arg was provided."
 
        
        super(EvgenExecutor, self).preExecute(input, output)
 
        def expand_if_archive(filename):
            "Function to expand a file if it is a zip archive or tarball"
            if ".tar" in filename or ".tgz" in filename:
                import tarfile
                with tarfile.open(filename) as tf:
                    tf.extractall()
            elif filename.endswith(".zip"):
                import zipfile
                zf = zipfile.ZipFile(filename)
                zf.extractall()
                zf.close()
            elif ".gz" in filename:
                shouldExtract = True
                if "avoidExtracting" in self._trf.argdict:
                    shouldExtract = not self._trf.argdict["avoidExtracting"].value
                if shouldExtract:
                    import gzip
                    with gzip.open(filename, 'rb') as f_in:
                       lfilename=os.path.basename(filename)
                       with open(lfilename[:-3], 'wb') as f_out:
                           shutil.copyfileobj(f_in, f_out)
                else:
                    msg.info("Avoid extracting gz input generator file.")
        def get_immediate_subdirectories(a_dir):
            return [name for name in os.listdir(a_dir)
                    if os.path.isdir(os.path.join(a_dir, name))]
 
        
        os.environ['LOCAL_INSTALL_DIR'] = (os.environ['JOBOPTSEARCHPATH']).split(":")[0]
        if os.path.exists('/cvmfs/atlas.cern.ch/repo/sw/Generators/MCJobOptions/common'):
           CommonCvmfsDir = '/cvmfs/atlas.cern.ch/repo/sw/Generators/MCJobOptions/common'
           os.environ["JOBOPTSEARCHPATH"] = CommonCvmfsDir+":"+os.environ["JOBOPTSEARCHPATH"]
        if os.path.exists('/cvmfs/atlas.cern.ch/repo/sw/Generators/MCJobOptions/common/MadGraphControl/dat/'):
            datCvmfsDir = '/cvmfs/atlas.cern.ch/repo/sw/Generators/MCJobOptions/common/MadGraphControl/dat/'
            os.environ["DATAPATH"] = datCvmfsDir+":"+os.environ["DATAPATH"]
#        dsidparam = (self._trf.argdict["jobConfig"].value).values()[0][0]
        dsidpar = (self._trf.argdict["jobConfig"].value).values()
        dsidparam = list(dsidpar)[0][0]
        # Adding cvmfs path to JOBOPTSEARCHPATH
        if os.path.exists('/cvmfs/atlas.cern.ch/repo/sw/Generators/MCJobOptions/'):
           BaseCvmfsPath = "/cvmfs/atlas.cern.ch/repo/sw/Generators/MCJobOptions/"
 
        if dsidparam.isdigit() and (len(dsidparam)==6 or len(dsidparam)==7):
            #only dsid is provided, add cvmfs folder like 123xxx to JOBOPTSEARCHPATH
            Jodir = dsidparam[:3]+'xxx'
            if len(dsidparam)==7:
               Jodir = dsidparam[0] + '/' + dsidparam[:4]+'xxx'
            cwdir = os.getcwd()
            cwd_ful = os.path.join(cwdir, dsidparam)
            if (os.path.isdir(cwd_ful)):
               os.environ["JOBOPTSEARCHPATH"] = cwd_ful+":"+os.environ["JOBOPTSEARCHPATH"]
               os.environ["DATAPATH"] = cwd_ful+":"+os.environ["DATAPATH"]
            else:
               cwd_Jodir = os.path.join(cwdir,Jodir)
               cwd_Jodir_ful = os.path.join(cwd_Jodir,dsidparam)
               if (os.path.isdir(cwd_Jodir_ful)):
                  os.environ["JOBOPTSEARCHPATH"] = cwd_Jodir_ful+":"+os.environ["JOBOPTSEARCHPATH"]
                  os.environ["DATAPATH"] = cwd_Jodir_ful+":"+os.environ["DATAPATH"]
               else:
                  if (os.path.isdir(BaseCvmfsPath)):
                     JoCvmfsPath = os.path.join(BaseCvmfsPath, Jodir)
                     JoCvmfsPath_ful = os.path.join(JoCvmfsPath, dsidparam)
                     os.environ["JOBOPTSEARCHPATH"] = JoCvmfsPath_ful+":"+os.environ["JOBOPTSEARCHPATH"]
                     os.environ["DATAPATH"] = JoCvmfsPath_ful+":"+os.environ["DATAPATH"]
                  else:
                     msg.error("No access to JOs cvmfs location. JOs should be placed in working directory or full path should be given.")
                     raise RuntimeError("JOs not found - no cvmfs access")            
        else:  #Suppose full path of dsid folder is provided(/afs/.../123xxx/123456), add cvmfs floder and local path(/afs/.../123xxx) to JOBOPTSEARCHPATH
            if dsidparam.startswith("Test"):  # for testing
                for token in (os.environ['JOBOPTSEARCHPATH']).split(":"):
                    if "jobOptions" in token:
                        search_token = token
                        break
                search_token += "/EvgenJobTransforms/EvgenTest/" + dsidparam.split("Test")[-1]
                os.environ["JOBOPTSEARCHPATH"] = search_token+":"+os.environ["JOBOPTSEARCHPATH"]
                os.environ["DATAPATH"] = search_token+":"+os.environ["DATAPATH"]
            elif (os.path.isdir(dsidparam)):
                os.environ["JOBOPTSEARCHPATH"] = dsidparam+":"+os.environ["JOBOPTSEARCHPATH"]
                os.environ["DATAPATH"] = dsidparam+":"+os.environ["DATAPATH"]
            else:
                msg.error("JOs not found, please check = '%s'", dsidparam)
                raise RuntimeError("JOs not found")
 
        msg.info("Using JOBOPTSEARCHPATH = '%s'", os.environ["JOBOPTSEARCHPATH"])
        msg.info("Using DATAPATH = '%s'", os.environ["DATAPATH"])
 
        if "evgenJobOpts" in self._trf.argdict: 
            tarball = self._trf.argdict["evgenJobOpts"].value
            
            if tarball.startswith("http"):
                url = tarball
                tarball = os.basename(tarball)
            else:
                url = "http://cern.ch/atlas-computing/links/kitsDirectory/EvgenJobOpts/" + tarball
            
            if not os.path.exists(tarball):
                from EvgenJobTransforms.download import downloadUsingProxy
                status, output = downloadUsingProxy(url)
                if status != 0:
                    raise EnvironmentError('Error downloading tarball %s. Downloader reports: %s' % (tarball, output))
                msg.info('Evgen tarball download success: %s', output)
            
            expand_if_archive(tarball)
            
 
        # copy config files to cwd
        FIRST_DIR = (os.environ['JOBOPTSEARCHPATH']).split(":")[0]
        configFiles = [f for f in os.listdir(FIRST_DIR) if ( "GRID" in f)]
        confFile=None
        if len(configFiles) == 1:
            msg.info("gridpack for only one energy available ")
        elif len(configFiles) >1:
            msg.info("more then one gridpack ! ")
        if len(configFiles) >=1:
          if "--ecmEnergy" in str(sys.argv[1:]):
             split_args=str(sys.argv[1:]).split("ecmEnergy",1)[1]
             split_args=split_args.lstrip("\',=")
             ener_GeV=split_args.split(",")[0].strip(" ,\']")
             energy=str(float(ener_GeV)/1000.0).replace('.','p').strip(r"=0\p']")
             msg.info("Should be used gridpack for energy "+energy)
          else:
             msg.info("no ecm energy given, assuming 13.6 TeV ")
             energy="13p6"
          for x in configFiles:
              gridS="mc_"+energy+"TeV"
              msg.info("Gridpack should start from "+gridS)
              if x.startswith(gridS):
                 confFile = os.path.join(FIRST_DIR, x)
                 msg.info("using gridpack = "+confFile)
          if confFile is None:
             msg.error("No *GRID* config files, for requested energy = '%s'  please check = '%s'", energy, dsidparam)
             sys.exit(1)
 
        if confFile is not None:
           expand_if_archive(confFile)
#       os.system("cp %s ." % confFile)
           msg.info("Configuration input gridpack found " + confFile)
 
        #Expand if a tarball is found in local directory
        loc_files = os.listdir(os.getcwd())
        for loc_file in loc_files:
            if "GRID" not in loc_file:
               expand_if_archive(loc_file)
 
        
        if "inputGeneratorFile" in self._trf.argdict:
#           expand_if_archive(self._trf.argdict["inputGeneratorFile"].value)
            myinputfiles = self._trf.argdict["inputGeneratorFile"].value
            genInputFiles = myinputfiles.split(',')
            for file in genInputFiles:
               expand_if_archive(file)
        if "inputGenConfFile" in self._trf.argdict:
            expand_if_archive(self._trf.argdict["inputGenConfFile"].value)
 

---

The documentation for this class was generated from the following file:

• Gen_tf.py