python.ConfigurationShelve Namespace Reference

Classes
class	ConfigurationJar
class	ConfigurationShelve
	representation of job databases ========================================== More...

Functions
def	storeJobOptionsCatalogue (cfg_fname)
def	loadJobOptionsCatalogue (cfg_fname)
def	saveToAscii (out, cfgName=None)
	further saving utilities ================================================= More...
def	loadFromAscii (fileName, cfgName=None)
def	saveToPickle (fileName, cfgName=None)
def	loadFromPickle (fileName, cfgName=None)
def	cmpConfigs (ref, chk, refName=None, chkName=None)

Variables
string	__version__ = '1.1.0'
	data ------------------------------------------------------------------— More...
string	__author__
list	__all__

Function Documentation

cmpConfigs()

def python.ConfigurationShelve.cmpConfigs	(		ref,
			chk,
			refName = None,
			chkName = None
	)

Helper function to compare 2 configurations.
 @param ref the path to a pickle file where a configuration was stored
 @param chk the path to a pickle file where a configuration was stored
 @param refName the name of the configuration jar to load
 @param chkName the name of the configuration jar to load
if `ref` and/or `chk` are joboptions (ie: their extension is '.py') a shelve
is automatically created on the fly

Definition at line 358 of file ConfigurationShelve.py.

def cmpConfigs (ref, chk, refName=None, chkName=None):
   """
   Helper function to compare 2 configurations.
    @param ref the path to a pickle file where a configuration was stored
    @param chk the path to a pickle file where a configuration was stored
    @param refName the name of the configuration jar to load
    @param chkName the name of the configuration jar to load
   if `ref` and/or `chk` are joboptions (ie: their extension is '.py') a shelve
   is automatically created on the fly
   """
   import os
   _path = os.path
   ref = _path.expanduser(_path.expandvars(ref))
   chk = _path.expanduser(_path.expandvars(chk))
   
   from PyUtils.Decorators import forking
   # if we are given joboptions, create ConfigurationShelves on the fly
   @forking
   def _create_shelve (joboname):
      from tempfile import NamedTemporaryFile
      fname = os.path.basename (joboname)
      fname = os.path.splitext(fname)[0] + '.pkl'
      fname = NamedTemporaryFile(prefix=fname, suffix='.pkl').name
      job = [ "from AthenaCommon.Include import include",
              "include ('%s')" % joboname,
              "from AthenaCommon.ConfigurationShelve import saveToPickle",
              "saveToPickle ('%s')" % fname,
              # exit before actually running the job
              "raise SystemExit(0)", 
              ]
      jobofile = NamedTemporaryFile(suffix='.py')
      map (jobofile.writelines, [l+os.linesep for l in job])
      jobofile.flush()
 
      from subprocess import getstatusoutput
      sc,out = getstatusoutput ('athena.py %s' % jobofile.name)
 
      jobofile.close()
      if sc==0:
         return fname
      return (sc, out)
 
   if os.path.splitext(ref)[1]=='.py':
      print ("::: creating a shelve on the fly for [%s]..."%ref)
      ref = _create_shelve (ref)
      if not isinstance(ref, str):
         raise RuntimeError (
            'could not create a shelve on the fly:\n%s'%ref[1])
      import atexit
      atexit.register (os.unlink, ref)
      
   if os.path.splitext(chk)[1]=='.py':
      print ("::: creating a shelve on the fly for [%s]..."%chk)
      chk = _create_shelve (chk)
      if not isinstance(chk, str):
         raise RuntimeError (
            'could not create a shelve on the fly:\n%s'%chk[1])
      import atexit
      atexit.register (os.unlink, chk)
      
   for fname in (ref, chk):
      if not _path.exists (fname):
         raise RuntimeError ('file [%s] does not exist'%fname)
 
   @forking
   def _dict_cfg (fname, cfgName):
      #from PyUtils.odict import OrderedDict as odict
      jar = loadFromPickle (fname, cfgName)
      d = list()
      all_cfgs = dict()
      all_cfgs.setdefault (dict)
      def _visit_cfg (cfg):
         name = cfg.getJobOptName()
         values = [name]
         all_cfgs[name] = dict()
         props = cfg.getProperties()
         def _get_value (cfg, k, v):
            _no_value = cfg.propertyNoValue
            if v == _no_value:
               v = cfg.getDefaultProperty(k)
            return v
 
         from AthenaCommon.Configurable import Configurable
         for k,v in props.items():
            if not isinstance(v, Configurable):
               all_cfgs[name][k] = _get_value(cfg,k,v)
            else:
               all_cfgs[name][k] = v.getJobOptName()
               if v not in cfg.getAllChildren():
                  values.extend (_visit_cfg(v))
 
         if hasattr(cfg, 'getAllChildren'):
            for c in cfg.getAllChildren():
               values.extend (_visit_cfg(c))
         return values
 
      for c in jar.athAlgSeq.getChildren():
         d.extend (_visit_cfg(c))
 
      for c in jar.ServiceMgr.getChildren():
         d.extend (_visit_cfg(c))
      return d, all_cfgs
 
   ref_list, ref_cfgs = _dict_cfg (ref, refName)
   chk_list, chk_cfgs = _dict_cfg (chk, chkName)
 
   report = []
   addNames,subNames = [],[]
   if ref_list != chk_list:
      report.append ("::: configurable list in 'ref' and in 'chk' differ !")
      addNames = sorted([ n for n in chk_cfgs if not (n in ref_cfgs) ])
      if len (addNames) > 0:
         report.append ("::: configurables in 'chk' and not in 'ref':")
         report.extend (["  + %s" % n for n in addNames])
      subNames = sorted([ n for n in ref_cfgs if not (n in chk_cfgs) ])
      if len (subNames) > 0:
         report.append ("::: configurables in 'ref' and not in 'chk':")
         report.extend (["  - %s" % n for n in subNames])
 
   common_cfgs = set (
      [n for n in ref_list if not (n in addNames or n in subNames)] +
      [n for n in chk_list if not (n in addNames or n in subNames)]
      )
   for n in common_cfgs:
      ref_cfg = ref_cfgs[n]
      chk_cfg = chk_cfgs[n]
      diff_props_report = []
      for k in ref_cfg.iterkeys():
         ref_val = str(ref_cfg.get(k, '**N/A**'))
         chk_val = str(chk_cfg.get(k, '**N/A**'))
         
         if ref_val != chk_val:
            diff_props_report.append (" ref: %s = %r" % (k, ref_val))
            diff_props_report.append (" chk: %s = %r" % (k, chk_val))
      if len(diff_props_report)>0:
         report.append ("::: properties differing for [%s]:"%n)
         report.extend (diff_props_report)
   return (ref_list, ref_cfgs), (chk_list, chk_cfgs), report

loadFromAscii()

def python.ConfigurationShelve.loadFromAscii	(		fileName,
			cfgName = None
	)

Helper function to retrieve a configuration from an ASCII file.

Definition at line 329 of file ConfigurationShelve.py.

def loadFromAscii(fileName, cfgName=None):
   """
   Helper function to retrieve a configuration from an ASCII file.
   """
   raise NotImplementedError("Sorry no restoration from ASCII file (yet?)")
 

loadFromPickle()

def python.ConfigurationShelve.loadFromPickle	(		fileName,
			cfgName = None
	)

Helper function to retrieve a configuration from a pickle file

Definition at line 346 of file ConfigurationShelve.py.

def loadFromPickle(fileName, cfgName=None):
   """
   Helper function to retrieve a configuration from a pickle file
   """
   if cfgName is None: cfgName = "Snapshot"
   import os
   fileName = os.path.expanduser(os.path.expandvars(fileName))
   shelve = ConfigurationShelve( fileName )
   # configuration is loaded by side-effects...
   jar = shelve.retrieve( cfgName )
   return jar
 

loadJobOptionsCatalogue()

def python.ConfigurationShelve.loadJobOptionsCatalogue

(

cfg_fname

)

Load properties from a pickle file, previously dumped by
storeConfiguration, back into the JobOptionsSvc.

Definition at line 245 of file ConfigurationShelve.py.

def loadJobOptionsCatalogue( cfg_fname ):
   """Load properties from a pickle file, previously dumped by
   storeConfiguration, back into the JobOptionsSvc.
   """
 # read jobopt catalogue dump and pycomps back in
   cfg = open( cfg_fname, 'rb' )
 
   jocat   = pickle.load( cfg )
   jocfg   = pickle.load( cfg )
   pycomps = pickle.load( cfg )
 
   cfg.close()
 
   kw = jocfg[ 'ApplicationMgr' ]
   from AthenaCommon.AppMgr import theApp
   theApp.JobOptionsSvcType = kw[ 'JobOptionsSvcType' ]
   theApp.MessageSvcType    = kw[ 'MessageSvcType' ]
   theApp.getHandle( kw )
   del jocfg[ 'ApplicationMgr' ]
 
 # no longer want to call setup(), since there are no Configurables to
 # setup; it would be a no-op, already, but __build_master_sequence is
 # broken, so make sure this can't be run ...
   def noop( self ):
      pass
   theApp.__class__.setup = noop
 
   import AthenaPython.PyAthena as PyAthena
   josvc = PyAthena.py_svc( 'JobOptionsSvc', createIf = False, iface = 'Gaudi::Interfaces::IOptionsSvc')
 
 # restore job catalogue entries
   import GaudiPython.Bindings as gaudi
   for client in jocat:
      if client == "ApplicationMgr":
         # ApplicationMgr properties are already set
         continue
      for n,v in jocat[ client ].items():
         josvc.set( client+'.'+n, v )
 
 # restore special services properties
   for client in jocfg:
      svc = PyAthena.py_svc( client, createIf = False, iface='IProperty' )
      for n,v in jocfg[ client ].items():
         # See comment above.
         p = gaudi.StringProperty()
         p.setName(n)
         p.fromString(v).ignore()
         svc.setProperty( p )
 
 # pycomps hack-around
   if pycomps:
      import AthenaPython.Configurables as _C
      _C.PyComponents.instances = dict( (p.name, p) for p in pycomps )
      for p in pycomps:
         if hasattr( p, 'setup' ):
            if callable( p.setup ):
                p.setup()
 
 

saveToAscii()

def python.ConfigurationShelve.saveToAscii	(		out,
			cfgName = None
	)

further saving utilities =================================================

Helper function to store the current configuration to (ASCII) file

Definition at line 305 of file ConfigurationShelve.py.

def saveToAscii(out, cfgName=None):
   """
   Helper function to store the current configuration to (ASCII) file
   """
   import os
 
 # temporarily set the Configurable log level to make sure that the printout
 # does not get suppressed
   from AthenaCommon import Configurable
   from AthenaCommon import Constants
   llevel = Configurable.log.level
   Configurable.log.setLevel( Constants.INFO )
 
   if cfgName is None: cfgName = "Snapshot"
   content = ConfigurationJar( cfgName ).content()
 
   if type(out) is str:
      out = open( out, "w" )
   for v in content.values(): out.write( str(v) + os.linesep )
 
   Configurable.log.setLevel( llevel )
 
   return
 

saveToPickle()

def python.ConfigurationShelve.saveToPickle	(		fileName,
			cfgName = None
	)

Helper function to store the current configuration to a pickle file

Definition at line 335 of file ConfigurationShelve.py.

def saveToPickle(fileName, cfgName=None):
   """
   Helper function to store the current configuration to a pickle file
   """
   if cfgName is None: cfgName = "Snapshot"
   shelve = ConfigurationShelve( fileName )
   jar = ConfigurationJar( cfgName )
   shelve.store( jar )
   
   return
 

storeJobOptionsCatalogue()

def python.ConfigurationShelve.storeJobOptionsCatalogue

(

cfg_fname

)

Inspect the 'configured' JobOptionsSvc and existing services,
then dump the properties of each component into a pickle file.

Definition at line 134 of file ConfigurationShelve.py.

def storeJobOptionsCatalogue( cfg_fname ):
   """Inspect the 'configured' JobOptionsSvc and existing services,
   then dump the properties of each component into a pickle file.
   """
 
   from collections import defaultdict
   jocat = defaultdict( dict )
   jocfg = defaultdict( dict )
 
   def _fillCfg( client, props ):
     for p in props:
        n, v = p
        if hasattr( v, 'toStringProperty' ):
           v = str( v.toStringProperty() )
        elif hasattr( v, 'toString' ):
           v = str( v.toString() )
        elif type (v) is float:
           # str(1.8e12) will give '1.8e+12' in py2
           # and `1800000000000.0' in py3.
           # Convert floats like this for consistency.
           v = '%g'%v
        else:
           v = str( v )
        jocfg[ client ][ n ] = v
 
   from AthenaCommon.AppMgr import theApp
   from AthenaCommon.AppMgr import ServiceMgr as svcMgr
   from AthenaCommon.Configurable import Configurable as C
 
 # tickle the Gaudi C++ side and configure the ApplicationMgr
   theApp.setup( recursive = True )
 
   app_props = [ (k,v.value())
                 for k,v in theApp.getHandle().properties().items() ]
   _fillCfg( 'ApplicationMgr', app_props )
 
 # get all services that now already exist, as they require special treatment;
 # the appmgr handle talks directly to the ServiceManager and services() returns
 # a copy of the services names
   svcs = theApp.getHandle().services()
 
 # now assume that if these services carry configuration, then they should exist
 # on the service manager configurable
   for svcname in svcs:
      svc = getattr( svcMgr, svcname, None )
      if svc:
         _fillCfg( svcname, svc.getValuedProperties().items() )
 
 # make sure to propagate the EventLoop properties through the josvc
   try:
      evLoopName = theApp.EventLoop.split('/')[-1]
      evLoop = getattr( svcMgr, evLoopName )
 
      props = []
      for k,v in evLoop.properties().items():
         if v != C.propertyNoValue:
            props.append( (k,v) )
      _fillCfg( evLoopName, props )
 
   except AttributeError:
      pass    # no properties defined for EventLoop type
 
 # get the values for all other components (these may contain duplicates of
 # the ones above in svcs, and there may even be conflicts)
   import AthenaPython.PyAthena as PyAthena
   josvc = PyAthena.py_svc( 'JobOptionsSvc',iface="Gaudi::Interfaces::IOptionsSvc")
   allProps=josvc.items() #a std::tuple<string,string>
   for prop in allProps: 
      cn=str(prop._0).split(".")
      v=str(prop._1)
      client=".".join(cn[:-1])
      n=cn[-1]
      jocat[ client ][ n ] = v
 
 # take care of some ancient history
   for k in ( 'Go', 'Exit' ):
      if k in jocfg[ 'ApplicationMgr' ]:
         del jocfg[ 'ApplicationMgr' ][k]
 
 # workaround for pycomps
   pycomps = []
   for c in C.allConfigurables.values():
      if not isinstance( c, (PyAthena.Alg,
                             PyAthena.AlgTool,
                             PyAthena.Svc,
                             PyAthena.Aud) ):
         continue
    # FIXME: should check if the component is still 'active'
    #        ie: is it in one of the TopAlg,SvcMgr,ToolSvc and children ?
      try:
         delattr( c, 'msg' )
      except AttributeError:
         pass
 
      pycomps.append( c )
 
 # write out all parts; start with the jocat, so that it can be loaded by
 # itself only, if need be
 
   cfg = open( cfg_fname, 'wb' )             # output pickle file
 
   pickle.dump( jocat, cfg )                 # jobopt catalogue contents
   pickle.dump( jocfg, cfg )                 # special services' properties
   pickle.dump( pycomps, cfg )               # python components workaround
 
   cfg.close()
 
   return cfg_fname
 
 
#----- loading from DB

Variable Documentation

__all__

list python.ConfigurationShelve.__all__

private

Initial value:

=  [ 'ConfigurationShelve', 'ConfigurationJar',
            'saveToAscii',  'loadFromAscii',
            'saveToPickle', 'loadFromPickle',
            'cmpConfigs',
            ]

Definition at line 30 of file ConfigurationShelve.py.

__author__

string python.ConfigurationShelve.__author__

private

Initial value:

=  """\
Wim Lavrijsen (WLavrijsen@lbl.gov),
Sebastien Binet <binet@cern.ch>"""

Definition at line 26 of file ConfigurationShelve.py.

__version__

string python.ConfigurationShelve.__version__ = '1.1.0'

private

data ------------------------------------------------------------------—

Definition at line 25 of file ConfigurationShelve.py.