ATLAS Offline Software
TransformConfig.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
2 
3 __author__ = "clat@hep.ph.bham.ac.uk"
4 
5 import os,sys
6 from PyJobTransformsCore.trferr import TransformConfigError
7 
8 
10  """Base class for all fixed types, implemented as descriptors. Each instance of a specific fixed type
11  (a derived class) can be set either to None (to indicate no value has been set), or to a type specified
12  by the derived class. The type is checked by the derived class by implementing the member function
13  _checkType()"""
14 
15  def __init__(self,doc,default,allowedValues=None):
16  self.__name = '\"%s\"' % doc # Temporary name overwritten by JobConfigMetaClass
17  self.__doc = doc
18  self.__allowed = []
19  if allowedValues is not None:
20  try:
21  allowedValues.__iter__
22  except AttributeError:
23  allowedValues = [ allowedValues ]
24  varName = "%s.allowedValues" % self.__name
25  self.__allowed = self._checkAllowedValues( varName, allowedValues )
26  # allow None as a value for all types
27  if default is not None:
28  # check type and value of default
29  varName = "%s default" % self.__name
30  default = self._checkType( varName, default )
31  default = self._checkValue( varName, default )
32  self.__default = default
33 
34  def __get__(self,instance,owner):
35  if instance is None:
36  return self
37  return instance._attributeDictionary().get(self._attributeName(instance),self.__default)
38 
39 
40  def __set__(self,instance,value):
41  varName = "%s.%s" % (instance.name(), self.__name) # for error/debug printout
42  # allow None as a value for all types
43  if value is not None:
44  # do value and/or type checking and possibly change type and/or value
45  value = self._checkType(varName,value)
46  value = self._checkValue(varName,value)
47  # Do possible additional actions
48  self._setValue(varName,value)
49  instance._attributeDictionary()[self._attributeName(instance)] = value # store the value
50 
51 
52  def _attributeName(self,instance):
53  return '_%s__%s' % (instance.__class__.__name__,self.__name)
54 
55 
56  def name(self):
57  return self.__name
58 
59 
60  def default(self):
61  return self.__default
62 
63 
64  def doc(self):
65  """The documentation string"""
66  return self.__doc
67 
68 
69  def help(self):
70  """The help string: type, documentation and possible values"""
71  hlp = '(%s) %s' % (self.__class__.__name__, self.__doc)
72  if self.__allowed:
73  hlp += '. Possible values: %s' % self.__allowed
74 
75  return hlp
76 
77 
78  def allowedValues(self):
79  return self.__allowed
80 
81 
82  def _checkType(self,variableName,value):
83  """Private helper functin to check the type of <value>. May convert to another type.
84  This implementation does nothing, it just returns value.
85  This function can be overridden in derived class to do type checking.
86  It should return the value (possible with new type), or raise a TransformConfigError
87  exception in case of problems. """
88  return value
89 
90 
91  def _checkValue(self,variableName,value):
92  """Private helper function to check the value of <value>. This function is
93  called after calling _checkType. <value> can therefore be considered to be
94  of the correct type.
95  This implementation checks that the value is one of the allowed values (if defined).
96  This function can be overridden in derived class to do type & additional value checking.
97  It has to return the value (adapted if needed) if all is OK. It has to raise
98  a TransformConfigError exception in case of problems.
99  <variableName> is the name of the variable that is being set and is typically
100  only used for error messages."""
101  if self.__allowed and value not in self.__allowed:
102  raise TransformConfigError( '%s value %r is not in %s' %
103  (variableName, value, self.__allowed) )
104 
105  return value
106 
107 
108  def _setValue(self,variableName,value):
109  """Private helper function which is called when the value of the object is set.
110  It is called after _checkType() and _checkValue(), so the value can be
111  assumed to be correct.
112  This function can be overridden in a derived class, typically to trigger additional action
113  when the value is set.
114  In case of error, raise a TransformConfigError exception, otherwise just return.
115  This implementation does nothing.
116  <variableName> is the name of the variable that is being set and is typically
117  only used for error messages."""
118  pass
119 
120 
121  def _checkAllowedValues(self,variableName,allowedValues):
122  # Check all allowed values
123  for v in allowedValues:
124  self._checkType(variableName,v)
125  return allowedValues
126 
127 
128 
130  def __init__(self,doc,default='',allowedValues=None):
131  Descriptor.__init__(self,doc,default,allowedValues)
132 
133 
134  def _checkValue(self,variableName,value):
135  valType = type(value).__name__
136  if valType != 'str':
137  raise TransformConfigError( '%s should be a string. Got %s instead.' % (variableName, valType) )
138 
139  # check the value against list of possible values
140  return Descriptor._checkValue(self,variableName,value)
141 
142 
143 
145  def __init__(self,doc,default):
146  Descriptor.__init__(self,doc,default,[True,False])
147 
148 
149 
151  def __init__(self,doc,default,allowedValues=None):
152  Descriptor.__init__(self,doc,default,allowedValues)
153 
154 
155  def _checkValue(self,variableName,value):
156  try:
157  value = float(value)
158  except Exception:
159  raise TransformConfigError( '%s value %s is not a float' % (variableName, value) )
160 
161  # check the value against list of possible values
162  return Descriptor._checkValue(self,variableName,value)
163 
164 
165 
167  def __init__(self,doc,default,allowedValues=None):
168  Descriptor.__init__(self,doc,default,allowedValues)
169 
170 
171  def _checkValue(self,variableName,value):
172  inValue = value
173  try:
174  value = int(value)
175  except Exception:
176  raise TransformConfigError( '%s value %s is not an int' % (variableName, value) )
177 
178  if value != float(inValue):
179  raise TransformConfigError( '%s value %s is not an int' % (variableName, value) )
180 
181  # check the value against list of possible values
182  return Descriptor._checkValue(self,variableName,value)
183 
184 
185 
187  """List with a unique set of entries (duplicates are removed).
188  List of allowed values are the entries that are allowed in the list."""
189  def __init__(self,doc,default=None,allowedValues=None):
190  if default is None: default = [] # trick needed to avoid sharing among instances
191  Descriptor.__init__(self,doc,default,allowedValues)
192 
193 
194  def _checkType(self,variableName,value):
195  """Check that <value> is of type list of tuple, and convert to a list if it is a tuple."""
196 # # check types
197 # valType = type(value).__name__
198 # if valType != 'list' and valType != 'tuple':
199 # raise TransformConfigError( '%s should be a list or tuple. Got %s instead.' % \
200 # (variableName, valType) )
201 # if valType == 'tuple':
202 # # convert to a list
203 # value = list(value)
204 # return value
205  try:
206  value.__iter__
207  return list( value )
208  except Exception:
209  raise TransformConfigError( '%s should be a list or tuple. Got %s instead.' %
210  ( variableName, type( value ).__name__ ) )
211 
212 
213  def _checkValue(self,variableName,value):
214  # check that entries are allowed
215  allowed = self.allowedValues()
216  if allowed:
217  for v in value:
218  if v not in allowed:
219  raise TransformConfigError( '%s value %r is not one of %s' %
220  (variableName, value, allowed) )
221  # make entries unique
222  newValue = set() #[]
223  for v in value:
224  newValue.add( v )
225  return list( newValue )
226 
227 
228  def _checkAllowedValues(self,variableName,allowedValues):
229  # Convert values to list before calling _checkType, and back to entry
230  newAllowed = set() # []
231  for v in allowedValues:
232  newAllowed.add( self._checkType(variableName,[v])[0] )
233  return list( newAllowed )
234 
235 
236 
238  """List with a unique set of strings (duplicates are removed)"""
239  def __init__(self,doc,default=None,allowedValues=None):
240  UniqueList.__init__(self,doc,default,allowedValues)
241 
242 
243  def _checkType(self,variableName,value):
244  """Add check that all entries are strings"""
245  # check that <value> is a list or tuple
246  value = UniqueList._checkType(self,variableName,value)
247  # check that all entries are strings
248  for v in value:
249  if not isinstance( v, str ):
250  raise TransformConfigError("Entry %r in %s is not a string (but an %s)" %
251  ( v, variableName, type( v ).__name__ ) )
252  return value
253 
254 
255 
257  def __init__(self,expression):
258  """Any python (string) boolean expression where 'value' stands for the value.
259  For example, for a number range check: \"0 < value < 100\" """
260  self.expression = expression
261 
262 
263  def __contains__(self,value):
264  return eval(self.expression)
265 
266 
267  def __str__(self):
268  return self.expression
269 
270 
271  def __iter__(self):
272  return self
273 
274 
275  def next(self):
276  """No iteration"""
277  raise StopIteration
278 
279 
280  def __next__(self):
281  """No iteration"""
282  raise StopIteration
283 
284 
285 
286 
288  def __init__( self, name, bases, dict ):
289  if '__slots__' not in dict:
290  raise TransformConfigError('Class %s does not have member __slots__. Please add __slots__ = ()'
291  ' to the class definition' % (name,) )
292  #
293  # add list of properties
294  #
295  # first add it to __slots__
296  slots = self.__slots__
297  descrName = '__properties'
298  if descrName not in slots:
299  # add variable
300  slots += ( descrName, )
301  # synchronise dict
302  dict['__slots__'] = slots
303  # then add the list itself
304  setattr(self,descrName,[])
305  descrList = getattr(self,descrName)
306  # set names of properties and add them to the list of properties
307  for n,attrib in dict.items():
308  if isinstance(attrib,Descriptor):
309  setattr(attrib,'_Descriptor__name',n) # noqa: B010 (private property)
310  descrList.append(attrib)
311 
312  type.__init__(self,name,bases,dict)
313 
314 
315 
316 class JobConfig(metaclass=JobConfigMetaClass):
317  __slots__ = ( '__name', '__attributes' )
318 
319  def __init__(self,name=None):
320  """name is used in printout. The default name is derived from the filename of the python file
321  where constructor is called"""
322 
323  if name is None:
324  filename = os.path.basename(sys._getframe(1).f_code.co_filename)
325  name = os.path.splitext(filename)[0]
326  self.__name = name
327  self.__attributes = {} # dictionary to store values of attributes (managed by Descriptors)
328 
329 
330  def __str__(self,prefix=''):
331  prefix += self.__name
332  me = []
333  for descr in self.properties():
334  name = descr.name()
335  value = getattr(self,name)
336  me.append( "%s.%s = %r \t# %s" % (prefix,name,value,descr.help()) )
337  prefix = ' '*len(prefix)
338  return os.linesep.join(me)
339 
340 
341  def __len__(self):
342  """Return the total number of properties (this class plus all base classes)"""
343  return len(self.properties())
344 
345 
346  def __iter__(self):
347  """Provide iteration over the full list of properties (the Descriptors, not the values)"""
348  return iter(self.properties())
349 
350 
352  """Dictionary to store the values. Used in the class Descriptor."""
353  return self.__attributes
354 
355 
356  def properties(self):
357  """Return the full list of properties (the descriptors, not the values)"""
358 
359  descr = []
360  for cl in self.__class__.__bases__ + (self.__class__,):
361  descr += getattr(cl,'__properties') # noqa: B009 (private property)
362  return descr
363 
364 
365  def getProperty(self,name):
366  """Get the property (the descriptor, not the value) named <name>.
367  It raises an TransformConfigError if the class does not have proporty <name>."""
368  for d in self.properties():
369  if d.name() == name: return d
370 
371  raise TransformConfigError('class %s does not have property %s' % (self.__class__.__name__, name))
372 
373 
374  def hasProperty(self,name):
375  """Return bool indicating if this class has property <name>"""
376  try:
377  self.getProperty(name)
378  return True
379  except TransformConfigError:
380  return False
381 
382 
383  def attributes(self):
384  """Dictionary with the attibutes (name:value). It is a copy to make it read-only."""
385  return self.__attributes.copy()
386 
387 
388  def name(self):
389  """The name of the configuration instance (mainly for printout)"""
390  return self.__name
391 
392 
393 
395  """Configuration class for JobTransforms.
396  It has a number of properties shared among all jobtransforms. Those properties are
397  processed by class JobTransform."""
398  __slots__ = ( '__metadata', )
399 
400  # add basic configuration properties
401  auxfiles = ListOfStrings("List of needed auxiliary files (to be found in DATAPATH)")
402  minevents = Integer("minimum number of events in output file",0, AllowedExpression("value >= 0"))
403  maxeventsfactor = Float("maximum number of output events = minevents * maxeventsfactor",2.0,
404  AllowedExpression("value >= 1.0"))
405  maxeventsstrategy = String("what to do if number of input events is less than maxEvents",'INPUTEVENTS',
406  ['IGNORE','INPUTEVENTS','ABORT'])
407  efficiency = Float("Event filtering efficiency (acceptance factor)", 1.0, AllowedExpression("0.0 < value <= 1.0") )
408 
409 
410  def __init__(self,name=None,metaData=None):
411  """name is used in printout. The default name is derived from the filename of the python file
412  where constructor is called"""
413  JobConfig.__init__(self,name)
414  self.__metadata = []
415  if metaData is not None:
416  propertyNames = [ d.name() for d in self.properties() ]
417  for m in metaData:
418  if m not in propertyNames:
419  raise TransformConfigError('Requested metadata %s is not an attribute of %s' % (m,self.name()) )
420  self.__metadata.append(m)
421 
422 
423  def __str__(self,prefix=''):
424  me = JobConfig.__str__(self,prefix)
425  #
426  # add list of metadata
427  #
428  prefix += ' '*len(self.name())
429  me += os.linesep + prefix + 'MetaData attributes: '
430  if self.__metadata:
431  me += ','.join(self.__metadata)
432  else:
433  me += '(None)'
434  return me
435 
436 
437  def metaData(self):
438  """A dictionary of metadata to be added to the metadata of the output files.
439  Values set to None will not be added to the list. Values will be converted to strings.
440  Python lists will be transformed to a string with comma separated entries."""
441  meta = {}
442  for name in self.__metadata:
443  value = getattr(self,name)
444  if value is not None:
445  # turn a python list into a string with comma separated entries
446  if type(value).__name__ == 'list': value = ','.join([str(v) for v in value])
447  meta[name] = str(value)
448  return meta
449 
python.TransformConfig.AllowedExpression.__contains__
def __contains__(self, value)
Definition: TransformConfig.py:263
python.TransformConfig.JobConfig.__attributes
__attributes
Definition: TransformConfig.py:327
createLinkingScheme.iter
iter
Definition: createLinkingScheme.py:62
python.TransformConfig.Descriptor.__allowed
__allowed
Definition: TransformConfig.py:18
python.TransformConfig.AllowedExpression.__iter__
def __iter__(self)
Definition: TransformConfig.py:271
python.TransformConfig.JobConfigMetaClass
Definition: TransformConfig.py:287
python.TransformConfig.JobConfigMetaClass.__init__
def __init__(self, name, bases, dict)
Definition: TransformConfig.py:288
python.TransformConfig.TransformConfig.__init__
def __init__(self, name=None, metaData=None)
Definition: TransformConfig.py:410
python.TransformConfig.Descriptor.__name
__name
Definition: TransformConfig.py:16
python.TransformConfig.Integer._checkValue
def _checkValue(self, variableName, value)
Definition: TransformConfig.py:171
python.TransformConfig.String
Definition: TransformConfig.py:129
python.TransformConfig.Descriptor.__doc
__doc
Definition: TransformConfig.py:17
python.TransformConfig.Descriptor.allowedValues
def allowedValues(self)
Definition: TransformConfig.py:78
python.TransformConfig.TransformConfig
Definition: TransformConfig.py:394
python.TransformConfig.getProperty
def getProperty(self, name)
Definition: TransformConfig.py:365
python.TransformConfig.Descriptor.default
def default(self)
Definition: TransformConfig.py:60
python.TransformConfig.JobConfig
Definition: TransformConfig.py:316
python.TransformConfig.JobConfig._attributeDictionary
def _attributeDictionary(self)
Definition: TransformConfig.py:351
python.TransformConfig.name
def name(self)
Definition: TransformConfig.py:388
python.TransformConfig.AllowedExpression.__next__
def __next__(self)
Definition: TransformConfig.py:280
python.TransformConfig.ListOfStrings.__init__
def __init__(self, doc, default=None, allowedValues=None)
Definition: TransformConfig.py:239
python.TransformConfig.Float.__init__
def __init__(self, doc, default, allowedValues=None)
Definition: TransformConfig.py:151
python.TransformConfig.Descriptor._checkValue
def _checkValue(self, variableName, value)
Definition: TransformConfig.py:91
python.TransformConfig.UniqueList._checkType
def _checkType(self, variableName, value)
Definition: TransformConfig.py:194
python.TransformConfig.ListOfStrings._checkType
def _checkType(self, variableName, value)
Definition: TransformConfig.py:243
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
python.TransformConfig.JobConfig.__len__
def __len__(self)
Definition: TransformConfig.py:341
python.CaloAddPedShiftConfig.type
type
Definition: CaloAddPedShiftConfig.py:42
python.TransformConfig.hasProperty
def hasProperty(self, name)
Definition: TransformConfig.py:374
python.TransformConfig.JobConfig.__init__
def __init__(self, name=None)
Definition: TransformConfig.py:319
python.TransformConfig.AllowedExpression.__str__
def __str__(self)
Definition: TransformConfig.py:267
python.TransformConfig.AllowedExpression.__init__
def __init__(self, expression)
Definition: TransformConfig.py:257
python.TransformConfig.TransformConfig.__metadata
__metadata
Definition: TransformConfig.py:414
python.TransformConfig.Descriptor.__default
__default
Definition: TransformConfig.py:32
python.TransformConfig.Descriptor
Definition: TransformConfig.py:9
python.TransformConfig.UniqueList._checkValue
def _checkValue(self, variableName, value)
Definition: TransformConfig.py:213
python.TransformConfig.UniqueList.__init__
def __init__(self, doc, default=None, allowedValues=None)
Definition: TransformConfig.py:189
python.TransformConfig.JobConfig.properties
def properties(self)
Definition: TransformConfig.py:356
histSizes.list
def list(name, path='/')
Definition: histSizes.py:38
python.TransformConfig.Integer.__init__
def __init__(self, doc, default, allowedValues=None)
Definition: TransformConfig.py:167
python.TransformConfig.String.__init__
def __init__(self, doc, default='', allowedValues=None)
Definition: TransformConfig.py:130
python.TransformConfig.Descriptor.__set__
def __set__(self, instance, value)
Definition: TransformConfig.py:40
python.TransformConfig.AllowedExpression.expression
expression
Definition: TransformConfig.py:260
python.TransformConfig.String._checkValue
def _checkValue(self, variableName, value)
Definition: TransformConfig.py:134
python.TransformConfig.Descriptor._checkType
def _checkType(self, variableName, value)
Definition: TransformConfig.py:82
CxxUtils::set
constexpr std::enable_if_t< is_bitmask_v< E >, E & > set(E &lhs, E rhs)
Convenience function to set bits in a class enum bitmask.
Definition: bitmask.h:232
python.TransformConfig.Descriptor.name
def name(self)
Definition: TransformConfig.py:56
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
python.TransformConfig.Descriptor.doc
def doc(self)
Definition: TransformConfig.py:64
python.TransformConfig.JobConfig.__iter__
def __iter__(self)
Definition: TransformConfig.py:346
python.TransformConfig.UniqueList
Definition: TransformConfig.py:186
python.TransformConfig.Descriptor._setValue
def _setValue(self, variableName, value)
Definition: TransformConfig.py:108
python.TransformConfig.JobConfig.__name
__name
Definition: TransformConfig.py:326
python.TransformConfig.Descriptor._attributeName
def _attributeName(self, instance)
Definition: TransformConfig.py:52
python.TransformConfig.AllowedExpression.next
def next(self)
Definition: TransformConfig.py:275
python.TransformConfig.Integer
Definition: TransformConfig.py:166
python.TransformConfig.TransformConfig.__str__
def __str__(self, prefix='')
Definition: TransformConfig.py:423
python.TransformConfig.Float._checkValue
def _checkValue(self, variableName, value)
Definition: TransformConfig.py:155
python.CaloAddPedShiftConfig.int
int
Definition: CaloAddPedShiftConfig.py:45
python.TransformConfig.Float
Definition: TransformConfig.py:150
python.TransformConfig.ListOfStrings
Definition: TransformConfig.py:237
python.TransformConfig.Descriptor.help
def help(self)
Definition: TransformConfig.py:69
python.TransformConfig.AllowedExpression
Definition: TransformConfig.py:256
python.TransformConfig.Descriptor._checkAllowedValues
def _checkAllowedValues(self, variableName, allowedValues)
Definition: TransformConfig.py:121
python.TransformConfig.UniqueList._checkAllowedValues
def _checkAllowedValues(self, variableName, allowedValues)
Definition: TransformConfig.py:228
python.TransformConfig.Descriptor.__get__
def __get__(self, instance, owner)
Definition: TransformConfig.py:34
get
T * get(TKey *tobj)
get a TObject* from a TKey* (why can't a TObject be a TKey?)
Definition: hcg.cxx:127
python.TransformConfig.TransformConfig.metaData
def metaData(self)
Definition: TransformConfig.py:437
python.TransformConfig.Boolean.__init__
def __init__(self, doc, default)
Definition: TransformConfig.py:145
pickleTool.object
object
Definition: pickleTool.py:29
str
Definition: BTagTrackIpAccessor.cxx:11
calibdata.copy
bool copy
Definition: calibdata.py:26
python.TransformConfig.JobConfig.__str__
def __str__(self, prefix='')
Definition: TransformConfig.py:330
python.TransformConfig.Descriptor.__init__
def __init__(self, doc, default, allowedValues=None)
Definition: TransformConfig.py:15
python.TransformConfig.Boolean
Definition: TransformConfig.py:144
python.TransformConfig.attributes
def attributes(self)
Definition: TransformConfig.py:383
python.LArMinBiasAlgConfig.float
float
Definition: LArMinBiasAlgConfig.py:65