ATLAS Offline Software
Tools/PyUtils/python/Dso.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
2 
3 
6 
7 __author__ = "Sebastien Binet"
8 
9 __all__ = ['DsoDb']
10 
11 import sys
12 import os
13 import re
14 
15 def _libName(lib):
16  import platform
17  if platform.system() == "Linux":
18  if lib[:3] != "lib": lib = "lib"+lib
19  if lib[-3:] != ".so": lib = lib+".so"
20  return lib
21 
22 
23 _aliases = {
24  'ElementLinkInt_p1' : 'ElementLink_p1<unsigned int>',
25  'basic_string<char>' : 'string',
26  'std::basic_string<char>' : 'string',
27  'vector<basic_string<char> >' : 'vector<string>',
28 
29  'INavigable4MomentumCollection' : 'DataVector<INavigable4Momentum>',
30  'IParticleContainer' : 'DataVector<IParticle>',
31  }
32 _typedefs = {
33 
34  'INavigable4MomentumCollection' : 'DataVector<INavigable4Momentum>',
35  'IParticleContainer' : 'DataVector<IParticle>',
36  }
37 
38 _cpp_builtins = (
39  'char', 'unsigned char', 'signed char',
40  'signed',
41  'short int', 'short signed', 'short signed int',
42  'short', 'unsigned short', 'signed short',
43  'int', 'unsigned int',
44 
45  'long int',
46  'long signed int',
47  'signed long int',
48 
49  'long',
50  'long signed', 'signed long',
51  'unsigned long',
52  'unsigned long int',
53  'long unsigned int',
54 
55  'long long',
56  'long long int',
57  'unsigned long long',
58  'longlong',
59 
60  # no clue from where this one comes from, who's requesting it nor who
61  # got the alien naming scheme idea...
62  'ulonglong',
63 
64  'float',
65  'double',
66  'long double',
67  'bool',
68  )
69 
70 _is_stl_sequence = re.compile (r'std::(?P<ContType>.*?)'
71  r'<(?P<TemplateArg>.*?)'
72  r',\s*?std::allocator<\2> >')
73 _is_stl_mapping = re.compile (r'std::map<'
74  r'(?P<TemplateArg1>.*?),\s*?'
75  r'(?P<TemplateArg2>.*?)'
76  r',\s*?std::allocator<\2> >')
77 
78 
79 
80 
81 
82 def _get_native_libname(libname):
83  """ return the OS-native name from an OS-indenpendent one """
84  plat = sys.platform
85  if plat.count('linux')>0:
86  lib_prefix,lib_suffix = 'lib', '.so'
87  elif plat == 'win32':
88  lib_prefix,lib_suffix = '', '.dll'
89  elif plat == 'darwin':
90  lib_prefix,lib_suffix = 'lib','.dylib'
91  else:
92  raise RuntimeError ("sorry platform [%s] is not (yet?) supported"%plat)
93  _sys_libname = libname
94  if not _sys_libname.startswith (lib_prefix):
95  _sys_libname = ''.join([lib_prefix,_sys_libname])
96  if not _sys_libname.endswith (lib_suffix):
97  _sys_libname = ''.join([_sys_libname, lib_suffix])
98  return _sys_libname
99 
100 def load_library (libname):
101  """
102  Helper method to load a library by its natural name, not the OS-native name.
103  But if the OS-native name is given, it is safely handled too.
104  usage:
105  >>> load_library ('AthenaServices')
106  >>> load_library ('AthenaServicesDict')
107  """
108  _sys_libname = _get_native_libname(libname)
109  import ctypes
110  return ctypes.cdll.LoadLibrary (_sys_libname)
111 
112 def find_library(libname):
113  """
114  Helper function to find the (full)path to a library given its natural name.
115  @return None on failure
116 
117  usage:
118  >>> find_library('AthenaServices')
119  '/afs/cern.ch/.../AtlasCore/[release]/InstallArea/.../libAthenaServices.so
120  """
121  import os
122 
128  _sys_libname = _get_native_libname(libname)
129  # FIXME: REALLY not portable...
130  if os.name != 'posix':
131  raise RuntimeError('sorry OS [%s] is not supported' % os.name)
132 
133  if 'LD_LIBRARY_PATH' in os.environ:
134  for d in os.environ['LD_LIBRARY_PATH'].split(os.pathsep):
135  lib = os.path.join(d, _sys_libname)
136  if os.path.exists(lib):
137  return lib
138  return
139 
140 
141 def _is_rootcint_dict (libname):
142  """helper function to reject rootcint libraries entries from rootmap
143  files (which appeared w/ ROOT v21/22)
144  It seems all of them (and on all platforms) are named like:
145  vector<char>: vector.dll
146  """
147  if libname == ".dll": # pathological case...
148  return False
149  pat = re.compile(r'\w*?.dll')
150  return not (libname.startswith("lib")) and \
151  not (pat.match (libname) is None)
152 
154  """
155  The repository of 'rootmap' files (location, content,...)
156  """
157  def __init__(self):
158  import cppyy # noqa: F401
159  # import root
160  import PyUtils.RootUtils as ru
161  ROOT = ru.import_root()
162  self._cxx = ROOT.Ath.DsoDb.instance()
163 
164  def _to_py(self, cxx):
165  dd = {}
166  kk = self._cxx.py_keys_from(cxx)
167  vv = self._cxx.py_vals_from(cxx)
168  for i in range(kk.size()):
169  dd[kk[i]] = list(vv[i])
170  return dd
171 
172  @property
173  def db(self):
174  return self._to_py(self._cxx.db())
175 
176  @property
177  def pf(self):
178  return self._to_py(self._cxx.pf())
179 
180  def has_type(self, typename):
181  return self._cxx.has_type(typename)
182 
183  def load_type(self, typename):
184  return self._cxx.load_type(typename)
185 
186  def capabilities(self, libname):
187  return list(self._cxx.capabilities(libname))
188 
189  def duplicates(self, libname, pedantic=False):
190  return self._to_py(self._cxx.duplicates(libname, pedantic))
191 
192  def dict_duplicates(self, pedantic=False):
193  return self._to_py(self._cxx.dict_duplicates(pedantic))
194 
195  dictDuplicates = dict_duplicates
196 
197  def pf_duplicates(self, pedantic=False):
198  return self._to_py(self._cxx.pf_duplicates(pedantic))
199 
200  pfDuplicates = pf_duplicates
201 
202  def libs(self, detailed=False):
203  return list(self._cxx.libs(detailed))
204 
205  def content(self, pedantic):
206  return self._to_py(self._cxx.content(pedantic))
207 
208  @property
209  def dso_files(self):
210  return list(self._cxx.dso_files())
211 
212  @property
213  def dsoFiles(self):
214  return self.dso_files
215 
216 def _to_rootmap_name(typename):
217  """
218  helper method to massage a typename into something understandable
219  by the rootmap files
220  """
221  global _aliases
222  typename = typename.replace(', ',',')
223  # first the easy case: builtins
224  if typename in _cpp_builtins:
225  return typename
226  # known missing aliases ?
227  if typename in _aliases.keys():
228  t = _aliases[typename]
229  return _to_rootmap_name(t)
230  # handle default template arguments of STL sequences
231  if _is_stl_sequence.match(typename):
232  # rootmap files do not contain the default template arguments
233  # for STL containers... consistency, again.
234  _m = _is_stl_sequence.match(typename)
235  _m_type = _m.group('TemplateArg')
236  # handle the dreaded 'std::Bla<Foo<d> >
237  _m_type = _to_rootmap_name(_m_type.strip())
238  if _m_type.endswith('>'):
239  _m_type += ' '
240  typename = 'std::%s<%s>' % (_m.group('ContType'),
241  _m_type)
242  # need to massage a bit the typename to match ROOT naming convention
243  typename = typename.replace('std::basic_string<char> ',
244  'string ')
245  typename = typename.replace('std::basic_string<char>',
246  'string')
247  typename = typename.replace('std::', '')
248  typename = typename.replace('> >', '>->')
249  typename = typename.replace(' >', '>')
250  typename = typename.replace('>->', '> >')
251  return typename
252 
253 def _to_rflx_name (typename):
254  """helper method to massage a typename into something understandable
255  by reflex (which doesn't understand the same thing than rootmaps).
256  """
257  global _aliases,_typedefs
258  typename = typename.replace(', ',',')
259  # first the easy case: builtins
260  if typename in _cpp_builtins:
261  return typename
262  # known missing typedefs ?
263  if typename in _typedefs.keys():
264  t = _typedefs[typename]
265  return _to_rflx_name(t)
266  # handle default template arguments of STL sequences
267  if _is_stl_sequence.match(typename):
268  # rootmap files do not contain the default template arguments
269  # for STL containers... consistency, again.
270  _m = _is_stl_sequence.match (typename)
271  _m_type = _m.group('TemplateArg')
272  # handle the dreaded 'std::Bla<Foo<d> >
273  _m_type = _to_rflx_name (_m_type.strip())
274  if _m_type.endswith('>'):
275  _m_type += ' '
276  typename = 'std::%s<%s>' % (_m.group('ContType'), _m_type)
277  typename = typename.replace('std::string>',
278  'std::basic_string<char> >')
279  typename = typename.replace('std::string',
280  'std::basic_string<char>')
281  return typename
282 
283 class PyDsoDb( object ):
284  """
285  The repository of 'rootmap' files (location, content,...)
286  """
287  RootMap = "rootmap"
288  DsoMap = "dsomap"
289  PluginNamespace = "__pf__"
290 
291  def __init__(self, name = "DsoDb"):
292  object.__init__(self)
293  self.name = name
294  self.db = { } # repository of components
295  self.pf = { } # repository of known components to the plugin svc
296 
297  import PyUtils.Logging as _L
298  self.msg = _L.logging.getLogger('DsoDb')
299 
300  self.dsoPath = os.environ['LD_LIBRARY_PATH']
301  self.__buildRepository()
302  return
303 
304  def __buildRepository(self):
305  msg = self.msg
306  self.dsoFiles = set()
307  dsoPath = [p for p in self.dsoPath.split( os.pathsep )
308  if not p.startswith(os.environ['ROOTSYS'])]
309  for path in dsoPath:
310  if not os.path.exists(path): continue
311  dir_content = None
312  try:
313  dir_content = os.listdir(path)
314  except Exception:
315  # try again...
316  try:
317  dir_content = os.listdir(path)
318  except Exception as err:
319  msg.warning("caught:\n%s", err)
320  if dir_content is None:
321  msg.warning("could not run os.listdir on [%s]", path)
322  dir_content = []
323  dsoFiles = [ f for f in dir_content
324  if f.endswith(self.RootMap) ]
325  for dsoFile in dsoFiles:
326  dsoFile = os.path.join( path, dsoFile )
327  if os.path.exists(dsoFile):
328  line_nbr = -1
329  self.dsoFiles.add(dsoFile)
330  for line in open(dsoFile, 'r'):
331  line_nbr += 1
332  line = line.strip()
333  if len(line) <= 0 or line[0] == "#":
334  continue
335  line = line.split()
336  # Note that as of LCG-55, rootmaps have the following
337  # format: 'symbol': libDict.so [listOfLinkedLibs.so...]
338  # we are only interested in libDict.so...
339  try:
340  dsoKey, libName = line[0], line[1]
341  except Exception as err:
342  msg.warning(
343  'could not parse %s:%i', dsoFile, line_nbr
344  )
345  msg.warning(
346  '(some) reflex-dicts may fail to be auto-loaded'
347  )
348  msg.warning(err)
349  continue
350  dsoKey = dsoKey\
351  .replace("Library.", "")\
352  .replace( ":", "" )\
353  .replace( "@", ":" )\
354  .replace( "-", " " )
355  if dsoKey.startswith( self.PluginNamespace ):
356  db = self.pf
357  else:
358  db = self.db
359  if dsoKey not in db: db[dsoKey] = list()
360  if _is_rootcint_dict (libName):
361  #print "## discarding [%s]..." % libName
362  continue
363  libName = os.path.join(path, _libName(libName))
364  db[dsoKey].append(libName)
365  pass # loop over dso-lines
366  pass # loop over dsoFiles
367  pass # iter over dsoPath
368  return
369 
370  def __str__(self):
371  s = os.linesep.join( [
372  "+--- %s ---" % self.name,
373  "|nbr of lib components: %i" % len(self.db.keys()),
374  "|nbr of pf components: %i" % len(self.pf.keys()),
375  "|nbr of dso files: %i" % len(self.dsoFiles),
376  "|nbr of known libs: %i" % len(self.libs()),
377  "+-------------------------"
378  ] )
379 
380  return s
381 
382  def __dups(self, db, pedantic):
383  dups = {}
384  for k in db.keys():
385  if len(db[k]) == 1: continue
386  if pedantic: libs = db[k]
387  else:
388  baseLibs = set()
389  libs = []
390  for lib in db[k]:
391  if os.path.basename(lib) not in baseLibs:
392  libs.append(lib)
393  baseLibs.add(os.path.basename(lib))
394  pass
395  pass
396  if len(libs) > 1:
397  dups[k] = [ lib for lib in libs ]
398  return dups
399 
400  def duplicates(self, libName, pedantic = False):
401  caps = self.capabilities(libName)
402  dups = {}
403  for dupDb in [ self.dictDuplicates(pedantic),
404  self.pfDuplicates(pedantic) ]:
405  for k in dupDb:
406  if k in caps:
407  if k not in dups: dups[k] = []
408  dups[k] += [ lib for lib in dupDb[k]
409  if libName not in os.path.basename(lib) ]
410  dups.keys().sort()
411  for k in dups.keys():
412  dups[k].sort()
413  return dups
414 
415  def dictDuplicates(self, pedantic = False):
416  return self.__dups(self.db, pedantic)
417 
418  def pfDuplicates(self, pedantic = False):
419  return self.__dups(self.pf, pedantic)
420 
421  def capabilities(self, libName):
422  libName = _libName(libName)
423  caps = set()
424  for db in [self.db, self.pf]:
425  for k in db.keys():
426  if libName in [ os.path.basename(lib) for lib in db[k] ]:
427  caps.add( k )
428  caps = [ cap for cap in caps ]
429  caps.sort()
430  if len(caps) == 0:
431  print ("::: ERROR: No such library [%s] in dsoDb !!" % libName)
432  raise ValueError ("")
433  return caps
434 
435  def libs(self, detailedDump = False):
436  if detailedDump: fct = lambda x : x
437  else: fct = os.path.basename
438  libs = set()
439  for db in [self.pf, self.db]:
440  for k in db.keys():
441  for lib in db[k]:
442  libs.add(fct(lib))
443  libs = [ lib for lib in libs ]
444  libs.sort()
445  return libs
446 
447  def content(self, pedantic):
448  d = {}
449  for db in [self.pf, self.db]:
450  for k in db.keys():
451  if pedantic: libs = db[k]
452  else:
453  baseLibs = set()
454  libs = []
455  for lib in db[k]:
456  if os.path.basename(lib) not in baseLibs:
457  libs.append(lib)
458  baseLibs.add(os.path.basename(lib))
459  pass
460  pass
461  d[k] = [ lib for lib in libs ]
462  return d
463 
464  def _to_rootmap_name(self, typename):
465  """
466  helper method to massage a typename into something understandable
467  by the rootmap files
468  """
469  return _to_rootmap_name(typename)
470 
471  def _to_rflx_name (self, typename):
472  """helper method to massage a typename into something understandable
473  by reflex (which doesn't understand the same thing than rootmaps).
474  """
475  return _to_rflx_name(typename)
476 
477 DsoDb = CxxDsoDb
478 #DsoDb = PyDsoDb
python.Dso.CxxDsoDb.dict_duplicates
def dict_duplicates(self, pedantic=False)
Definition: Tools/PyUtils/python/Dso.py:192
replace
std::string replace(std::string s, const std::string &s2, const std::string &s3)
Definition: hcg.cxx:307
python.Dso.PyDsoDb.RootMap
RootMap
Definition: Tools/PyUtils/python/Dso.py:287
python.Dso.PyDsoDb.__str__
def __str__(self)
Definition: Tools/PyUtils/python/Dso.py:370
python.Dso.PyDsoDb.libs
def libs(self, detailedDump=False)
Definition: Tools/PyUtils/python/Dso.py:435
python.Dso.PyDsoDb.__buildRepository
def __buildRepository(self)
Definition: Tools/PyUtils/python/Dso.py:304
python.Dso.find_library
def find_library(libname)
Definition: Tools/PyUtils/python/Dso.py:112
python.Dso.CxxDsoDb.load_type
def load_type(self, typename)
Definition: Tools/PyUtils/python/Dso.py:183
python.Dso.PyDsoDb.dsoFiles
dsoFiles
Definition: Tools/PyUtils/python/Dso.py:306
python.Dso.CxxDsoDb._to_py
def _to_py(self, cxx)
Definition: Tools/PyUtils/python/Dso.py:164
python.Dso.load_library
def load_library(libname)
Definition: Tools/PyUtils/python/Dso.py:100
python.AthDsoLogger.fct
fct
Definition: AthDsoLogger.py:42
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
python.Dso.CxxDsoDb.capabilities
def capabilities(self, libname)
Definition: Tools/PyUtils/python/Dso.py:186
python.Dso.CxxDsoDb.db
def db(self)
Definition: Tools/PyUtils/python/Dso.py:173
python.Dso.CxxDsoDb.duplicates
def duplicates(self, libname, pedantic=False)
Definition: Tools/PyUtils/python/Dso.py:189
python.Dso.CxxDsoDb.has_type
def has_type(self, typename)
Definition: Tools/PyUtils/python/Dso.py:180
python.Dso.PyDsoDb._to_rootmap_name
def _to_rootmap_name(self, typename)
Definition: Tools/PyUtils/python/Dso.py:464
python.Dso._to_rootmap_name
def _to_rootmap_name(typename)
Definition: Tools/PyUtils/python/Dso.py:216
python.Dso.PyDsoDb.dictDuplicates
def dictDuplicates(self, pedantic=False)
Definition: Tools/PyUtils/python/Dso.py:415
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:194
python.Dso.CxxDsoDb.dsoFiles
def dsoFiles(self)
Definition: Tools/PyUtils/python/Dso.py:213
add
bool add(const std::string &hname, TKey *tobj)
Definition: fastadd.cxx:55
histSizes.list
def list(name, path='/')
Definition: histSizes.py:38
python.Dso.CxxDsoDb.libs
def libs(self, detailed=False)
Definition: Tools/PyUtils/python/Dso.py:202
python.Dso.PyDsoDb.__init__
def __init__(self, name="DsoDb")
Definition: Tools/PyUtils/python/Dso.py:291
python.Dso.PyDsoDb.pfDuplicates
def pfDuplicates(self, pedantic=False)
Definition: Tools/PyUtils/python/Dso.py:418
python.Dso.PyDsoDb._to_rflx_name
def _to_rflx_name(self, typename)
Definition: Tools/PyUtils/python/Dso.py:471
python.Dso.PyDsoDb.dsoPath
dsoPath
Definition: Tools/PyUtils/python/Dso.py:300
CxxUtils::set
constexpr std::enable_if_t< is_bitmask_v< E >, E & > set(E &lhs, E rhs)
Convenience function to set bits in a class enum bitmask.
Definition: bitmask.h:232
python.Dso.CxxDsoDb
Definition: Tools/PyUtils/python/Dso.py:153
python.Dso.CxxDsoDb._cxx
_cxx
Definition: Tools/PyUtils/python/Dso.py:162
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
python.Dso._to_rflx_name
def _to_rflx_name(typename)
Definition: Tools/PyUtils/python/Dso.py:253
python.Dso._libName
def _libName(lib)
Definition: Tools/PyUtils/python/Dso.py:15
python.Dso.PyDsoDb.duplicates
def duplicates(self, libName, pedantic=False)
Definition: Tools/PyUtils/python/Dso.py:400
python.Dso.PyDsoDb.pf
pf
Definition: Tools/PyUtils/python/Dso.py:295
python.Dso.PyDsoDb.db
db
Definition: Tools/PyUtils/python/Dso.py:294
Trk::open
@ open
Definition: BinningType.h:40
python.Dso._is_rootcint_dict
def _is_rootcint_dict(libname)
Definition: Tools/PyUtils/python/Dso.py:141
python.Dso.CxxDsoDb.pf_duplicates
def pf_duplicates(self, pedantic=False)
Definition: Tools/PyUtils/python/Dso.py:197
python.Dso.PyDsoDb.content
def content(self, pedantic)
Definition: Tools/PyUtils/python/Dso.py:447
python.Dso.PyDsoDb.capabilities
def capabilities(self, libName)
Definition: Tools/PyUtils/python/Dso.py:421
python.Dso.CxxDsoDb.dso_files
def dso_files(self)
Definition: Tools/PyUtils/python/Dso.py:209
python.Dso.PyDsoDb
Definition: Tools/PyUtils/python/Dso.py:283
pickleTool.object
object
Definition: pickleTool.py:29
python.Dso.PyDsoDb.msg
msg
Definition: Tools/PyUtils/python/Dso.py:298
python.Dso.PyDsoDb.__dups
def __dups(self, db, pedantic)
Definition: Tools/PyUtils/python/Dso.py:382
python.Bindings.keys
keys
Definition: Control/AthenaPython/python/Bindings.py:801
python.Dso.CxxDsoDb.pf
def pf(self)
Definition: Tools/PyUtils/python/Dso.py:177
python.Dso._get_native_libname
def _get_native_libname(libname)
functions --------------------------------------------------------------—
Definition: Tools/PyUtils/python/Dso.py:82
python.Dso.PyDsoDb.name
name
Definition: Tools/PyUtils/python/Dso.py:293
Trk::split
@ split
Definition: LayerMaterialProperties.h:38
python.Dso.CxxDsoDb.content
def content(self, pedantic)
Definition: Tools/PyUtils/python/Dso.py:205
python.Dso.CxxDsoDb.__init__
def __init__(self)
Definition: Tools/PyUtils/python/Dso.py:157
python.Dso.PyDsoDb.PluginNamespace
PluginNamespace
Definition: Tools/PyUtils/python/Dso.py:289