ATLAS Offline Software
Namespaces | Classes | Functions | Variables
python.Utils Namespace Reference

Namespaces

 unixtools
 

Classes

class  Campaign
 

Functions

def loadDefaultComps (allcomps)
 
def exposeHandles (allcomps)
 
def setupLoggingLevels (flags, ca)
 
def getRunFromName (name, default='', asInt=False)
 
def fileListSnippet (files, dsName, taskName, jobDir=None)
 
def blankIfNone (s)
 
def getUserName (default='UNKNOWN')
 
def getHostName ()
 
def getMCCampaign (files)
 
def getEDMVersionFromBS (filename)
 
def edmDictToList (edmDict)
 
def edmListToDict (edmList)
 
def getEDMListFromWriteHandles (configurables)
 

Variables

string __version__ = '1.0.0'
 
string __author__ = 'Wim Lavrijsen (WLavrijsen@lbl.gov)'
 
list __all__ = [ 'unixtools' ]
 
 campaign_runs
 
 log
 
 maxsize
 

Function Documentation

◆ blankIfNone()

def python.Utils.blankIfNone (   s)

Definition at line 41 of file InnerDetector/InDetExample/InDetBeamSpotExample/python/Utils.py.

41 def blankIfNone(s):
42  if s:
43  return s
44  else:
45  return ''
46 
47 

◆ edmDictToList()

def python.Utils.edmDictToList (   edmDict) 
Convert EDM dictionary in the format:
    {'type1': ['key1','key2'], 'type2': ['key3']}
to a flat list in the format:
    ['type1#key1', 'type1#key2', 'type2#key3']

Definition at line 68 of file Trigger/TriggerCommon/TrigEDMConfig/python/Utils.py.

68 def edmDictToList(edmDict):
69  '''
70  Convert EDM dictionary in the format:
71  {'type1': ['key1','key2'], 'type2': ['key3']}
72  to a flat list in the format:
73  ['type1#key1', 'type1#key2', 'type2#key3']
74  '''
75  return [ f"{type}#{name}" for type, names in edmDict.items() for name in names ]
76 
77 

◆ edmListToDict()

def python.Utils.edmListToDict (   edmList) 
Convert EDM list in the format:
    ['type1#key1', 'type1#key2', 'type2#key3']
to a dictionary in the format:
    {'type1': ['key1','key2'], 'type2': ['key3']}

Definition at line 78 of file Trigger/TriggerCommon/TrigEDMConfig/python/Utils.py.

78 def edmListToDict(edmList):
79  '''
80  Convert EDM list in the format:
81  ['type1#key1', 'type1#key2', 'type2#key3']
82  to a dictionary in the format:
83  {'type1': ['key1','key2'], 'type2': ['key3']}
84  '''
85  edmDict = defaultdict(list)
86  for typeName in edmList:
87  edmType, edmKey = typeName.split('#')
88  edmDict[edmType].append(edmKey)
89  return edmDict
90 
91 

◆ exposeHandles()

def python.Utils.exposeHandles (   allcomps) 
Sets all handle keys explicitly

Definition at line 32 of file Control/AthenaConfiguration/python/Utils.py.

32 def exposeHandles(allcomps):
33  """Sets all handle keys explicitly"""
34  def __getDefault(d):
35  if isinstance(d, collections.abc.Sequence):
36  return [el for el in d]
37  else:
38  return d.Path
39 
40  for comp in allcomps:
41  for propName, value in comp._descriptors.items():
42  if propName in comp._properties: continue
43  if "HandleKey" in value.cpp_type:
44  comp._properties[propName] = __getDefault(value.default)
45 

◆ fileListSnippet()

def python.Utils.fileListSnippet (   files,
  dsName,
  taskName,
  jobDir = None 
)

Definition at line 30 of file InnerDetector/InDetExample/InDetBeamSpotExample/python/Utils.py.

30 def fileListSnippet(files,dsName,taskName,jobDir=None):
31  s = ''
32  if files:
33  for f in files.split():
34  if jobDir and not os.path.exists('%s/%s/%s/%s' % (jobDir,dsName,taskName,f)):
35  s += '%s (archived)<br>' % f
36  else:
37  s += '<a href="/jobfiles/%s/%s/%s">%s</a><br>' % (dsName,taskName,f,f)
38  return s
39 
40 

◆ getEDMListFromWriteHandles()

def python.Utils.getEDMListFromWriteHandles (   configurables) 
Build OutputStream ItemList from all WriteHandles in a list of components (configurables),
for example a list of AlgTools. The output is in flat list format:
    ['type1#key1', 'type1#key2', 'type2#key3']

Definition at line 92 of file Trigger/TriggerCommon/TrigEDMConfig/python/Utils.py.

92 def getEDMListFromWriteHandles(configurables):
93  '''
94  Build OutputStream ItemList from all WriteHandles in a list of components (configurables),
95  for example a list of AlgTools. The output is in flat list format:
96  ['type1#key1', 'type1#key2', 'type2#key3']
97  '''
98 
99  def getWriteHandles(comp):
100  properties = [getattr(comp,propName) for propName in comp.getDefaultProperties().keys()]
101  return [prop for prop in properties if isinstance(prop,DataHandle) and prop.mode()=='W']
102 
103  def formatItem(containerType, containerKey):
104  auxType = containerType.replace('Container','AuxContainer')
105  return [f'{containerType}#{containerKey}',
106  f'{auxType}#{containerKey}Aux.-']
107 
108  def containerTypedef(containerType):
109  if containerType.startswith('xAOD::') and containerType.endswith('Container'):
110  # Already the right typedef
111  return containerType
112  m = re.match(r'DataVector<xAOD::(\w+)_v[0-9]+,', containerType)
113  if m and len(m.groups()) > 0:
114  return f'xAOD::{m.group(1)}Container'
115  raise RuntimeError(f'Failed to convert type {containerType} into a container typedef')
116 
117  def itemListFromConfigurable(comp):
118  items = []
119  for handle in getWriteHandles(comp):
120  sgKey = handle.Path.replace('StoreGateSvc+','')
121  if not sgKey:
122  continue
123  items += formatItem(containerTypedef(handle.Type), handle.Path)
124  return items
125 
126  itemList = []
127  for comp in configurables:
128  itemList += itemListFromConfigurable(comp)
129  return itemList

◆ getEDMVersionFromBS()

def python.Utils.getEDMVersionFromBS (   filename) 
Determine Trigger EDM version based on the input ByteStream file.

Run-3 EDM is indicated by HLT ROD version > 1.0. For Run 1 and 2 the
HLT ROD version was 0.0 and the run number is used to disambiguate between them.

Definition at line 11 of file Trigger/TriggerCommon/TrigEDMConfig/python/Utils.py.

11 def getEDMVersionFromBS(filename):
12  """Determine Trigger EDM version based on the input ByteStream file.
13 
14  Run-3 EDM is indicated by HLT ROD version > 1.0. For Run 1 and 2 the
15  HLT ROD version was 0.0 and the run number is used to disambiguate between them.
16  """
17 
18  boundary_run12 = 230000
19  boundary_run23 = 368000
20 
21  import eformat
22  from libpyeformat_helper import SubDetector
23  bs = eformat.istream(filename)
24 
25  rodVersionM = None
26  rodVersionL = None
27  # Find the first HLT ROBFragment in the first event
28  for robf in bs[0]:
29  if robf.rob_source_id().subdetector_id()==SubDetector.TDAQ_HLT:
30  rodVersionM = robf.rod_minor_version() >> 8
31  rodVersionL = robf.rod_minor_version() & 0xFF
32  log.debug("HLT ROD minor version from input file is %d.%d", rodVersionM, rodVersionL)
33  break
34 
35  # Case 1: failed to read ROD version
36  if rodVersionM is None or rodVersionL is None:
37  log.warning("Cannot determine HLT ROD version from input file, falling back to run-number-based decision")
38  # Case 2: ROD version indicating Run 3
39  elif rodVersionM >= 1:
40  log.info("Determined EDMVersion to be 3, because running on BS file with HLT ROD version %d.%d",
41  rodVersionM, rodVersionL)
42  return 3
43 
44  # Case 3: ROD version indicating Run 1 or 2 - use run number to disambiguate
45  runNumber = bs[0].run_no()
46  log.debug("Read run number %s", runNumber)
47 
48  if not runNumber or runNumber <= 0:
49  log.warning("Cannot determine EDM version because run number %s is invalid. ", runNumber)
50  return None
51  elif runNumber < boundary_run12:
52  # Run-1 data
53  log.info("Determined EDMVersion to be 1 based on BS file run number (runNumber < %d)",
54  boundary_run12)
55  return 1
56  elif runNumber < boundary_run23:
57  # Run-2 data
58  log.info("Determined EDMVersion to be 2 based on BS file run number (%d < runNumber < %d)",
59  boundary_run12, boundary_run23)
60  return 2
61  else:
62  # Run-3 data
63  log.info("Determined EDMVersion to be 3 based on BS file run number (runNumber > %d)",
64  boundary_run23)
65  return 3
66 
67 

◆ getHostName()

def python.Utils.getHostName ( )

Definition at line 60 of file InnerDetector/InDetExample/InDetBeamSpotExample/python/Utils.py.

60 def getHostName():
61  import platform
62  return platform.node()

◆ getMCCampaign()

def python.Utils.getMCCampaign (   files)

Definition at line 27 of file Tools/Campaigns/python/Utils.py.

27 def getMCCampaign(files):
28  # Auto-configure from file
29  from AthenaConfiguration.AutoConfigFlags import GetFileMD
30  metadata = GetFileMD(files)
31  mc_campaign = Campaign(metadata.get('mc_campaign', ''))
32  project_name = metadata.get('project_name', '')
33  run_numbers = metadata.get('runNumbers', [])
34 
35  if mc_campaign is not Campaign.Unknown:
36  return mc_campaign
37 
38  if run_numbers:
39  mc_campaign = campaign_runs.get(run_numbers[0], Campaign.Unknown)
40 
41  # MC-equivalent projects for data
42  if 'data24' in project_name:
43  return Campaign.MC23e
44  elif 'data23' in project_name:
45  return Campaign.MC23c
46  elif 'data22' in project_name:
47  return Campaign.MC21a
48  elif 'data18' in project_name:
49  return Campaign.MC20e
50  elif 'data17' in project_name:
51  return Campaign.MC20d
52  elif 'data16' in project_name or 'data15' in project_name:
53  return Campaign.MC20a
54 
55  return mc_campaign

◆ getRunFromName()

def python.Utils.getRunFromName (   name,
  default = '',
  asInt = False 
) 
Extract the run number from a file name whose first part is a standard dataset name.
   If the run number cannot be determined, returns the default value, which, by
   default is an empty string.

Definition at line 13 of file InnerDetector/InDetExample/InDetBeamSpotExample/python/Utils.py.

13 def getRunFromName(name,default='',asInt=False):
14  """Extract the run number from a file name whose first part is a standard dataset name.
15  If the run number cannot be determined, returns the default value, which, by
16  default is an empty string."""
17  name = os.path.basename(name)
18  try:
19  run = re.sub('^0*','',name.split('.')[1])
20  if not re.search(r'^\d+$',run):
21  # Probably wasn't the run number, so use the default instead
22  run = default
23  else:
24  run = int(run) if asInt else run
25  except Exception:
26  run = default
27  return run
28 
29 

◆ getUserName()

def python.Utils.getUserName (   default = 'UNKNOWN') 
Get login name in a platform-independent manner.

Definition at line 48 of file InnerDetector/InDetExample/InDetBeamSpotExample/python/Utils.py.

48 def getUserName(default='UNKNOWN'):
49  """Get login name in a platform-independent manner."""
50  user = ''
51  try:
52  user = os.getlogin() # this doesn't seem to work with acrontab
53  except Exception:
54  pass
55  if not user:
56  user = os.getenv('USER',default)
57  return user
58 
59 

◆ loadDefaultComps()

def python.Utils.loadDefaultComps (   allcomps) 
Attempts to load all default components (those that are not actually configured)

Definition at line 6 of file Control/AthenaConfiguration/python/Utils.py.

6 def loadDefaultComps(allcomps):
7  """Attempts to load all default components (those that are not actually configured)"""
8  loadedSome = False
9  def __load(comp, prop):
10  descr = comp._descriptors[prop]
11  if descr.default.getType() == "":
12  return
13  try:
14  childComp = CompFactory.getComp(descr.default.getType())(descr.default.getName())
15  comp._properties[prop] = childComp
16  nonlocal loadedSome
17  loadedSome = True
18  except Exception:
19  msg=logging.getLogger('loadDefaultComps')
20  msg.warning("Default component %s can not be loaded", descr.default.typeAndName )
21  pass
22 
23  for comp in allcomps:
24  for propName,value in comp._descriptors.items():
25  if propName in comp._properties: continue
26  if isinstance(value.default, GaudiHandle):
27  __load(comp, propName )
28 
29  if loadedSome:
30  loadDefaultComps(allcomps)
31 

◆ setupLoggingLevels()

def python.Utils.setupLoggingLevels (   flags,
  ca 
) 
Read the Exec.*MessageComponents flags and modify output level of components

    The specification of components  uses python fnmatch library and resembles UNIX paths.
    For instance an event algorithm MyAlgo/MyInstance would have following path:
    MasterSeq/AthAllAlgSeq/AthAlgSeq/MyAlgo/MyInstance
    A private tool MyTool of name ToolInstance used by this algo:
    MasterSeq/AthAllAlgSeq/AthAlgSeq/MyAlgo/MyInstance/MyTool/ToolInstance
    A public tool would have path:
    ToolSvc/MyTool/ToolInstance 
    and so on.
    The path specification can look like thi:
    */ToolInstance - all tools that have matching instance name
    */MyTool/* - all instances of the tool
    */MyAlgo/MyInstance - specific algorithm
    */MyAlg0/* - all instance of specific algorithm
    */AthAlgSeq/* - all algorithms (with deeper sequences structure e.g. in HLT sub-components of sequences can controlled this way)
    ToolSvc/My*/* - all public tools that instance name starts with My

    The modifications to the OutputLevel are applied form ERROR to VERBOSE.
    This is to support pattern when large group of components is set to less verbose logging mode
    and with more specific selection more verbosity is enabled. E.g. all algorithms can be set to WARNING, 
    the all having calo in the name to INFO, and CaloCellMaker to DEBUG.  

    Each setting can be either a string or a list of strings.

    If the component-path contains no '/' it is assumed to be a plain component-name. In this case, 
    the output-level is set using the property MessageSvc.setDebug (or equivalent). This works also
    for converters that do not inherit PropertyHolder.

Definition at line 46 of file Control/AthenaConfiguration/python/Utils.py.

46 def setupLoggingLevels(flags, ca):
47  """Read the Exec.*MessageComponents flags and modify output level of components
48 
49  The specification of components uses python fnmatch library and resembles UNIX paths.
50  For instance an event algorithm MyAlgo/MyInstance would have following path:
51  MasterSeq/AthAllAlgSeq/AthAlgSeq/MyAlgo/MyInstance
52  A private tool MyTool of name ToolInstance used by this algo:
53  MasterSeq/AthAllAlgSeq/AthAlgSeq/MyAlgo/MyInstance/MyTool/ToolInstance
54  A public tool would have path:
55  ToolSvc/MyTool/ToolInstance
56  and so on.
57  The path specification can look like thi:
58  */ToolInstance - all tools that have matching instance name
59  */MyTool/* - all instances of the tool
60  */MyAlgo/MyInstance - specific algorithm
61  */MyAlg0/* - all instance of specific algorithm
62  */AthAlgSeq/* - all algorithms (with deeper sequences structure e.g. in HLT sub-components of sequences can controlled this way)
63  ToolSvc/My*/* - all public tools that instance name starts with My
64 
65  The modifications to the OutputLevel are applied form ERROR to VERBOSE.
66  This is to support pattern when large group of components is set to less verbose logging mode
67  and with more specific selection more verbosity is enabled. E.g. all algorithms can be set to WARNING,
68  the all having calo in the name to INFO, and CaloCellMaker to DEBUG.
69 
70  Each setting can be either a string or a list of strings.
71 
72  If the component-path contains no '/' it is assumed to be a plain component-name. In this case,
73  the output-level is set using the property MessageSvc.setDebug (or equivalent). This works also
74  for converters that do not inherit PropertyHolder.
75  """
76 
77  msgSvc=ca.getService("MessageSvc") #by the time this snippet runs, the ComponentAccumulator should really contain the MessageSvc
78 
79 
80 
81  from AthenaCommon.Constants import ERROR,WARNING,INFO,DEBUG,VERBOSE
82  def __tolist(d):
83  return ([d] if d != '' else []) if isinstance(d, str) else d
84  for flag_val, level,setter in [(flags.Exec.ErrorMessageComponents, ERROR, msgSvc.setError),
85  (flags.Exec.WarningMessageComponents, WARNING, msgSvc.setWarning),
86  (flags.Exec.InfoMessageComponents, INFO, msgSvc.setInfo),
87  (flags.Exec.DebugMessageComponents, DEBUG, msgSvc.setDebug),
88  (flags.Exec.VerboseMessageComponents, VERBOSE, msgSvc.setVerbose) ]:
89  for c in __tolist(flag_val):
90  if "/" in c:
91  ca.foreach_component(c).OutputLevel=level
92  else: # a plain component name is given
93  setter.append(c)

Variable Documentation

◆ __all__

list python.Utils.__all__ = [ 'unixtools' ]
private

Definition at line 9 of file Control/AthenaCommon/python/Utils/__init__.py.

◆ __author__

string python.Utils.__author__ = 'Wim Lavrijsen (WLavrijsen@lbl.gov)'
private

Definition at line 7 of file Control/AthenaCommon/python/Utils/__init__.py.

◆ __version__

string python.Utils.__version__ = '1.0.0'
private

Definition at line 6 of file Control/AthenaCommon/python/Utils/__init__.py.

◆ campaign_runs

python.Utils.campaign_runs

Definition at line 19 of file Tools/Campaigns/python/Utils.py.

◆ log

python.Utils.log

Definition at line 8 of file Trigger/TriggerCommon/TrigEDMConfig/python/Utils.py.

◆ maxsize

python.Utils.maxsize

Definition at line 10 of file Trigger/TriggerCommon/TrigEDMConfig/python/Utils.py.

python.Utils.getHostName
def getHostName()
Definition: InnerDetector/InDetExample/InDetBeamSpotExample/python/Utils.py:60
CaloCellPos2Ntuple.int
int
Definition: CaloCellPos2Ntuple.py:24
python.Utils.getEDMVersionFromBS
def getEDMVersionFromBS(filename)
Definition: Trigger/TriggerCommon/TrigEDMConfig/python/Utils.py:11
python.AutoConfigFlags.GetFileMD
def GetFileMD(filenames, allowEmpty=True)
Definition: AutoConfigFlags.py:51
python.Utils.edmDictToList
def edmDictToList(edmDict)
Definition: Trigger/TriggerCommon/TrigEDMConfig/python/Utils.py:68
python.Utils.getRunFromName
def getRunFromName(name, default='', asInt=False)
Definition: InnerDetector/InDetExample/InDetBeamSpotExample/python/Utils.py:13
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
python.Utils.setupLoggingLevels
def setupLoggingLevels(flags, ca)
Definition: Control/AthenaConfiguration/python/Utils.py:46
python.Utils.getEDMListFromWriteHandles
def getEDMListFromWriteHandles(configurables)
Definition: Trigger/TriggerCommon/TrigEDMConfig/python/Utils.py:92
python.Utils.fileListSnippet
def fileListSnippet(files, dsName, taskName, jobDir=None)
Definition: InnerDetector/InDetExample/InDetBeamSpotExample/python/Utils.py:30
Constants
some useful constants -------------------------------------------------—
python.Utils.blankIfNone
def blankIfNone(s)
Definition: InnerDetector/InDetExample/InDetBeamSpotExample/python/Utils.py:41
python.Utils.exposeHandles
def exposeHandles(allcomps)
Definition: Control/AthenaConfiguration/python/Utils.py:32
python.Utils.edmListToDict
def edmListToDict(edmList)
Definition: Trigger/TriggerCommon/TrigEDMConfig/python/Utils.py:78
python.Bindings.keys
keys
Definition: Control/AthenaPython/python/Bindings.py:798
python.Utils.loadDefaultComps
def loadDefaultComps(allcomps)
Definition: Control/AthenaConfiguration/python/Utils.py:6
python.Utils.getMCCampaign
def getMCCampaign(files)
Definition: Tools/Campaigns/python/Utils.py:27
python.Utils.getUserName
def getUserName(default='UNKNOWN')
Definition: InnerDetector/InDetExample/InDetBeamSpotExample/python/Utils.py:48
Generated on Thu Nov 7 2024 22:24:25 for ATLAS Offline Software by   doxygen 1.8.18