python.MetaReader Namespace Reference

Functions
	read_metadata (filenames, file_type=None, mode='lite', promote=None, meta_key_filter=None, unique_tag_info_values=True, ignoreNonExistingLocalFiles=False)
	_check_project ()
	_get_pfn (filename)
	_read_guid (filename)
	_extract_fields (obj)
	_convert_value (value, aux=None)
	_get_attribute_val (iov_container, attr_name, attr_idx)
	_extract_fields_iov (iov_container, idx_range)
	_extract_iov_for_run (iov_container, run)
	_extract_iov_detailed (iov_container)
	_extract_fields_iovmdc (value)
	_extract_fields_esi (value)
	_extract_fields_ef (value)
	_extract_fields_cbk (interface=None, aux=None)
	_extract_fields_fmd (interface=None, aux=None)
	_extract_fields_tmd (interface=None, aux=None)
	_extract_fields_triggermenu (interface, aux)
	_extract_fields_triggermenujson (interface, aux)
	_convert_event_type_user_type (value)
	_convert_event_type_bitmask (value)
	make_lite (meta_dict)
	make_peeker (meta_dict)
	promote_keys (meta_dict, mode)
	convert_itemList (metadata, layout)
	dataheader_nentries (infile)
dict	get_meta_filter (mode="lite", meta_key_filter=None)
	denormalize_metadata_types (metadata_dict)
	should_keep_meta (normalizedName, typeName, meta_filter)

Variables
	msg = logging.getLogger('MetaReader')
	regexEventStreamInfo = re.compile(r'^EventStreamInfo(_p\d+)?$')
	regexIOVMetaDataContainer = re.compile(r'^IOVMetaDataContainer(_p\d+)?$')
	regexByteStreamMetadataContainer = re.compile(r'^ByteStreamMetadataContainer(_p\d+)?$')
	regexXAODCutBookkeeperContainer = re.compile(r'^xAOD::CutBookkeeperContainer(_v\d+)?$')
	regexXAODCutBookkeeperContainerAux = re.compile(r'^xAOD::CutBookkeeperAuxContainer(_v\d+)?$')
	regexXAODEventFormat = re.compile(r'^xAOD::EventFormat(_v\d+)?$')
	regexXAODFileMetaData = re.compile(r'^xAOD::FileMetaData(_v\d+)?$')
	regexXAODFileMetaDataAux = re.compile(r'^xAOD::FileMetaDataAuxInfo(_v\d+)?$')
	regexXAODFileMetaDataAuxDyn = re.compile(r'^(xAOD::)?FileMetaData.*AuxDyn(\.[a-zA-Z0-9]+)?$')
	regexXAODTriggerMenu = re.compile(r'^DataVector<xAOD::TriggerMenu(_v\d+)?>$')
	regexXAODTriggerMenuAux = re.compile(r'^xAOD::TriggerMenuAuxContainer(_v\d+)?$')
	regexXAODTriggerMenuJson = re.compile(r'^DataVector<xAOD::TriggerMenuJson(_v\d+)?>$')
	regexXAODTriggerMenuJsonAux = re.compile(r'^xAOD::TriggerMenuJsonAuxContainer(_v\d+)?$')
	regexXAODTruthMetaData = re.compile(r'^DataVector<xAOD::TruthMetaData(_v\d+)?>$')
	regexXAODTruthMetaDataAux = re.compile(r'^xAOD::TruthMetaDataAuxContainer(_v\d+)?$')
	regex_cppname = re.compile(r'^([\w:]+)(<.*>)?$')
	regex_persistent_class = re.compile(r'^([a-zA-Z]+(_[pv]\d+)?::)*[a-zA-Z]+_[pv]\d+$')
	regex_BS_files = re.compile(r'^(\w+):.*((\.D?RAW\..*)|(\.data$))')
	regex_URI_scheme = re.compile(r'^([A-Za-z0-9\+\.\-]+)\:')
list	lite_primary_keys_to_keep
list	lite_TagInfo_keys_to_keep
list	trigger_keys
dict	trigger_menu_json_map

Function Documentation

_check_project()

python.MetaReader._check_project ( )

protected

Definition at line 957 of file MetaReader.py.

def _check_project():
    import os
    if 'AthSimulation_DIR' in os.environ:
        return 'AthSimulation'
    if 'AthGeneration_DIR' in os.environ:
        return 'AthGeneration'
    return 'Athena'
 
 

_convert_event_type_bitmask()

python.MetaReader._convert_event_type_bitmask ( value )

protected

Definition at line 1494 of file MetaReader.py.

def _convert_event_type_bitmask(value):
 
    types = None
    for key in value:
        if key == 'bit_mask':
            val = value[key]
 
            bitmask_length = len(val)
 
            is_simulation = False
            is_testbeam = False
            is_calibration = False
 
            if bitmask_length > 0:  # ROOT.EventType.IS_SIMULATION
                is_simulation = val[0]
 
            if bitmask_length > 1:  # ROOT.EventType.IS_TESTBEAM
                is_testbeam = val[1]
 
            if bitmask_length > 2:  # ROOT.EventType.IS_CALIBRATION:
                is_calibration = val[2]
 
            types = [
                'IS_SIMULATION' if is_simulation else 'IS_DATA',
                'IS_TESTBEAM' if is_testbeam else 'IS_ATLAS',
                'IS_CALIBRATION' if is_calibration else 'IS_PHYSICS'
            ]
 
    value['type'] = types
    return value
 
 

_convert_event_type_user_type()

python.MetaReader._convert_event_type_user_type ( value )

protected

Definition at line 1487 of file MetaReader.py.

def _convert_event_type_user_type(value):
    if 'user_type' in value:
        items = value['user_type'].split('#')[3:]
        for i in range(0, len(items), 2):
            value[items[i]] = _convert_value(items[i+1])
    return value
 

_convert_value()

python.MetaReader._convert_value ( value, aux = None )

protected

Definition at line 1060 of file MetaReader.py.

def _convert_value(value, aux = None):
    cl=value.__class__
    if hasattr(cl, '__cpp_name__'):
        result = regex_cppname.match(cl.__cpp_name__)
        if result:
            cpp_type = result.group(1)
            if cpp_type == 'vector' or cpp_type == 'std::vector':
                return [_convert_value(val) for val in value]
            elif cpp_type == 'set' or cpp_type == 'std::set':
                return {_convert_value(val) for val in value}
            elif cpp_type == 'pair' or cpp_type == 'std::pair':
                return _convert_value(value.first), _convert_value(value.second)
            
            # elif cpp_type == 'long':
            #   return int(value)
            
            elif cpp_type == 'string' or cpp_type == 'std::string':
                return str(value)
            
            elif cl.__cpp_name__ == "_Bit_reference":
                return bool(value)
 
            # special case which extracts data in a better format from IOVPayloadContainer_p1 class
            elif cl.__cpp_name__ == 'IOVMetaDataContainer_p1':
                return _extract_fields_iovmdc(value)
 
            elif cl.__cpp_name__ == 'IOVPayloadContainer_p1':
                if _gbl_mode == 'iov':
                    return _extract_iov_detailed(value)
                elif _gbl_run is not None:
                    r =  _extract_iov_for_run(value, _gbl_run)
                    if r is not None:
                        return r
                    # fall through to the default extraction if no IOV found for the run
                return _extract_fields_iov( value, range(value.m_attrIndexes.size()) )
 
            elif cl.__cpp_name__ == 'xAOD::EventFormat_v1':
                return _extract_fields_ef(value)
            elif cl.__cpp_name__ == 'xAOD::CutBookkeeperContainer_v1':
                return _extract_fields_cbk(interface=value, aux=aux)
            elif cl.__cpp_name__ == 'xAOD::FileMetaData_v1':
                return _extract_fields_fmd(interface=value, aux=aux)
            elif cl.__cpp_name__ == 'DataVector<xAOD::TruthMetaData_v1>':
                return _extract_fields_tmd(interface=value, aux=aux)
 
            elif cl.__cpp_name__ == 'DataVector<xAOD::TriggerMenu_v1>' :
                return _extract_fields_triggermenu(interface=value, aux=aux)
 
            elif cl.__cpp_name__ == 'DataVector<xAOD::TriggerMenuJson_v1>' :
                return _extract_fields_triggermenujson(interface=value, aux=aux)
 
            elif (cl.__cpp_name__ == 'EventStreamInfo_p1' or
                  cl.__cpp_name__ == 'EventStreamInfo_p2' or
                  cl.__cpp_name__ == 'EventStreamInfo_p3'):
                return _extract_fields_esi(value)
 
            elif (cl.__cpp_name__ == 'EventType_p1' or
                  cl.__cpp_name__ == 'EventType_p3'):
                fields = _extract_fields(value)
                fields = _convert_event_type_bitmask(fields)
                fields = _convert_event_type_user_type(fields)
                return fields
 
            elif regex_persistent_class.match(cl.__cpp_name__):
                return _extract_fields(value)
 
    return value
 
 

_extract_fields()

python.MetaReader._extract_fields ( obj )

protected

Definition at line 1045 of file MetaReader.py.

def _extract_fields(obj):
    result = {}
 
    for meth in dir(obj):
        if not meth.startswith('_'):
            if meth.startswith('m_'):
 
                field_name = str(meth)[2:]
                field_value = getattr(obj, meth)
 
                result[field_name] = _convert_value(field_value)
 
    return result
 
 

_extract_fields_cbk()

python.MetaReader._extract_fields_cbk ( interface = None, aux = None )

protected

Extract CutBookkeeper content into dictionary

This function takes the CutBookkeeperContainer_v1 and CutBookkeeperAuxContainer_v1 objects.
It makes sure the the interface object uses the auxiliary object as store.
    Args:
        interface (CutBookkeeperContainer_v1):     the interface class
        aux       (CutBookkeeperAuxContainer_v1):  auxiliary container object
    Returns
        dict: with the cycle number and last stream

Definition at line 1305 of file MetaReader.py.

def _extract_fields_cbk(interface=None, aux=None):
    """Extract CutBookkeeper content into dictionary
 
    This function takes the CutBookkeeperContainer_v1 and CutBookkeeperAuxContainer_v1 objects.
    It makes sure the the interface object uses the auxiliary object as store.
        Args:
            interface (CutBookkeeperContainer_v1):     the interface class
            aux       (CutBookkeeperAuxContainer_v1):  auxiliary container object
        Returns
            dict: with the cycle number and last stream
    """
    if not interface or not aux:
        return {}
    interface.setStore(aux)
 
    max_cycle = -1
    input_stream = ''
 
    for cbk in interface:
        current_cycle = int(cbk.cycle())
        if current_cycle > max_cycle:
            max_cycle = current_cycle
            input_stream = str(cbk.inputStream())
 
    result = {
        'currentCutCycle': max_cycle,
        'currentCutInputStream': input_stream,
    }
    return result
 
 

_extract_fields_ef()

python.MetaReader._extract_fields_ef ( value )

protected

Definition at line 1296 of file MetaReader.py.

def _extract_fields_ef(value):
    result = {}
 
    for ef_element in value:
        result[ef_element.first] = ef_element.second.className()
 
    return result
 
 

_extract_fields_esi()

python.MetaReader._extract_fields_esi ( value )

protected

Definition at line 1270 of file MetaReader.py.

def _extract_fields_esi(value):
    result = {}
 
    result['eventTypes'] = []
    for eventType in value.m_eventTypes:
        result['eventTypes'].append(_convert_value(eventType))
 
    result['numberOfEvents'] = value.m_numberOfEvents
    result['runNumbers'] = list(value.m_runNumbers)
    result['lumiBlockNumbers'] = list(value.m_lumiBlockNumbers)
    result['processingTags'] = [str(v) for v in value.m_processingTags]
    result['itemList'] = []
 
    # Get the class name in the repository with CLID <clid>
    from CLIDComps.clidGenerator import clidGenerator
    cgen = clidGenerator("")
    for clid, sgkey in value.m_itemList:
        if isinstance(sgkey, bytes):
            sgkey = sgkey.decode()
        else:
            sgkey = str(sgkey)
        result['itemList'].append((cgen.getNameFromClid(clid), sgkey))
 
    return result
 
 

_extract_fields_fmd()

python.MetaReader._extract_fields_fmd ( interface = None, aux = None )

protected

Turn static FileMetaData content into dictionary

This function takes the FileMetaData_v1 and FileMetaDataAuxInfo_v1 objects.
It makes sure the the interface object uses the auxiliary object as store.
Next the two static variables of FileMetaDataAuxInfo_v1 are retrieved and
added to the dictionary that is returned.
    Args:
        interface (FileMetaData_v1):        the interface class
        aux       (FileMetaDataAuxInfo_v1): auxiliary container object
    Returns
        dict: with the production release and dataType

Definition at line 1336 of file MetaReader.py.

def _extract_fields_fmd(interface=None, aux=None):
    """Turn static FileMetaData content into dictionary
 
    This function takes the FileMetaData_v1 and FileMetaDataAuxInfo_v1 objects.
    It makes sure the the interface object uses the auxiliary object as store.
    Next the two static variables of FileMetaDataAuxInfo_v1 are retrieved and
    added to the dictionary that is returned.
        Args:
            interface (FileMetaData_v1):        the interface class
            aux       (FileMetaDataAuxInfo_v1): auxiliary container object
        Returns
            dict: with the production release and dataType
    """
    import ROOT
    if not interface or not aux:
        return {}
    interface.setStore(aux)
    metaContent = {
        "productionRelease": ROOT.std.string(),
        "dataType": ROOT.std.string(),
        "runNumbers": ROOT.std.vector('unsigned int')(),
        "lumiBlocks": ROOT.std.vector('unsigned int')(),
    }
    # Note: using this for dynamic attributes returns empty content
    for k, v in metaContent.items():
        try:
            interface.value(getattr(interface, k), v)
        except AttributeError:
            interface.value(k, v)
    # Now return python objects
    result = {k: str(v) for k, v in metaContent.items() if type(v) is ROOT.std.string}
    result.update({k: list(v) for k, v in metaContent.items() if type(v) is ROOT.std.vector('unsigned int')})
    return result
 
 

_extract_fields_iov()

python.MetaReader._extract_fields_iov ( iov_container, idx_range )

protected

Definition at line 1182 of file MetaReader.py.

def _extract_fields_iov( iov_container, idx_range ):
     result = {}
     for idx in idx_range:
         attr_idx = iov_container.m_attrIndexes[idx]
         name_idx = attr_idx.nameIndex()
         attr_name = iov_container.m_attrName[name_idx]
         attr_value = _get_attribute_val(iov_container, attr_name, attr_idx)
 
         if attr_name not in result:
             result[attr_name] = [attr_value]
         else:
             result[attr_name].append(attr_value)
 
     max_element_count = 0
     for content in result.values():
         if len(content) > max_element_count:
             max_element_count = len(content)
 
     if max_element_count <= 1:
         for name, content in result.items():
             if len(content) > 0:
                 result[name] = content[0]
             else:
                 result[name] = None
 
     return result
 
 

_extract_fields_iovmdc()

python.MetaReader._extract_fields_iovmdc ( value )

protected

Definition at line 1266 of file MetaReader.py.

def _extract_fields_iovmdc(value):
    return _convert_value(value.m_payload)
 
 

_extract_fields_tmd()

python.MetaReader._extract_fields_tmd ( interface = None, aux = None )

protected

Definition at line 1371 of file MetaReader.py.

def _extract_fields_tmd(interface=None, aux=None):
    import ROOT
    BadAuxVarException = ROOT.SG.ExcBadAuxVar
    """Extract TruthMetaData content into dictionary
 
    This function takes the TruthMetaDataContainer_v1 and TruthMetaDataAuxContainer_v1 objects.
    It makes sure the the interface object uses the auxiliary object as store.
        Args:
            interface (TruthMetaDataContainer_v1):     the interface class
            aux       (TruthMetaDataAuxContainer_v1):  auxiliary container object
        Returns
            dict
    """
    if not interface or not aux:
        return {}
    interface.setStore(aux)
 
    # return the first as we do not really expect more than one
    result = {}
    for tmd in interface:
        result['mcChannelNumber'] = tmd.mcChannelNumber()
 
        try:
            result['weightNames'] = [str(v) for v in tmd.weightNames()]
        except BadAuxVarException:
            result['weightNames'] = []
 
        try:
            result['lhefGenerator'] = str(tmd.lhefGenerator())
        except BadAuxVarException:
            result['lhefGenerator'] = ''
    
        try:
            result['generators'] = str(tmd.generators())
        except BadAuxVarException:
            result['generators'] = ''
    
        try:
            result['evgenProcess'] = str(tmd.evgenProcess())
        except BadAuxVarException:
            result['evgenProcess'] = ''
    
        try:
            result['evgenTune'] = str(tmd.evgenTune())
        except BadAuxVarException:
            result['evgenTune'] = ''
    
        try:
            result['hardPDF'] = str(tmd.hardPDF())
        except BadAuxVarException:
            result['hardPDF'] = ''
    
        try:
            result['softPDF'] = str(tmd.softPDF())
        except BadAuxVarException:
            result['softPDF'] = ''
 
    return result
 
 
""" Note: Deprecated. Legacy support for Run 2 AODs produced in release 21 or in release 22 prior to April 2021
"""

_extract_fields_triggermenu()

python.MetaReader._extract_fields_triggermenu ( interface, aux )

protected

Definition at line 1433 of file MetaReader.py.

def _extract_fields_triggermenu(interface, aux):
    if aux is None:
        return {}
 
    L1Items = []
    HLTChains = []
 
    try:
        interface.setStore( aux )
        if interface.size() > 0:
            # We make the assumption that the first stored SMK is
            # representative of all events in the input collection.
            firstMenu = interface.at(0)
            L1Items   = [ _convert_value(item) for item in firstMenu.itemNames() ]
            HLTChains = [ _convert_value(chain) for chain in firstMenu.chainNames() ]
    except Exception as err: # noqa: F841
        msg.warning('Problem reading xAOD::TriggerMenu:')
 
    result = {}
    result['L1Items'] = L1Items
    result['HLTChains'] = HLTChains
 
    return result
 

_extract_fields_triggermenujson()

python.MetaReader._extract_fields_triggermenujson ( interface, aux )

protected

Definition at line 1457 of file MetaReader.py.

def _extract_fields_triggermenujson(interface, aux):
    result = {}
 
    try:
        interface.setStore( aux )
        if interface.size() > 0:
            # We make the assumption that the first stored SMK is
            # representative of all events in the input collection.
            firstMenu = interface.at(0)
            import json
            decoded = json.loads(firstMenu.payload())
            result['RAWTriggerMenuJson'] = firstMenu.payload()
            result['name'] = firstMenu.name()
            result['dbkey'] = firstMenu.key()
            if decoded['filetype'] == 'hltmenu':
                result['HLTChains'] = [ _convert_value(chain) for chain in decoded['chains'] ] 
            elif decoded['filetype'] == 'l1menu':
                result['L1Items'] = [ _convert_value(item) for item in decoded['items'] ] 
            elif decoded['filetype'] in ['bunchgroupset', 'hltprescale', 'l1prescale', 'hltmonitoringsummary']:
                return result
 
            else:
                msg.warning('Got an xAOD::TriggerMenuJson called {0} but only expecting hltmenu or l1menu'.format(decoded['filetype']))
                return {}
                
    except Exception as err: # noqa: F841
        msg.warning('Problem reading xAOD::TriggerMenuJson')
 
    return result
 

_extract_iov_detailed()

python.MetaReader._extract_iov_detailed ( iov_container )

protected

Definition at line 1239 of file MetaReader.py.

def _extract_iov_detailed(iov_container):
    def iovtostr(t):
        # break iov time into high and low halves (run number usually in the higher half)
        return "({h}:{l})".format(h=t>>32, l=t&(2**32-1))
 
    def extract_list_collection(iov_container, listCollection ):
        result = {}
        ln = 0
        for list in listCollection.m_attrLists:
            ln = ln + 1
            lname = 'List {ln}: iov=[{s} ,{e}];  Channel#={ch}'.format(
                ln=ln, s=iovtostr(list.m_range.m_start),
                e=iovtostr(list.m_range.m_stop),
                ch=list.m_channelNumber )
            result[ lname ] = _extract_fields_iov( iov_container, range(list.m_firstIndex, list.m_lastIndex) )
        return result
 
    result = {}
    pn = 0
    for listCollection in iov_container.m_payloadVec:
        pn = pn + 1
        pname = 'IOV range {n}: [{s}, {e}]'.format(n=pn, s=iovtostr(listCollection.m_start),
                                                   e=iovtostr(listCollection.m_stop))
        result[ pname ] = extract_list_collection(iov_container, listCollection )
    return result
 
 

_extract_iov_for_run()

python.MetaReader._extract_iov_for_run ( iov_container, run )

protected

Definition at line 1210 of file MetaReader.py.

def _extract_iov_for_run(iov_container, run):
    def extract_payload(iov_container, payload):
        result = {}
        for attrList in payload.m_attrLists:
            r = _extract_fields_iov( iov_container, range(attrList.m_firstIndex, attrList.m_lastIndex) )
            for name, content in r.items():
                if name not in result:
                    result[name] = [content]
                elif content not in result[name]:
                    result[name].append(content)
        # turn single element lists into single values
        for name, content in result.items():
            if len(content) == 1:
                result[name] = content[0]
            elif len(content) == 0:
                # to be consistent with the old code
                result[name] = None
        return result
 
    for payload in iov_container.m_payloadVec:
        if run >= payload.m_start>>32 and run < payload.m_stop>>32:
            return extract_payload(iov_container, payload)
    # IOV for the requested run not found
    if len(iov_container.m_payloadVec) > 0:
        # print a warning only if there was actually some payload in the container
        msg.warning('No IOV found for run %d', run)
    return None
 
 

_get_attribute_val()

python.MetaReader._get_attribute_val ( iov_container, attr_name, attr_idx )

protected

Definition at line 1129 of file MetaReader.py.

def _get_attribute_val(iov_container, attr_name, attr_idx):
    type_idx = attr_idx.typeIndex()
    obj_idx = attr_idx.objIndex()
 
    attr_value = None
 
    if type_idx == 0:
        attr_value = bool(iov_container.m_bool[obj_idx])
    elif type_idx == 1:
        attr_value = int(iov_container.m_char[obj_idx])
    elif type_idx == 2:
        attr_value = int(iov_container.m_unsignedChar[obj_idx])
    elif type_idx == 3:
        attr_value = int(iov_container.m_short[obj_idx])
    elif type_idx == 4:
        attr_value = int(iov_container.m_unsignedShort[obj_idx])
    elif type_idx == 5:
        attr_value = int(iov_container.m_int[obj_idx])
    elif type_idx == 6:
        attr_value = int(iov_container.m_unsignedInt[obj_idx])
    elif type_idx == 7:
        attr_value = int(iov_container.m_long[obj_idx])
    elif type_idx == 8:
        attr_value = int(iov_container.m_unsignedLong[obj_idx])
    elif type_idx == 9:
        attr_value = int(iov_container.m_longLong[obj_idx])
    elif type_idx == 10:
        attr_value = int(iov_container.m_unsignedLongLong[obj_idx])
    elif type_idx == 11:
        attr_value = float(iov_container.m_float[obj_idx])
    elif type_idx == 12:
        attr_value = float(iov_container.m_double[obj_idx])
    elif type_idx == 13:
        # skipping this type because is file IOVPayloadContainer_p1.h (line 120) is commented and not considered
        pass
    elif type_idx == 14:
        attr_value = str(iov_container.m_string[obj_idx])
        # Cleaning class name from value
        if attr_value.startswith('IOVMetaDataContainer_p1_'):
            attr_value = attr_value.replace('IOVMetaDataContainer_p1_', '')
        if attr_value.startswith('_'):
            attr_value = attr_value.replace('_', '/')
        # Now it is clean
    elif type_idx == 15:
        attr_value = int(iov_container.m_date[obj_idx])
    elif type_idx == 16:
        attr_value = int(iov_container.m_timeStamp[obj_idx])
    else:
        raise ValueError('Unknown type id {0} for attribute {1}'.format(type_idx, attr_name))
 
    return attr_value
 
 

_get_pfn()

python.MetaReader._get_pfn ( filename )

protected

Extract the actual filename if LFN or PFN notation is used

Definition at line 966 of file MetaReader.py.

def _get_pfn(filename):
    """
    Extract the actual filename if LFN or PFN notation is used
    """
    pfx = filename[0:4]
    if pfx == 'PFN:':
        return filename[4:]
    if pfx == 'LFN:':
        import subprocess, os
        os.environ['POOL_OUTMSG_LEVEL'] = 'Error'
        output = subprocess.check_output(['FClistPFN','-l',filename[4:]],text=True).split('\n')
        if len(output) == 2:
            return output[0]
        msg.error( 'FClistPFN({0}) returned unexpected number of lines:'.format(filename) )
        msg.error( '\n'.join(output) )
    return filename
 
 

_read_guid()

python.MetaReader._read_guid ( filename )

protected

Extracts the "guid" (Globally Unique Identifier) in POOL files and Grid catalogs) value from a POOL file.
:param filename: the input file
:return: the guid value, None if unavailable

Definition at line 984 of file MetaReader.py.

def _read_guid(filename):
    """
    Extracts the "guid" (Globally Unique Identifier) in POOL files and Grid catalogs) value from a POOL file.
    :param filename: the input file
    :return: the guid value, None if unavailable
    """
    import ROOT
    root_file = ROOT.TFile.Open( _get_pfn(filename) )
    params = root_file.Get('##Params')
    try:
        from ROOT import RNTuple as rnt
    except ImportError:
        from ROOT.Experimental import RNTuple as rnt
    if not params:
        return
    if not isinstance(params, ROOT.TTree) and not isinstance(params, rnt) and not isinstance(params, ROOT.TDirectory):
        raise NotImplementedError(f"Cannot extract GUID from object {params!r} of type {type(params)!r}")
 
    regex = re.compile(r'\‍[NAME=(\w+)\‍]\‍[VALUE=(.*)\‍]', re.ASCII)
    fid = None
 
    if isinstance(params, ROOT.TTree):
        for entry in params:
            param = entry.GetLeaf('db_string').GetValueString()
            result = regex.match(param)
            if result and result.group(1) == 'FID' :
                # don't exit yet, it's the last FID entry that counts
                fid = result.group(2)
    elif isinstance(params, rnt):
        try:
            from ROOT import RNTupleReader
        except ImportError:
            from ROOT.Experimental import RNTupleReader
        reader = RNTupleReader.Open(params)
        try:
            entry = reader.CreateEntry()
        except AttributeError:
            entry = reader.GetModel().CreateEntry()
        for idx in range(reader.GetNEntries()):
            reader.LoadEntry(idx, entry)
            try:
                result = regex.match(str(entry['db_string']))
            except (AttributeError, TypeError) as err:
                # Early RNTuple implementation doesn't allow reading
                # strings on the python side, might be triggering it...
                msg.error(f"Cannot read FID from ##Params in RNTuple w/ ROOT error: {err}")
                return None
            if result and result.group(1) == 'FID' :
                # don't exit yet, it's the last FID entry that counts
                fid = result.group(2)
    elif isinstance(params, ROOT.TDirectory):
        for key in params.GetListOfKeys():
            param = params.Get(key.GetName())
            result = regex.match(str(param))
            if result and result.group(1) == 'FID' :
                # don't exit yet, it's the last FID entry that counts
                fid = result.group(2)
 
    return fid
 
 

convert_itemList()

python.MetaReader.convert_itemList	(		metadata,
			layout )

This function will rearrange the itemList values to match the format of 'eventdata_items', 'eventdata_itemsList'
or 'eventdata_itemsDic' generated with the legacy file peeker tool
:param metadata: a dictionary obtained using read_metadata method.
                 The mode for read_metadata must be 'peeker of 'full'
:param layout: the mode in which the data will be converted:
            * for 'eventdata_items' use: layout= None
            * for 'eventdata_itemsList' use: layout= '#join'
            * for 'eventdata_itemsDic' use: layout= 'dict'

Definition at line 1754 of file MetaReader.py.

def convert_itemList(metadata, layout):
    """
    This function will rearrange the itemList values to match the format of 'eventdata_items', 'eventdata_itemsList'
    or 'eventdata_itemsDic' generated with the legacy file peeker tool
    :param metadata: a dictionary obtained using read_metadata method.
                     The mode for read_metadata must be 'peeker of 'full'
    :param layout: the mode in which the data will be converted:
                * for 'eventdata_items' use: layout= None
                * for 'eventdata_itemsList' use: layout= '#join'
                * for 'eventdata_itemsDic' use: layout= 'dict'
    """
 
    # Find the itemsList:
    item_list = None
 
    if 'itemList' in metadata:
        item_list = metadata['itemList']
    else:
 
        current_key = None
 
        for key in metadata:
            if 'metadata_items' in metadata and key in metadata['metadata_items'] and metadata['metadata_items'][key] == 'EventStreamInfo_p3':
                current_key = key
                break
        if current_key is not None:
            item_list = metadata[current_key]['itemList']
 
    if item_list is not None:
 
        if layout is None:
            return item_list
 
        elif layout == '#join':
            return [k + '#' + v for k, v in item_list if k]
 
 
        elif layout == 'dict':
            from collections import defaultdict
            dic = defaultdict(list)
 
            for k, v in item_list:
                dic[k].append(v)
 
            return dict(dic)
 
 

dataheader_nentries()

python.MetaReader.dataheader_nentries

(

infile

)

Extract number of entries from DataHeader.

infile  ROOT TFile object or filename string
return  Number of entries as returned by DataHeader object in infile,
        None in absence of DataHeader object

Definition at line 1801 of file MetaReader.py.

def dataheader_nentries(infile):
    """Extract number of entries from DataHeader.
 
    infile  ROOT TFile object or filename string
    return  Number of entries as returned by DataHeader object in infile,
            None in absence of DataHeader object
    """
    import ROOT
    from PyUtils.PoolFile import PoolOpts
    if not isinstance(infile, ROOT.TFile):
        infile = ROOT.TFile.Open(infile)
 
    for name in {PoolOpts.TTreeNames.DataHeader, PoolOpts.RNTupleNames.DataHeader}:
        obj = infile.Get(name)
        msg.debug(f"dataheader_nentries: {name=}, {obj=}, {type(obj)=}")
        if not obj:
            continue
        if isinstance(obj, ROOT.TTree):
            return obj.GetEntriesFast()
        else:
            # check early to avoid scary ROOT read errors
            if ROOT.gROOT.GetVersionInt() < 63100:
                raise RuntimeError("ROOT ver. 6.31/01 or greater needed to read RNTuple files")
            if isRNTuple(obj):
                try:
                    return ROOT.Experimental.RNTupleReader.Open(obj).GetNEntries()
                except AttributeError:
                    return ROOT.RNTupleReader.Open(obj).GetNEntries()
            else:
                raise NotImplementedError(f"Keys of type {type(obj)!r} not supported")
 

denormalize_metadata_types()

python.MetaReader.denormalize_metadata_types

(

metadata_dict

)

Convert canonical/C++ STL types in the metadata_items dictionary back to their
ROOT equivalents for backward compatibility.
- 'float' => 'Float_t'
- 'char' => 'Char_t'
- 'std::string' => 'string'
- 'xAOD::FileMetaData_v1' => 'FileMetaData'
- 'xAOD::FileMetaDataAuxInfo_v1' => 'FileMetaDataAux'
(add more as needed)

Definition at line 1907 of file MetaReader.py.

def denormalize_metadata_types(metadata_dict):
    """
    Convert canonical/C++ STL types in the metadata_items dictionary back to their
    ROOT equivalents for backward compatibility.
    - 'float' => 'Float_t'
    - 'char' => 'Char_t'
    - 'std::string' => 'string'
    - 'xAOD::FileMetaData_v1' => 'FileMetaData'
    - 'xAOD::FileMetaDataAuxInfo_v1' => 'FileMetaDataAux'
    (add more as needed)
    """
    type_map = {
        "float": "Float_t",
        "char": "Char_t",
        "std::string": "string",
        "std::uint32_t": "UInt_t",
        "xAOD::FileMetaData_v1": "FileMetaData",
    }
    denormalized = {}
    for k, v in metadata_dict.items():
        new_v = v
        for old, new in type_map.items():
            if new_v == old:
                new_v = new
            elif new_v.endswith("." + old):
                new_v = new_v.rsplit(".", 1)[0] + "." + new
        denormalized[k] = new_v
    return denormalized
 
 

get_meta_filter()

dict python.MetaReader.get_meta_filter	(		mode = "lite",
			meta_key_filter = None )

Return a dictionary of metadata filters based on the mode and
optional meta_key_filter.

Definition at line 1832 of file MetaReader.py.

def get_meta_filter(mode="lite", meta_key_filter=None) -> dict:
    """Return a dictionary of metadata filters based on the mode and
    optional meta_key_filter.
    """
 
    if meta_key_filter is None:
        meta_key_filter = []
 
    # create a container for the list of filters used for the lite version
    meta_filter = {}
 
    # set the filters for name
    if mode == 'lite':
        if isGaudiEnv():
            meta_filter = {
                '/TagInfo': 'IOVMetaDataContainer_p1',
                'IOVMetaDataContainer_p1__TagInfo': 'IOVMetaDataContainer_p1',
                '*': 'EventStreamInfo_p*'
            }
        else:
            meta_filter = {
                'FileMetaData': '*',
                'FileMetaDataAux.': 'xAOD::FileMetaDataAuxInfo_v1',
            }
 
    # set the filters for name
    if mode == 'peeker':
        meta_filter.update({
            'TriggerMenu': 'DataVector<xAOD::TriggerMenu_v1>', # R2 trigger metadata format AOD (deprecated)
            'TriggerMenuAux.': 'xAOD::TriggerMenuAuxContainer_v1',
            'DataVector<xAOD::TriggerMenu_v1>_TriggerMenu': 'DataVector<xAOD::TriggerMenu_v1>', # R2 trigger metadata format ESD (deprecated)
            'xAOD::TriggerMenuAuxContainer_v1_TriggerMenuAux.': 'xAOD::TriggerMenuAuxContainer_v1',
            'TriggerMenuJson_HLT': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format AOD
            'TriggerMenuJson_HLTAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'TriggerMenuJson_HLTMonitoring': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format AOD
            'TriggerMenuJson_HLTMonitoringAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'TriggerMenuJson_HLTPS': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format AOD
            'TriggerMenuJson_HLTPSAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'TriggerMenuJson_L1': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format AOD
            'TriggerMenuJson_L1Aux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'TriggerMenuJson_L1PS': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format AOD
            'TriggerMenuJson_L1PSAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'CutBookkeepers': 'xAOD::CutBookkeeperContainer_v1',
            'CutBookkeepersAux.': 'xAOD::CutBookkeeperAuxContainer_v1',
            'FileMetaData': '*',
            'FileMetaDataAux.': 'xAOD::FileMetaDataAuxInfo_v1',
            'TruthMetaData': '*',
            'TruthMetaDataAux.': 'xAOD::TruthMetaDataAuxContainer_v1',
            'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLT': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format ESD
            'xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTMonitoring': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format ESD
            'xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTMonitoringAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTPS': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format ESD
            'xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTPSAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format ESD
            'xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_L1Aux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1PS': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format ESD
            'xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_L1PSAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1'
        })
 
        if isGaudiEnv():
            meta_filter.update({
                '/TagInfo': 'IOVMetaDataContainer_p1',
                'IOVMetaDataContainer_p1__TagInfo': 'IOVMetaDataContainer_p1',
                '/Simulation/Parameters': 'IOVMetaDataContainer_p1',
                '/Digitization/Parameters': 'IOVMetaDataContainer_p1',
                '/EXT/DCS/MAGNETS/SENSORDATA': 'IOVMetaDataContainer_p1',
                '*': 'EventStreamInfo_p*'
            })
 
    if (mode == 'full' or mode == 'iov') and meta_key_filter:
        meta_filter = {f: '*' for f in meta_key_filter}
 
    return meta_filter
 

make_lite()

python.MetaReader.make_lite

(

meta_dict

)

Definition at line 1526 of file MetaReader.py.

def make_lite(meta_dict):
    for filename, file_content in meta_dict.items():
        for key in file_content:
            if key in meta_dict[filename]['metadata_items'] and regexEventStreamInfo.match(meta_dict[filename]['metadata_items'][key]):
                for item in list(meta_dict[filename][key]):
                    if item not in lite_primary_keys_to_keep:
                        meta_dict[filename][key].pop(item)
 
        if '/TagInfo' in file_content:
            
 
            for item in list(meta_dict[filename]['/TagInfo']):
                if item not in lite_TagInfo_keys_to_keep:
                    meta_dict[filename]['/TagInfo'].pop(item)
    return meta_dict
 
 

make_peeker()

python.MetaReader.make_peeker

(

meta_dict

)

Definition at line 1543 of file MetaReader.py.

def make_peeker(meta_dict):
    for filename, file_content in meta_dict.items():
        for key in file_content:
            if key in meta_dict[filename]['metadata_items'] and regexEventStreamInfo.match(meta_dict[filename]['metadata_items'][key]):
                keys_to_keep = [
                    'lumiBlockNumbers',
                    'runNumbers',
                    'mc_event_number',
                    'mc_channel_number',
                    'eventTypes',
                    'processingTags',
                    'itemList'
                ]
                for item in list(meta_dict[filename][key]):
                    if item not in keys_to_keep:
                        meta_dict[filename][key].pop(item)
 
        if '/TagInfo' in file_content:
            keys_to_keep = [
                'beam_energy',
                'beam_type',
                'GeoAtlas',
                'IOVDbGlobalTag',
                'AODFixVersion',
                'AMITag',
                'project_name',
                'triggerStreamOfFile',
                'AtlasRelease',
                'specialConfiguration',
                'mc_campaign',
                'hepmc_version',
                'generators',
                'keywords',
                'data_year',
            ]
            for item in list(meta_dict[filename]['/TagInfo']):
                if item not in keys_to_keep:
                    meta_dict[filename]['/TagInfo'].pop(item)
 
        if '/Simulation/Parameters' in file_content:
            keys_to_keep = [
                'G4Version',
                'TruthStrategy',
                'SimBarcodeOffset',
                'RegenerationIncrement',
                'TRTRangeCut',
                'SimulationFlavour',
                'Simulator',
                'PhysicsList',
                'SimulatedDetectors',
                'IsDataOverlay',
            ]
            for item in list(meta_dict[filename]['/Simulation/Parameters']):
                if item not in keys_to_keep:
                    meta_dict[filename]['/Simulation/Parameters'].pop(item)
 
        if '/Digitization/Parameters' in file_content:
            keys_to_keep = [
                'numberOfCollisions',
                'intraTrainBunchSpacing',
                'BeamIntensityPattern'
                'physicsList',
                'digiSteeringConf',
                'pileUp',
                'DigitizedDetectors',
            ]
            for item in list(meta_dict[filename]['/Digitization/Parameters']):
                if item not in keys_to_keep:
                    meta_dict[filename]['/Digitization/Parameters'].pop(item)
 
        if 'CutBookkeepers' in file_content:
            keys_to_keep = [
                'currentCutCycle',
                'currentCutInputStream',
            ]
            for item in list(meta_dict[filename]['CutBookkeepers']):
                if item not in keys_to_keep:
                    meta_dict[filename]['CutBookkeepers'].pop(item)
 
        if 'TruthMetaData' in file_content:
            keys_to_keep = [
                'mcChannelNumber',
                'weightNames',
            ]
            for item in list(meta_dict[filename]['TruthMetaData']):
                if item not in keys_to_keep:
                    meta_dict[filename]['TruthMetaData'].pop(item)
 
    return meta_dict
 
 

promote_keys()

python.MetaReader.promote_keys	(		meta_dict,
			mode )

Definition at line 1634 of file MetaReader.py.

def promote_keys(meta_dict, mode):
    for filename, file_content in meta_dict.items():
        md = meta_dict[filename]
        for key in file_content:
            if key in md['metadata_items'] and regexEventStreamInfo.match(md['metadata_items'][key]):
                md.update(md[key])
 
                if 'eventTypes' in md and len(md['eventTypes']):
                    et = md['eventTypes'][0]
                    md['mc_event_number'] = et.get('mc_event_number', md['runNumbers'][0])
                    if 'mc_channel_number' in et:
                        md['mc_channel_number'] = et.get('mc_channel_number', None)
                    md['eventTypes'] = et['type']
 
                    # For very old files
                    if 'GeoAtlas' in et:
                        md['GeoAtlas']       = et.get('GeoAtlas', None)
                    if 'IOVDbGlobalTag' in et:
                        md['IOVDbGlobalTag'] = et.get('IOVDbGlobalTag', None)
 
                if 'lumiBlockNumbers' in md[key]:
                    md['lumiBlockNumbers'] = md[key]['lumiBlockNumbers']
                
                if 'processingTags' in md[key]:
                    md['processingTags'] = md[key]['processingTags']
 
                meta_dict[filename].pop(key)
                break
 
            if not isGaudiEnv() and key in md['metadata_items'] and 'FileMetaData' in key:
                if 'beamType' in md[key]:
                    md['beam_type'] = md[key]['beamType']
 
                if 'runNumbers' in md[key]:
                    md['runNumbers'] = md[key]['runNumbers']
 
                if 'mcProcID' in md[key]:
                    md['mc_channel_number'] = int(md[key]['mcProcID'])
 
                if 'mcCampaign' in md[key]:
                    md['mc_campaign'] = md[key]['mcCampaign']
 
                if 'dataYear' in md[key]:
                    md['data_year'] = int(md[key]['dataYear'])
 
                if 'lumiBlocks' in md[key]:
                    md['lumiBlockNumbers'] = md[key]['lumiBlocks']
                
                if mode == 'peeker' and 'amiTag' in md[key]:
                    md['AMITag'] = md[key]['amiTag']
 
                if 'beamEnergy' in md[key]:
                    md['beam_energy'] = int(md[key]['beamEnergy'])
 
                if 'geometryVersion' in md[key]:
                    md['GeoAtlas'] = md[key]['geometryVersion']
 
                # EventType checks
                md['eventTypes'] = []
                if mode == 'peeker' and 'simFlavour' in md[key]:
                    md['SimulationFlavour'] = md[key]['simFlavour']
 
                if mode == 'peeker' and 'isDataOverlay' in md[key]:
                    md['IsDataOverlay'] = md[key]['isDataOverlay']
 
                if 'dataType' in md[key]:
                    md['processingTags'] = [md[key]['dataType']]
 
                if (
                    ('simFlavour' in md[key] and ('FullG4' in md[key]['simFlavour'] or 'ATLFAST' in md[key]['simFlavour']))
                    or 'DAOD_TRUTH' in md[key]['dataType']
                ):
                    md['eventTypes'].append('IS_SIMULATION')
                else:
                    md['eventTypes'].append('IS_DATA')
 
                if (
                    'GeoAtlas' in md and 'ATLAS' in md['GeoAtlas']
                    or 'DAOD_TRUTH' in md[key]['dataType']
                ):
                    md['eventTypes'].append('IS_ATLAS')
                    # this is probably safe to assume for all files used in AnalysisBase
                    md['eventTypes'].append('IS_PHYSICS')
                else:
                    md['eventTypes'].append('IS_TESTBEAM')
 
                if mode == 'peeker':
                    if 'productionRelease' in md[key]:
                        md['AtlasRelease'] = md[key]['productionRelease']
                    
                    if 'generatorsInfo' in md[key]:
                        md['generators']  = md[key]['generatorsInfo']
 
                if mode == 'lite':
                    meta_dict[filename].pop(key)
                break
 
        if '/TagInfo' in file_content:
            md.update(md['/TagInfo'])
            md.pop('/TagInfo')
 
        if '/Generation/Parameters' in file_content:
            md.update(md['/Generation/Parameters'])
            md.pop('/Generation/Parameters')
 
        if '/Simulation/Parameters' in file_content:
            md.update(md['/Simulation/Parameters'])
            md.pop('/Simulation/Parameters')
 
        if '/Digitization/Parameters' in file_content:
            md.update(md['/Digitization/Parameters'])
            md.pop('/Digitization/Parameters')
 
        if 'CutBookkeepers' in file_content:
            md.update(md['CutBookkeepers'])
            md.pop('CutBookkeepers')
 
    return meta_dict
 
 

read_metadata()

python.MetaReader.read_metadata	(		filenames,
			file_type = None,
			mode = 'lite',
			promote = None,
			meta_key_filter = None,
			unique_tag_info_values = True,
			ignoreNonExistingLocalFiles = False )

This tool is independent of Athena framework and returns the metadata from a given file.
:param filenames: the input file from which metadata needs to be extracted.
:param file_type: the type of file. POOL or BS (bytestream: RAW, DRAW) files.
:param mode: if true, will return all metadata associated with the filename. By default, is false and this will
return a "tiny" version which have only the following keys: 'file_guid', 'file_size', 'file_type', 'nentries'.
:return: a dictionary of metadata for the given input file.

Definition at line 70 of file MetaReader.py.

                  unique_tag_info_values = True, ignoreNonExistingLocalFiles=False):
    """
    This tool is independent of Athena framework and returns the metadata from a given file.
    :param filenames: the input file from which metadata needs to be extracted.
    :param file_type: the type of file. POOL or BS (bytestream: RAW, DRAW) files.
    :param mode: if true, will return all metadata associated with the filename. By default, is false and this will
    return a "tiny" version which have only the following keys: 'file_guid', 'file_size', 'file_type', 'nentries'.
    :return: a dictionary of metadata for the given input file.
    """
 
    # make the mode and run available in the _convert methods
    global _gbl_mode, _gbl_run
    _gbl_mode = mode
    _gbl_run = None
 
    from RootUtils import PyROOTFixes  # noqa F401
 
    # Check if the input is a file or a list of files.
    if isinstance(filenames, str):
        filenames = [filenames]
 
    # Check if file_type is an allowed value
    if file_type is not None:
        if file_type not in ('POOL', 'BS'):
            raise NameError('Allowed values for \'file_type\' parameter are: "POOL" or "BS": you provided "' + file_type + '"')
        else:
            msg.info('Forced file_type: {0}'.format(file_type))
 
    # Check the value of mode parameter
    if mode not in ('tiny', 'lite', 'full', 'peeker', 'iov'):
        raise NameError('Allowed values for "mode" parameter are: "tiny", "lite", "peeker", "iov" or "full"')
 
    if meta_key_filter is None:
        meta_key_filter = []
 
    # Disable 'full' and 'iov' in non-Gaudi environments
    if not isGaudiEnv():
        if mode in ('full', 'iov'):
            raise NameError('The following modes are not available in AnalysisBase: "iov" and "full"')
 
    msg.info('Current mode used: {0}'.format(mode))
    msg.info('Current filenames: {0}'.format(filenames))
 
    if mode != 'full' and mode !='iov' and len(meta_key_filter) > 0:
        raise NameError('It is possible to use the meta_key_filter option only for full mode')
    if meta_key_filter:
        msg.info('Filter used: {0}'.format(meta_key_filter))
 
    # create the storage object for metadata.
    meta_dict = {}
 
    # ----- retrieve metadata from all filename or filenames --------------------------------------------------------#
    for filename in filenames:
        meta_dict[filename] = {}
        current_file_type = None
        # reset the global Run for each file to not propagate and old value
        _gbl_run = None
        # Determine the file_type of the input and store this information into meta_dict
        if not file_type:
            if os.path.isfile(filename):
                
                if ignoreNonExistingLocalFiles and not regex_URI_scheme.match(filename) and gSystem.AccessPathName(filename): # Attention, bizarre convention of return value!! 
                    msg.warning('Ignoring not accessible file: {}'.format(filename))
                    continue
                    
                with open(filename, 'rb') as binary_file:
                    magic_file = binary_file.read(4)
 
                    if magic_file == 'root' or magic_file == b'root':
                        current_file_type = 'POOL'
                        meta_dict[filename]['file_type'] = 'POOL'
 
                    elif Project.determine() in (
                            Project.AnalysisBase, Project.AthAnalysis):
                        raise RuntimeError(
                            f"{filename} is not a ROOT file, assumed bytestream"
                            ", this is not supported in Analysis releases")
                    else:
                        current_file_type = 'BS'
                        meta_dict[filename]['file_type'] = 'BS'
 
                    # add information about the file_size of the input filename
                    meta_dict[filename]['file_size'] = os.path.getsize(filename)
 
            # determine the file type for the remote input files
            else:
                if regex_BS_files.match(filename):
                    current_file_type = 'BS'
                    meta_dict[filename]['file_type'] = 'BS'
                else:
                    current_file_type = 'POOL'
                    meta_dict[filename]['file_type'] = 'POOL'
 
                # add information about the file_size of the input filename
                meta_dict[filename]['file_size'] = None  # None -> we can't read the file size for a remote file
 
        else:
            current_file_type = file_type
 
        # ----- retrieves metadata from POOL files ------------------------------------------------------------------#
        if current_file_type == 'POOL':
            
            if ignoreNonExistingLocalFiles and not regex_URI_scheme.match(filename) and gSystem.AccessPathName(filename): # Attention, bizarre convention of return value!! 
                msg.warning('Ignoring not accessible file: {}'.format(filename))
                continue
                    
            import ROOT
            # open the file using ROOT.TFile
            current_file = ROOT.TFile.Open( _get_pfn(filename) )
 
            # get auto flush setting from the main EventData TTree
            from PyUtils.PoolFile import PoolOpts
            collectionTree = current_file.Get(PoolOpts.TTreeNames.EventData)
            if isinstance(collectionTree, ROOT.TTree):
                meta_dict[filename]['auto_flush'] = collectionTree.GetAutoFlush()
 
            # read and add the 'GUID' value
            meta_dict[filename]['file_guid'] = _read_guid(filename)
 
            # read and add compression level and algorithm
            meta_dict[filename]['file_comp_alg'] = current_file.GetCompressionAlgorithm()
            meta_dict[filename]['file_comp_level'] = current_file.GetCompressionLevel()
 
            if (
                isRNTuple(md:=current_file.Get(PoolOpts.RNTupleNames.MetaData))
                and mode != "tiny"
            ):
                msg.warning(
                    "Reading in-file metadata from RNTuple is currently of limited support"
                )
                meta_dict[filename]["metadata_items"] = {}
 
                try:
                    from ROOT import RNTupleReader
                except ImportError:
                    from ROOT.Experimental import RNTupleReader
 
                reader = RNTupleReader.Open(md)
                entry = reader.CreateEntry()
                reader.LoadEntry(0, entry)
                auxes = {}
                classes_with_aux = {
                    "xAOD::FileMetaData_v1",
                    "xAOD::FileMetaDataAuxInfo_v1",
                    "xAOD::TriggerMenuJsonAuxContainer_v1",
                    "DataVector<xAOD::TriggerMenuJson_v1>",
                    "xAOD::TruthMetaDataAuxContainer_v1",
                    "DataVector<xAOD::TruthMetaData_v1>",
                    "xAOD::CutBookkeeperContainer_v1",
                    "xAOD::CutBookkeeperAuxContainer_v1",
                    "xAOD::LumiBlockRangeAuxContainer_v1",
                    "DataVector<xAOD::LumiBlockRange_v1>",
                }
 
                dynamic_fmd_items = {}
 
                meta_filter = get_meta_filter(mode, meta_key_filter)
 
                for field in reader.GetDescriptor().GetTopLevelFields():
                    normalizedName = field.GetFieldName()
                    if "index_ref" in normalizedName:
                        continue
                    if regexIOVMetaDataContainer.match(field.GetTypeName()):
                        # if field name is e.g. IOVMetaDataContainer_p1__Digitization_Parameters,
                        # strip the prefix and change underscore to slash to slash
                        normalizedName = (
                            field.GetFieldName()
                            .replace("IOVMetaDataContainer_p1_", "")
                            .replace("_", "/")
                        )
                        meta_dict[filename]["metadata_items"][normalizedName] = (
                            "IOVMetaDataContainer"
                        )
                    elif regexByteStreamMetadataContainer.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][field.GetFieldName()] = (
                            "ByteStreamMetadataContainer"
                        )
                    elif regexEventStreamInfo.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][field.GetFieldName()] = (
                            "EventStreamInfo"
                        )
                    elif regexXAODFileMetaData.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName().replace("xAOD__", "xAOD::")
                        ] = field.GetTypeName()
                    elif regexXAODFileMetaDataAuxDyn.match(
                        normalizedName := field.GetFieldName()
                        .replace("xAOD__", "xAOD::")
                        .replace("AuxDyn:", "AuxDyn.")
                    ):
                        result = (
                            False
                            if entry[field.GetFieldName()] == "\x00"
                            else entry[field.GetFieldName()]
                        )
                        dynamic_fmd_items[normalizedName.split(".")[1]] = result
                        meta_dict[filename]["metadata_items"][normalizedName] = (
                            field.GetTypeName()
                        )
                        continue
                    elif regexXAODFileMetaDataAux.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("Aux:", "Aux.")
                        ] = field.GetTypeName()
                    elif regexXAODTruthMetaData.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("DataVector_", "DataVector<")
                            .replace("__Truth", ">_Truth")
                        ] = "TruthMetaData"
                    elif regexXAODTruthMetaDataAux.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("Aux:", "Aux.")
                        ] = field.GetTypeName()
                    elif regexXAODEventFormat.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName().replace("xAOD__", "xAOD::")
                        ] = field.GetTypeName()
                    elif regexXAODTriggerMenuJson.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("DataVector_", "DataVector<")
                            .replace("__Trigger", ">_Trigger")
                        ] = field.GetTypeName()
                    elif regexXAODTriggerMenuJsonAux.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("Aux:", "Aux.")
                        ] = field.GetTypeName()
                    elif regexXAODCutBookkeeperContainer.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("DataVector_", "DataVector<")
                            .replace("__CutBookkeeper", ">_CutBookkeeper")
                        ] = field.GetTypeName()
                    elif regexXAODCutBookkeeperContainerAux.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("Aux:", "Aux.")
                        ] = field.GetTypeName()
                    else:
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName().replace("Aux:", "Aux.")
                        ] = field.GetTypeName()
 
                    if field.GetTypeName() in classes_with_aux:
                        # handle aux classes later
                        auxes[field.GetFieldName()] = field.GetTypeName()
                        continue
 
                    if not should_keep_meta(
                        normalizedName, field.GetTypeName(), meta_filter
                    ):
                        continue
 
                    try:
                        meta_dict[filename][normalizedName] = _convert_value(
                            entry[field.GetFieldName()]
                        )
                    except KeyError:
                        msg.warning(f"missing type {field.GetTypeName()}")
 
                meta_dict[filename]["metadata_items"] = denormalize_metadata_types(
                    meta_dict[filename]["metadata_items"]
                )
 
                def _get_aux_base(aux_key: str) -> str:
                    # Remove known prefixes
                    key = aux_key
                    key = key.replace("xAOD__TriggerMenuJsonAuxContainer_v1_", "")
                    key = key.replace("xAOD__FileMetaDataAuxInfo_v1_", "")
                    key = key.replace("xAOD__TruthMetaDataAuxContainer_v1_", "")
                    # Remove known suffixes
                    if key.endswith("Aux:"):
                        key = key[:-4]
                    elif key.endswith("Aux"):
                        key = key[:-3]
                    # Remove any trailing ':' or '_'
                    key = key.strip("_:")
                    return key
 
                def _get_main_base(main_key: str) -> str:
                    main_base = main_key
                    # For DataVectors
                    if main_key.startswith("DataVector_xAOD__TriggerMenuJson_v1__"):
                        main_base = main_key.replace(
                            "DataVector_xAOD__TriggerMenuJson_v1__", ""
                        )
                    # For FileMetaData
                    elif main_key.startswith("xAOD__FileMetaData_v1_"):
                        main_base = main_key.replace("xAOD__FileMetaData_v1_", "")
                    # For TruthMetaData
                    elif main_key.startswith("DataVector_xAOD__TruthMetaData_v1__"):
                        main_base = main_key.replace(
                            "DataVector_xAOD__TruthMetaData_v1__", ""
                        )
                    return main_base
 
                def _find_associated_pairs(auxes: dict) -> list[tuple[str, str]]:
                    # Build lookup tables
                    aux_map = {}
                    for k in auxes:
                        if "Aux" in k:
                            aux_map[_get_aux_base(k)] = k
 
                    main_map = {}
                    for k in auxes:
                        base = _get_main_base(k)
                        if base:
                            main_map[base] = k
 
                    # Find pairs
                    pairs = []
                    for base, aux_key in aux_map.items():
                        if base in main_map:
                            pairs.append((aux_key, main_map[base]))
                    return pairs
 
                for pair in _find_associated_pairs(auxes):
                    return_obj = _convert_value(
                        entry[pair[1]],
                        entry[pair[0]],
                    )
                    key = next(
                        (
                            k
                            for k, v in trigger_menu_json_map.items()
                            if v
                            == pair[1]
                            .replace("xAOD__", "xAOD::")
                            .replace("DataVector_", "DataVector<")
                            .replace("__Trigger", ">_Trigger")
                        ),
                        auxes[pair[0]],
                    )
 
                    try:
                        key = (
                            key.replace("xAOD__", "xAOD::")
                            if key.count("_") <= 1
                            else key.replace("xAOD__", "xAOD::").rsplit("_", 2)[0]
                        )
                    except IndexError:
                        pass
 
                    if not should_keep_meta(
                        pair[0]
                        .replace("xAOD__", "xAOD::")
                        .replace("DataVector_", "DataVector<")
                        .replace("__Trigger", ">_Trigger")
                        .replace("Aux:", "Aux."),
                        key,
                        meta_filter,
                    ):
                        continue
 
                    if "TriggerMenuJson" in pair[0]:
                        if "RAWTriggerMenuJson" in return_obj:
                            key = (
                                pair[1]
                                if pair[0].startswith("Trigger")
                                else trigger_menu_json_map[pair[0]]
                            )
                            meta_dict[filename][key] = return_obj["RAWTriggerMenuJson"]
                            del return_obj["RAWTriggerMenuJson"]
                        if "TriggerConfigInfo" not in meta_dict[filename]:
                            meta_dict[filename]["TriggerConfigInfo"] = {}
                        if "dbkey" in return_obj:
                            meta_dict[filename]["TriggerConfigInfo"][
                                pair[0].split("_")[-1].replace("Aux:", "")
                            ] = {"key": return_obj["dbkey"], "name": return_obj["name"]}
                            del return_obj["dbkey"]
                            del return_obj["name"]
                        if "TriggerMenu" not in meta_dict[filename]:
                            meta_dict[filename]["TriggerMenu"] = {}
                        meta_dict[filename]["TriggerMenu"].update(return_obj)
                    elif "FileMetaData" in pair[0]:
                        if "FileMetaData" not in meta_dict[filename]:
                            meta_dict[filename]["FileMetaData"] = dynamic_fmd_items
                        meta_dict[filename]["FileMetaData"].update(return_obj)
                    elif "TruthMetaData" in pair[0]:
                        if pair == ("TruthMetaDataAux:", "TruthMetaData"):
                            if "TruthMetaData" not in meta_dict[filename]:
                                meta_dict[filename]["TruthMetaData"] = {}
                            meta_dict[filename]["TruthMetaData"].update(return_obj)
                        else:
                            # for backward compatibility
                            meta_dict[filename][
                                pair[1]
                                .replace("xAOD__", "xAOD::")
                                .replace("DataVector_", "DataVector<")
                                .replace("__Truth", ">_Truth")
                            ] = {}
                            meta_dict[filename][
                                pair[0]
                                .replace("xAOD__", "xAOD::")
                                .replace("Aux:", "Aux.")
                            ] = {}
                    elif pair == ("CutBookkeepersAux:", "CutBookkeepers"):
                        meta_dict[filename]["CutBookkeepers"] = return_obj
 
                msg.debug(f"Read metadata from RNTuple: {meta_dict[filename]}")
 
            else:
                # ----- read extra metadata required for 'lite' and 'full' modes ----------------------------------------#
                if mode != 'tiny':
                    # selecting from all tree the only one which contains metadata, respectively "MetaData"
                    metadata_tree = current_file.Get('MetaData')
                    # read all list of branches stored in "MetaData" tree
                    metadata_branches = metadata_tree.GetListOfBranches()
                    nr_of_branches = metadata_branches.GetEntriesFast()
 
                    # object to store the names of metadata containers and their corresponding class name.
                    meta_dict[filename]['metadata_items'] = {}
 
                    meta_filter = get_meta_filter(mode, meta_key_filter)
 
                    # store all persistent classes for metadata container existing in a POOL/ROOT file.
                    persistent_instances = {}
                    dynamic_fmd_items = {}
 
                    # Protect non-Gaudi environments from meta-data classes it doesn't know about
                    if not isGaudiEnv():
                        metadata_tree.SetBranchStatus("*", False)
 
                    for i in range(0, nr_of_branches):
                        branch = metadata_branches.At(i)
                        name = branch.GetName()
                        if name == 'index_ref':
                            # skip the index branch
                            continue
 
                        class_name = branch.GetClassName()
 
                        if regexIOVMetaDataContainer.match(class_name):
                            name = name.replace('IOVMetaDataContainer_p1_', '').replace('_', '/')
 
                        if regexIOVMetaDataContainer.match(class_name):
                            meta_dict[filename]['metadata_items'][name] = 'IOVMetaDataContainer'
                        elif regexByteStreamMetadataContainer.match(class_name):
                            meta_dict[filename]['metadata_items'][name] = 'ByteStreamMetadataContainer'
                        elif regexEventStreamInfo.match(class_name):
                            meta_dict[filename]['metadata_items'][name] = 'EventStreamInfo'
                        elif regexXAODFileMetaData.match(class_name):
                            meta_dict[filename]['metadata_items'][name] = 'FileMetaData'
                        elif regexXAODTruthMetaData.match(class_name):
                            meta_dict[filename]['metadata_items'][name] = 'TruthMetaData'
                        else:
                            type_name = class_name
                            if not type_name:
                                try:
                                    type_name = branch.GetListOfLeaves()[0].GetTypeName()
                                except IndexError:
                                    pass
                            meta_dict[filename]['metadata_items'][name] = type_name
 
                        if len(meta_filter) > 0:
                            keep = False
                            for filter_key, filter_class in meta_filter.items():
                                if (filter_key.replace('/', '_') in name.replace('/', '_') or filter_key == '*') and fnmatchcase(class_name, filter_class):
                                    if 'CutBookkeepers' in filter_key:
                                        keep = filter_key == name
                                        if keep:
                                            break
                                    else:
                                        keep = True
                                        break
 
                            if not keep:
                                continue
                        else:
                            # CutBookkeepers should always be filtered:
                            if 'CutBookkeepers' in name and name not in ['CutBookkeepers', 'CutBookkeepersAux.']:
                                continue
 
                        if not isGaudiEnv():
                            metadata_tree.SetBranchStatus(f"{name}*", True)
 
                        # assign the corresponding persistent class based of the name of the metadata container
                        if regexEventStreamInfo.match(class_name):
                            if class_name.endswith('_p1'):
                                persistent_instances[name] = ROOT.EventStreamInfo_p1()
                            elif class_name.endswith('_p2'):
                                persistent_instances[name] = ROOT.EventStreamInfo_p2()
                            else:
                                persistent_instances[name] = ROOT.EventStreamInfo_p3()
                        elif regexIOVMetaDataContainer.match(class_name):
                            persistent_instances[name] = ROOT.IOVMetaDataContainer_p1()
                        elif regexXAODEventFormat.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.EventFormat_v1()
                        elif regexXAODTriggerMenu.match(class_name) and _check_project() not in ['AthGeneration']:
                            persistent_instances[name] = ROOT.xAOD.TriggerMenuContainer_v1()
                        elif regexXAODTriggerMenuAux.match(class_name) and _check_project() not in ['AthGeneration']:
                            persistent_instances[name] = ROOT.xAOD.TriggerMenuAuxContainer_v1()
                        elif regexXAODTriggerMenuJson.match(class_name) and _check_project() not in ['AthGeneration']:
                            persistent_instances[name] = ROOT.xAOD.TriggerMenuJsonContainer_v1()
                        elif regexXAODTriggerMenuJsonAux.match(class_name) and _check_project() not in ['AthGeneration']:
                            persistent_instances[name] = ROOT.xAOD.TriggerMenuJsonAuxContainer_v1()
                        elif regexXAODCutBookkeeperContainer.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.CutBookkeeperContainer_v1()
                        elif regexXAODCutBookkeeperContainerAux.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.CutBookkeeperAuxContainer_v1()
                        elif regexXAODFileMetaData.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.FileMetaData_v1()
                        elif regexXAODFileMetaDataAux.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.FileMetaDataAuxInfo_v1()
                        elif regexXAODTruthMetaData.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.TruthMetaDataContainer_v1()
                        elif regexXAODTruthMetaDataAux.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.TruthMetaDataAuxContainer_v1()
 
                        if name in persistent_instances:
                            branch.SetAddress(ROOT.AddressOf(persistent_instances[name]))
 
                        # This creates a dict to store the dynamic attributes of the xAOD::FileMetaData
                        dynamicFMD = regexXAODFileMetaDataAuxDyn.match(name)
                        if dynamicFMD:
                            dynamicName = dynamicFMD.group().split('.')[-1]
                            dynamicType = regex_cppname.match(class_name)
                            if dynamicType:
                                # this should be a string
                                dynamic_fmd_items[dynamicName] = ROOT.std.string()
                                branch.SetAddress(ROOT.AddressOf(dynamic_fmd_items[dynamicName]))
                            else:
                                dynamic_fmd_items[dynamicName] = None
 
 
                    metadata_tree.GetEntry(0)
 
                    # This loads the dynamic attributes of the xAOD::FileMetaData from the TTree
                    for key in dynamic_fmd_items:
                        if dynamic_fmd_items[key] is None:
                            try:
                                if key.startswith("is"):
                                    # this is probably a boolean
                                    dynamic_fmd_items[key] = getattr(metadata_tree, key) != '\x00'
                                else:
                                    # this should be a float
                                    dynamic_fmd_items[key] = getattr(metadata_tree, key)
                            except AttributeError:
                                # should not happen, but just ignore missing attributes
                                pass
                        else:
                            # convert ROOT.std.string objects to python equivalent
                            dynamic_fmd_items[key] = str(dynamic_fmd_items[key])
 
                    # clean the meta-dict if the meta_key_filter flag is used, to return only the key of interest
                    if meta_key_filter:
                        meta_dict[filename] = {}
 
                    # Find the RunNumber from EventStreamInfo, so only the relevant IOVs are read later
                    for key, obj in persistent_instances.items():
                        if key.startswith('EventStreamInfo'):
                            data = _convert_value(obj, None)
                            if data and 'runNumbers' in data and len(data['runNumbers']) == 1:
                                _gbl_run = data['runNumbers'][0]
                                msg.debug("Found RunNumber in EventStreamInfo: %d", _gbl_run)
 
                    # read the metadata
                    for name, content in persistent_instances.items():
                        key = name
                        if hasattr(content, 'm_folderName'):
                            key = content.m_folderName
 
                        # Some transition AODs contain both the Run2 and Run3 metadata formats. We only wish to read the Run3 format if such a file is encountered.
                        has_r3_trig_meta = ('TriggerMenuJson_HLT' in persistent_instances or 'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLT' in persistent_instances)
                        aux = None
                        if key.startswith('TriggerMenuJson_') and not key.endswith('Aux.'): # interface container for the menu (AOD)
                            aux = persistent_instances[key+'Aux.']
                        elif key.startswith('DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_') and not key.endswith('Aux.'): # interface container for the menu (ESD)
                            menuPart = key.split('_')[-1]
                            aux = persistent_instances['xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_'+menuPart+'Aux.']    
                        elif key == 'TriggerMenu' and 'TriggerMenuAux.' in persistent_instances and not has_r3_trig_meta: # AOD case (legacy support, HLT and L1 menus)
                            aux = persistent_instances['TriggerMenuAux.']
                        elif key == 'DataVector<xAOD::TriggerMenu_v1>_TriggerMenu' and 'xAOD::TriggerMenuAuxContainer_v1_TriggerMenuAux.' in persistent_instances and not has_r3_trig_meta: # ESD case (legacy support, HLT and L1 menus)
                            aux = persistent_instances['xAOD::TriggerMenuAuxContainer_v1_TriggerMenuAux.']
                        elif (key == 'CutBookkeepers'
                            and 'CutBookkeepersAux.' in persistent_instances):
                            aux = persistent_instances['CutBookkeepersAux.']
                        elif key == 'CutBookkeepersAux.':
                            continue   # Extracted using the interface object
                        elif (key == 'FileMetaData'
                            and 'FileMetaDataAux.' in persistent_instances):
                            aux = persistent_instances['FileMetaDataAux.']
                        elif (key == 'xAOD::FileMetaData_v1_FileMetaData'
                            and 'xAOD::FileMetaDataAuxInfo_v1_FileMetaDataAux.' in persistent_instances):
                            aux = persistent_instances['xAOD::FileMetaDataAuxInfo_v1_FileMetaDataAux.']
                        elif (key == 'TruthMetaData'
                            and 'TruthMetaDataAux.' in persistent_instances):
                            aux = persistent_instances['TruthMetaDataAux.']
                        elif key == 'TruthMetaDataAux.':
                            continue   # Extracted using the interface object
                        elif 'Menu' in key and key.endswith('Aux.'):
                            continue   # Extracted using the interface object
 
                        return_obj = _convert_value(content, aux)
 
                        if 'TriggerMenuJson' in key  or ('TriggerMenu' in key and not has_r3_trig_meta):
                            if 'RAWTriggerMenuJson' in return_obj:
                                meta_dict[filename][key] = return_obj['RAWTriggerMenuJson']
                                del return_obj['RAWTriggerMenuJson']
                            if 'TriggerConfigInfo' not in meta_dict[filename]:
                                meta_dict[filename]['TriggerConfigInfo'] = {}
                            if 'dbkey' in return_obj:
                                meta_dict[filename]['TriggerConfigInfo'][key.split('_')[-1]] = {
                                    'key' : return_obj['dbkey'],
                                    'name': return_obj['name']
                                    }
                                del return_obj['dbkey']
                                del return_obj['name']
                            if 'TriggerMenu' not in meta_dict[filename]:
                                meta_dict[filename]['TriggerMenu'] = {}
                            meta_dict[filename]['TriggerMenu'].update(return_obj)
                        elif "FileMetaData" in key:
                            if "FileMetaData" not in meta_dict[filename]:
                                meta_dict[filename]["FileMetaData"] = dynamic_fmd_items
                            meta_dict[filename]["FileMetaData"].update(return_obj)
                        else:
                            meta_dict[filename][key] = return_obj
 
            try:
                # get the number of events from EventStreamInfo
                esi_dict = next(key for key, value in meta_dict[filename].items()
                                if isinstance(value, dict) and "numberOfEvents" in value and
                                meta_dict[filename]["metadata_items"][key] == "EventStreamInfo")
                msg.debug(f"{esi_dict=}")
                meta_dict[filename]["nentries"] = meta_dict[filename][esi_dict]["numberOfEvents"]
            except StopIteration as err:
                msg.debug(f"Caught {err=}, {type(err)=}, falling back on opening the DataHeader"
                          " Container to read the number of entries")
                meta_dict[filename]['nentries'] = dataheader_nentries(current_file)
                msg.debug(f"{meta_dict[filename]['nentries']=}")
 
            if unique_tag_info_values and mode=='iov':
                unique_tag_info_values = False
                msg.info('disabling "unique_tag_info_values" option for "iov" mode')
 
            # This is a required workaround which will temporarily be fixing ATEAM-560 originated from  ATEAM-531
            # ATEAM-560: https://its.cern.ch/jira/browse/ATEAM-560
            # ATEAM-531: https://its.cern.ch/jira/browse/ATEAM-531
            # This changes will remove all duplicates values presented in some files due
            # to the improper merging of two IOVMetaDataContainers.
            if unique_tag_info_values:
                msg.info('MetaReader is called with the parameter "unique_tag_info_values" set to True. '
                         'This is a workaround to remove all duplicate values from "/TagInfo" key')
                if '/TagInfo' in meta_dict[filename]:
                    for key, value in meta_dict[filename]['/TagInfo'].items():
                        if isinstance(value, list) and value:
                            if len(unique_values := set(value)) > 1:
                                msg.warning(
                                    f"Found multiple values for {key}: {value}. "
                                    "Looking for possible duplicates."
                                )
                                maybe_ok = False
                                if key == "AMITag":
                                    # curate duplicates like: ['s3681_q453', 's3681_q453_'] or ["s3681_q453", "q453_s3681"]
                                    unique_amitags = set()
                                    for amitags in unique_values:
                                        unique_amitags.add(
                                            "_".join([tag for tag in amitags.split("_") if tag])
                                        )
                                    # Remove tags that are parents - are contained in other tags
                                    # Such that ["s3681","s3681_d1485"] keeps only the latter
                                    parent_tags = []
                                    for atag in unique_amitags:
                                        if any(atag+'_' in x for x in unique_amitags if x != atag):
                                            parent_tags += [atag]
                                    for atag in parent_tags:
                                        # Do not remove the last tag!
                                        if len(unique_amitags)>1:
                                            msg.warning(f"Removing parent AMI tag {atag}")
                                            unique_amitags.remove(atag)
                                    if len(unique_amitags) == 1:
                                        maybe_ok = True
                                        # Make sure we keep the one we want to keep
                                        value.insert(0,list(unique_amitags)[0])
                                elif key == "beam_energy":
                                    # handle duplicates like: ['6500000', '6500000.0'] or [3, "3"]
                                    unique_energies = set()
                                    for energy in unique_values:
                                        try:
                                            energy = int(energy)
                                        except ValueError:
                                            try:
                                                energy = float(energy)
                                            except ValueError:
                                                pass
                                        unique_energies.add(energy)
                                    if len(unique_energies) == 1:
                                        maybe_ok = True
                                elif key in ["AtlasRelease", "IOVDbGlobalTag", "AODFixVersion"]:
                                    maybe_ok = True
                                if maybe_ok:
                                    msg.warning(
                                        f"Multiple values for {key} may mean the same, or "
                                        "the input file was produced in multi-step job. "
                                        f"Ignoring all but the first entry: {key} = {value[0]}"
                                    )
                                else:
                                    raise ValueError(
                                        f"{key} from /TagInfo contains more than 1 unique value: {value}"
                                    )
 
                            meta_dict[filename]['/TagInfo'][key] = value[0]
 
            if promote is None:
                promote = mode == 'lite' or mode == 'peeker'
 
            # Filter the data and create a prettier output for the 'lite' mode
            if mode == 'lite':
                meta_dict = make_lite(meta_dict)
 
            if mode == 'peeker':
                meta_dict = make_peeker(meta_dict)
 
            if promote:
                meta_dict = promote_keys(meta_dict, mode)
 
            # If AnalysisBase the itemList must be grabbed another way
            if not isGaudiEnv():
                if isinstance(collectionTree, ROOT.TTree):
                    meta_dict[filename]['itemList'] = [ (b.GetClassName(), b.GetName()) for b in collectionTree.GetListOfBranches() ]
 
        # ----- retrieves metadata from bytestream (BS) files (RAW, DRAW) ------------------------------------------#
        elif current_file_type == 'BS':
            
            if ignoreNonExistingLocalFiles and not regex_URI_scheme.match(filename) and not os.path.isfile(filename): 
                msg.warning('Ignoring not accessible file: {}'.format(filename))
                continue
            
            import eformat
 
            # store the number of entries
            bs = eformat.istream(filename)
            meta_dict[filename]['nentries'] = bs.total_events
 
            # store the 'guid' value
            data_reader = eformat.EventStorage.pickDataReader(filename)
            assert data_reader, 'problem picking a data reader for file [%s]' % filename
 
            # set auto flush equivalent, which for BS is always 1
            meta_dict[filename]['auto_flush'] = 1
 
            if hasattr(data_reader, 'GUID'):
                meta_dict[filename]['file_guid'] = data_reader.GUID()
 
            # compression level and algorithm, for BS always ZLIB
            meta_dict[filename]['file_comp_alg'] = 1
            meta_dict[filename]['file_comp_level'] = 1
 
 
            # if the flag full is set to true then grab all metadata
            # ------------------------------------------------------------------------------------------------------#
            if mode != "tiny":
                bs_metadata = {}
 
                for md in data_reader.freeMetaDataStrings():
                    if md.startswith('Event type:'):
                        k = 'eventTypes'
                        v = []
                        if 'is sim' in md:
                            v.append('IS_SIMULATION')
                        else:
                            v.append('IS_DATA')
 
                        if 'is atlas' in md:
                            v.append('IS_ATLAS')
                        else:
                            v.append('IS_TESTBEAM')
 
                        if 'is physics' in md:
                            v.append('IS_PHYSICS')
                        else:
                            v.append('IS_CALIBRATION')
 
                        bs_metadata[k] = tuple(v)
 
                    elif md.startswith('GeoAtlas:'):
                        k = 'geometry'
                        v = md.split('GeoAtlas:')[1].strip()
                        bs_metadata[k] = v
 
                    elif md.startswith('IOVDbGlobalTag:'):
                        k = 'conditions_tag'
                        v = md.split('IOVDbGlobalTag:')[1].strip()
                        bs_metadata[k] = v
 
                    elif '=' in md:
                        k, v = md.split('=', 1)  # Split on first '=' only
                        bs_metadata[k] = v
 
                bs_metadata['detectorMask'] = data_reader.detectorMask()
                bs_metadata['runNumbers'] = data_reader.runNumber()
                bs_metadata['lumiBlockNumbers'] = data_reader.lumiblockNumber()
                bs_metadata['projectTag'] = data_reader.projectTag()
                bs_metadata['stream'] = data_reader.stream()
                #bs_metadata['beamType'] = getattr(data_reader, 'beamType')()
                beamTypeNbr= data_reader.beamType()
                #According to info from Rainer and Guiseppe the beam type is
                #O: no beam
                #1: protons
                #2: ions
                if (beamTypeNbr==0): bs_metadata['beamType'] = 'cosmics'
                elif (beamTypeNbr==1 or beamTypeNbr==2):  bs_metadata['beamType'] = 'collisions'
                else: bs_metadata['beamType'] = 'unknown'
 
                bs_metadata['beamEnergy'] = data_reader.beamEnergy()
 
                meta_dict[filename]['eventTypes'] = bs_metadata.get('eventTypes', [])
                meta_dict[filename]['GeoAtlas'] = bs_metadata.get('geometry', None)
                meta_dict[filename]['conditions_tag'] = bs_metadata.get('conditions_tag', None)
                meta_dict[filename]['project_name'] = bs_metadata.get('projectTag', None)
 
                # Promote up one level
                meta_dict[filename]['detectorMask'] = [bs_metadata.get('detectorMask', None)]
                meta_dict[filename]['runNumbers'] = [bs_metadata.get('runNumbers', None)]
                meta_dict[filename]['lumiBlockNumbers'] = [bs_metadata.get('lumiBlockNumbers', None)]
                meta_dict[filename]['beam_type'] = bs_metadata.get('beamType', None)
                meta_dict[filename]['beam_energy'] = bs_metadata.get('beamEnergy', None)
                meta_dict[filename]['stream'] = bs_metadata.get('stream', None)
 
                if not data_reader.good():
                    # event-less file...
                    meta_dict[filename]['runNumbers'].append(bs_metadata.get('run_number', 0))
                    meta_dict[filename]['lumiBlockNumbers'].append(bs_metadata.get('LumiBlock', 0))
 
                msg.debug(f"{meta_dict[filename]=}")
                msg.debug(f"{len(bs)=}")
                if len(bs):
                    evt = bs[0]
                    try:
                        evt.check()
                        meta_dict[filename]['processingTags'] = [tag.name for tag in evt.stream_tag()]
                        meta_dict[filename]['evt_number'] = [evt.global_id()]
                        meta_dict[filename]['run_type'] = [eformat.helper.run_type2string(evt.run_type())]
                        # ATLASRECTS-7126: If there is no valid lumiblock information
                        # in the ByteStream header, get the info from the first event.
                        if meta_dict[filename]['lumiBlockNumbers'] == [0]:
                            msg.debug('Taking the luminosity block info from the first event (%i)', evt.lumi_block())
                            meta_dict[filename]['lumiBlockNumbers'] = [evt.lumi_block()]
                        # ATLASRECTS-7126: If there is no valid run number information
                        # in the ByteStream header, get the info from the first event.
                        if meta_dict[filename]['runNumbers'] == [0]:
                            msg.debug('Taking the run number info from the first event (%i)', evt.run_no())
                            meta_dict[filename]['runNumbers'] = [evt.run_no()]
                    except RuntimeError as err:
                        msg.error("Issue while reading the first event of BS file %r: %r", filename, err)
                    else:
                        msg.debug(f"{meta_dict[filename]=}")
                else:
                    msg.warning(f"Event-less BS {filename=}, will not read metadata information from the first event")
 
                # fix for ATEAM-122
                if len(bs_metadata.get('eventTypes', '')) == 0:  # see: ATMETADATA-6
                    evt_type = ['IS_DATA', 'IS_ATLAS']
                    if bs_metadata.get('stream', '').startswith('physics_'):
                        evt_type.append('IS_PHYSICS')
                    elif bs_metadata.get('stream', '').startswith('calibration_'):
                        evt_type.append('IS_CALIBRATION')
                    elif bs_metadata.get('projectTag', '').endswith('_calib'):
                        evt_type.append('IS_CALIBRATION')
                    else:
                        evt_type.append('Unknown')
 
                    meta_dict[filename]['eventTypes'] = evt_type
 
                if mode == 'full':
                    meta_dict[filename]['bs_metadata'] = bs_metadata
 
        # ------ Throw an error if the user provide other file types -------------------------------------------------#
        else:
            msg.error('Unknown filetype for {0} - there is no metadata interface for type {1}'.format(filename, current_file_type))
            return None
 
    return meta_dict
 
 

should_keep_meta()

python.MetaReader.should_keep_meta	(		normalizedName,
			typeName,
			meta_filter )

Helper function to determine if metadata should be kept based on meta_filter.

Definition at line 1937 of file MetaReader.py.

def should_keep_meta(normalizedName, typeName, meta_filter):
    """
    Helper function to determine if metadata should be kept based on meta_filter.
    """
    if len(meta_filter) == 0:
        return True
 
    for filter_key, filter_class in meta_filter.items():
        if (
            filter_key.replace("/", "_") in normalizedName.replace("/", "_")
            or filter_key == "*"
        ) and fnmatchcase(typeName, filter_class):
            if "CutBookkeepers" in filter_key:
                keep = filter_key == normalizedName
                if keep:
                    return True
            else:
                return True
    return False

Variable Documentation

lite_primary_keys_to_keep

list python.MetaReader.lite_primary_keys_to_keep

Initial value:

=  [
    'lumiBlockNumbers', 'runNumbers', 'mc_event_number', 'mc_channel_number',
    'eventTypes', 'processingTags', 'itemList']

Definition at line 37 of file MetaReader.py.

lite_TagInfo_keys_to_keep

list python.MetaReader.lite_TagInfo_keys_to_keep

Initial value:

=  [
    'beam_energy', 'beam_type', 'GeoAtlas', 'IOVDbGlobalTag',
    'AODFixVersion', 'project_name', 'mc_campaign', 'keywords']

Definition at line 40 of file MetaReader.py.

msg

python.MetaReader.msg = logging.getLogger('MetaReader')

Definition at line 13 of file MetaReader.py.

regex_BS_files

python.MetaReader.regex_BS_files = re.compile(r'^(\w+):.*((\.D?RAW\..*)|(\.data$))')

Definition at line 34 of file MetaReader.py.

regex_cppname

python.MetaReader.regex_cppname = re.compile(r'^([\w:]+)(<.*>)?$')

Definition at line 31 of file MetaReader.py.

regex_persistent_class

python.MetaReader.regex_persistent_class = re.compile(r'^([a-zA-Z]+(_[pv]\d+)?::)*[a-zA-Z]+_[pv]\d+$')

Definition at line 33 of file MetaReader.py.

regex_URI_scheme

python.MetaReader.regex_URI_scheme = re.compile(r'^([A-Za-z0-9\+\.\-]+)\:')

Definition at line 35 of file MetaReader.py.

regexByteStreamMetadataContainer

python.MetaReader.regexByteStreamMetadataContainer = re.compile(r'^ByteStreamMetadataContainer(_p\d+)?$')

Definition at line 18 of file MetaReader.py.

regexEventStreamInfo

python.MetaReader.regexEventStreamInfo = re.compile(r'^EventStreamInfo(_p\d+)?$')

Definition at line 16 of file MetaReader.py.

regexIOVMetaDataContainer

python.MetaReader.regexIOVMetaDataContainer = re.compile(r'^IOVMetaDataContainer(_p\d+)?$')

Definition at line 17 of file MetaReader.py.

regexXAODCutBookkeeperContainer

python.MetaReader.regexXAODCutBookkeeperContainer = re.compile(r'^xAOD::CutBookkeeperContainer(_v\d+)?$')

Definition at line 19 of file MetaReader.py.

regexXAODCutBookkeeperContainerAux

python.MetaReader.regexXAODCutBookkeeperContainerAux = re.compile(r'^xAOD::CutBookkeeperAuxContainer(_v\d+)?$')

Definition at line 20 of file MetaReader.py.

regexXAODEventFormat

python.MetaReader.regexXAODEventFormat = re.compile(r'^xAOD::EventFormat(_v\d+)?$')

Definition at line 21 of file MetaReader.py.

regexXAODFileMetaData

python.MetaReader.regexXAODFileMetaData = re.compile(r'^xAOD::FileMetaData(_v\d+)?$')

Definition at line 22 of file MetaReader.py.

regexXAODFileMetaDataAux

python.MetaReader.regexXAODFileMetaDataAux = re.compile(r'^xAOD::FileMetaDataAuxInfo(_v\d+)?$')

Definition at line 23 of file MetaReader.py.

regexXAODFileMetaDataAuxDyn

python.MetaReader.regexXAODFileMetaDataAuxDyn = re.compile(r'^(xAOD::)?FileMetaData.*AuxDyn(\.[a-zA-Z0-9]+)?$')

Definition at line 24 of file MetaReader.py.

regexXAODTriggerMenu

python.MetaReader.regexXAODTriggerMenu = re.compile(r'^DataVector<xAOD::TriggerMenu(_v\d+)?>$')

Definition at line 25 of file MetaReader.py.

regexXAODTriggerMenuAux

python.MetaReader.regexXAODTriggerMenuAux = re.compile(r'^xAOD::TriggerMenuAuxContainer(_v\d+)?$')

Definition at line 26 of file MetaReader.py.

regexXAODTriggerMenuJson

python.MetaReader.regexXAODTriggerMenuJson = re.compile(r'^DataVector<xAOD::TriggerMenuJson(_v\d+)?>$')

Definition at line 27 of file MetaReader.py.

regexXAODTriggerMenuJsonAux

python.MetaReader.regexXAODTriggerMenuJsonAux = re.compile(r'^xAOD::TriggerMenuJsonAuxContainer(_v\d+)?$')

Definition at line 28 of file MetaReader.py.

regexXAODTruthMetaData

python.MetaReader.regexXAODTruthMetaData = re.compile(r'^DataVector<xAOD::TruthMetaData(_v\d+)?>$')

Definition at line 29 of file MetaReader.py.

regexXAODTruthMetaDataAux

python.MetaReader.regexXAODTruthMetaDataAux = re.compile(r'^xAOD::TruthMetaDataAuxContainer(_v\d+)?$')

Definition at line 30 of file MetaReader.py.

trigger_keys

list python.MetaReader.trigger_keys

Initial value:

=  [
    'TriggerConfigInfo',
    'TriggerMenu', 'TriggerMenuJson_BG', 'TriggerMenuJson_HLT', 'TriggerMenuJson_HLTMonitoring', 'TriggerMenuJson_HLTPS', 'TriggerMenuJson_L1', 'TriggerMenuJson_L1PS',
    '/TRIGGER/HLT/Groups', '/TRIGGER/HLT/HltConfigKeys', '/TRIGGER/HLT/Menu', '/TRIGGER/HLT/PrescaleKey', '/TRIGGER/HLT/Prescales',
    '/TRIGGER/LVL1/ItemDef', '/TRIGGER/LVL1/Lvl1ConfigKey', '/TRIGGER/LVL1/Menu', '/TRIGGER/LVL1/Prescales', '/TRIGGER/LVL1/Thresholds',
    'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenu', 'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_BG', 'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLT',
    'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTMonitoring', 'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTPS',
    'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1', 'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1PS',
]

Definition at line 44 of file MetaReader.py.

trigger_menu_json_map

dict python.MetaReader.trigger_menu_json_map

Initial value:

=  {
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_L1PSAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1PS",
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_BGAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_BG",
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLT",
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTMonitoringAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTMonitoring",
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTPSAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTPS",
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_L1Aux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1",
    "TriggerMenuJson_L1PSAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1PS",
    "TriggerMenuJson_BGAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_BG",
    "TriggerMenuJson_HLTAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLT",
    "TriggerMenuJson_HLTMonitoringAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTMonitoring",
    "TriggerMenuJson_HLTPSAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTPS",
    "TriggerMenuJson_L1Aux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1",
}

Definition at line 54 of file MetaReader.py.