MetaReader.py

Go to the documentation of this file.

# Copyright (C) 2002-2026 CERN for the benefit of the ATLAS collaboration
 
import functools
import os
import re
import warnings
from fnmatch import fnmatchcase
from AthenaCommon.Logging import logging
from AthenaConfiguration.AthConfigFlags import isGaudiEnv
from PyUtils.PoolFile import isRNTuple
from ROOT import gSystem
from AthenaConfiguration.Enums import Project
 
 
msg = logging.getLogger('MetaReader')
 
# compile the regex needed in _convert_value() outside it to optimize the code.
regexEventStreamInfo = re.compile(r'^EventStreamInfo(_p\d+)?$')
regexIOVMetaDataContainer = re.compile(r'^IOVMetaDataContainer(_p\d+)?$')
regexByteStreamMetadataContainer = re.compile(r'^ByteStreamMetadataContainer(_p\d+)?$')
regexXAODCutBookkeeperContainer = re.compile(r'^xAOD::CutBookkeeperContainer(_v\d+)?$')
regexXAODCutBookkeeperContainerAux = re.compile(r'^xAOD::CutBookkeeperAuxContainer(_v\d+)?$')
regexXAODEventFormat = re.compile(r'^xAOD::EventFormat(_v\d+)?$')
regexXAODFileMetaData = re.compile(r'^xAOD::FileMetaData(_v\d+)?$')
regexXAODFileMetaDataAux = re.compile(r'^xAOD::FileMetaDataAuxInfo(_v\d+)?$')
regexXAODFileMetaDataAuxDyn = re.compile(r'^(xAOD::)?FileMetaData.*AuxDyn(\.[a-zA-Z0-9]+)?$')
regexXAODTriggerMenu = re.compile(r'^DataVector<xAOD::TriggerMenu(_v\d+)?>$') # Run 2
regexXAODTriggerMenuAux = re.compile(r'^xAOD::TriggerMenuAuxContainer(_v\d+)?$') # Run 2
regexXAODTriggerMenuJson = re.compile(r'^DataVector<xAOD::TriggerMenuJson(_v\d+)?>$') # Run 3
regexXAODTriggerMenuJsonAux = re.compile(r'^xAOD::TriggerMenuJsonAuxContainer(_v\d+)?$') # Run 3
regexXAODTruthMetaData = re.compile(r'^DataVector<xAOD::TruthMetaData(_v\d+)?>$')
regexXAODTruthMetaDataAux = re.compile(r'^xAOD::TruthMetaDataAuxContainer(_v\d+)?$')
regex_cppname = re.compile(r'^([\w:]+)(<.*>)?$')
# regex_persistent_class = re.compile(r'^([a-zA-Z]+_p\d+::)*[a-zA-Z]+_p\d+$')
regex_persistent_class = re.compile(r'^([a-zA-Z]+(_[pv]\d+)?::)*[a-zA-Z]+_[pv]\d+$')
regex_BS_files = re.compile(r'^(\w+):.*((\.D?RAW\..*)|(\.data$))')
regex_URI_scheme = re.compile(r'^([A-Za-z0-9\+\.\-]+)\:')
 
lite_primary_keys_to_keep = [
    'lumiBlockNumbers', 'runNumbers', 'mc_event_number', 'mc_channel_number',
    'eventTypes', 'processingTags', 'itemList']
lite_TagInfo_keys_to_keep = [
    'beam_energy', 'beam_type', 'GeoAtlas', 'IOVDbGlobalTag',
    'AODFixVersion', 'project_name', 'mc_campaign', 'keywords']
 
trigger_keys = [
    'TriggerConfigInfo',
    'TriggerMenu', 'TriggerMenuJson_BG', 'TriggerMenuJson_HLT', 'TriggerMenuJson_HLTMonitoring', 'TriggerMenuJson_HLTPS', 'TriggerMenuJson_L1', 'TriggerMenuJson_L1PS',
    '/TRIGGER/HLT/Groups', '/TRIGGER/HLT/HltConfigKeys', '/TRIGGER/HLT/Menu', '/TRIGGER/HLT/PrescaleKey', '/TRIGGER/HLT/Prescales',
    '/TRIGGER/LVL1/ItemDef', '/TRIGGER/LVL1/Lvl1ConfigKey', '/TRIGGER/LVL1/Menu', '/TRIGGER/LVL1/Prescales', '/TRIGGER/LVL1/Thresholds',
    'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenu', 'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_BG', 'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLT',
    'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTMonitoring', 'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTPS',
    'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1', 'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1PS',
]
 
trigger_menu_json_map = {
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_L1PSAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1PS",
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_BGAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_BG",
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLT",
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTMonitoringAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTMonitoring",
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTPSAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTPS",
    "xAOD__TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_L1Aux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1",
    "TriggerMenuJson_L1PSAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1PS",
    "TriggerMenuJson_BGAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_BG",
    "TriggerMenuJson_HLTAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLT",
    "TriggerMenuJson_HLTMonitoringAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTMonitoring",
    "TriggerMenuJson_HLTPSAux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTPS",
    "TriggerMenuJson_L1Aux:": "DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1",
}
 
# fixed-width integer types used in RNTuple type names and their ROOT class name equivalents
_rntuple_integer_types = {
    'std::int8_t': 'signed char',
    'std::uint8_t': 'unsigned char',
    'std::int16_t': 'short',
    'std::uint16_t': 'unsigned short',
    'std::int32_t': 'int',
    'std::uint32_t': 'unsigned int',
    'std::int64_t': 'long',
    'std::uint64_t': 'unsigned long',
}
_rntuple_fundamental_types = {'bool', 'char', 'float', 'double', *_rntuple_integer_types}
 
 
def ignore_missing_dictionary_warnings(func):
    """Ignore ROOT warnings about missing class dictionaries while func runs."""
 
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        with warnings.catch_warnings():
            warnings.filterwarnings(
                "ignore",
                message=r".*no dictionary for class",
                category=RuntimeWarning,
            )
            return func(*args, **kwargs)
 
    return wrapper
 
 
@ignore_missing_dictionary_warnings
def read_metadata(filenames, file_type = None, mode = 'lite', promote = None, meta_key_filter = None,
                  unique_tag_info_values = True, ignoreNonExistingLocalFiles=False, normalize_type_names = False):
    """
    This tool is independent of Athena framework and returns the metadata from a given file.
    :param filenames: the input file from which metadata needs to be extracted.
    :param file_type: the type of file. POOL or BS (bytestream: RAW, DRAW) files.
    :param mode: if true, will return all metadata associated with the filename. By default, is false and this will
    return a "tiny" version which have only the following keys: 'file_guid', 'file_size', 'file_type', 'nentries'.
    :param normalize_type_names: report the RNTuple item types using the class names of the equivalent TTree branches,
    and the TTree item names using the names of the equivalent RNTuple fields, with the items sorted, e.g. to compare
    the content of RNTuple and TTree files.
    :return: a dictionary of metadata for the given input file.
    """
 
    # make the mode and run available in the _convert methods
    global _gbl_mode, _gbl_run
    _gbl_mode = mode
    _gbl_run = None
 
    from RootUtils import PyROOTFixes  # noqa F401
 
    # Check if the input is a file or a list of files.
    if isinstance(filenames, str):
        filenames = [filenames]
 
    # Check if file_type is an allowed value
    if file_type is not None:
        if file_type not in ('POOL', 'BS'):
            raise NameError('Allowed values for \'file_type\' parameter are: "POOL" or "BS": you provided "' + file_type + '"')
        else:
            msg.info('Forced file_type: {0}'.format(file_type))
 
    # Check the value of mode parameter
    if mode not in ('tiny', 'lite', 'full', 'peeker', 'iov'):
        raise NameError('Allowed values for "mode" parameter are: "tiny", "lite", "peeker", "iov" or "full"')
 
    if meta_key_filter is None:
        meta_key_filter = []
 
    # Disable 'full' and 'iov' in non-Gaudi environments
    if not isGaudiEnv():
        if mode in ('full', 'iov'):
            raise NameError('The following modes are not available in AnalysisBase: "iov" and "full"')
 
    msg.info('Current mode used: {0}'.format(mode))
    msg.info('Current filenames: {0}'.format(filenames))
 
    if mode != 'full' and mode !='iov' and len(meta_key_filter) > 0:
        raise NameError('It is possible to use the meta_key_filter option only for full mode')
    if meta_key_filter:
        msg.info('Filter used: {0}'.format(meta_key_filter))
 
    # create the storage object for metadata.
    meta_dict = {}
 
    # ----- retrieve metadata from all filename or filenames --------------------------------------------------------#
    for filename in filenames:
        meta_dict[filename] = {}
        current_file_type = None
        # reset the global Run for each file to not propagate and old value
        _gbl_run = None
        # Determine the file_type of the input and store this information into meta_dict
        if not file_type:
            if os.path.isfile(filename):
                
                if ignoreNonExistingLocalFiles and not regex_URI_scheme.match(filename) and gSystem.AccessPathName(filename): # Attention, bizarre convention of return value!! 
                    msg.warning('Ignoring not accessible file: {}'.format(filename))
                    continue
                    
                with open(filename, 'rb') as binary_file:
                    magic_file = binary_file.read(4)
 
                    if magic_file == 'root' or magic_file == b'root':
                        current_file_type = 'POOL'
                        meta_dict[filename]['file_type'] = 'POOL'
 
                    elif Project.determine() in (
                            Project.AnalysisBase, Project.AthAnalysis):
                        raise RuntimeError(
                            f"{filename} is not a ROOT file, assumed bytestream"
                            ", this is not supported in Analysis releases")
                    else:
                        current_file_type = 'BS'
                        meta_dict[filename]['file_type'] = 'BS'
 
                    # add information about the file_size of the input filename
                    meta_dict[filename]['file_size'] = os.path.getsize(filename)
 
            # determine the file type for the remote input files
            else:
                if regex_BS_files.match(filename):
                    current_file_type = 'BS'
                    meta_dict[filename]['file_type'] = 'BS'
                else:
                    current_file_type = 'POOL'
                    meta_dict[filename]['file_type'] = 'POOL'
 
                # add information about the file_size of the input filename
                meta_dict[filename]['file_size'] = None  # None -> we can't read the file size for a remote file
 
        else:
            current_file_type = file_type
 
        # ----- retrieves metadata from POOL files ------------------------------------------------------------------#
        if current_file_type == 'POOL':
            
            if ignoreNonExistingLocalFiles and not regex_URI_scheme.match(filename) and gSystem.AccessPathName(filename): # Attention, bizarre convention of return value!! 
                msg.warning('Ignoring not accessible file: {}'.format(filename))
                continue
                    
            import ROOT
            # open the file using ROOT.TFile
            current_file = ROOT.TFile.Open( _get_pfn(filename) )
 
            # get auto flush setting from the main EventData TTree
            from PyUtils.PoolFile import PoolOpts
            collectionTree = current_file.Get(PoolOpts.TTreeNames.EventData)
            if isinstance(collectionTree, ROOT.TTree):
                meta_dict[filename]['auto_flush'] = collectionTree.GetAutoFlush()
 
            # read and add the 'GUID' value
            meta_dict[filename]['file_guid'] = _read_guid(filename)
 
            # read and add compression level and algorithm
            meta_dict[filename]['file_comp_alg'] = current_file.GetCompressionAlgorithm()
            meta_dict[filename]['file_comp_level'] = current_file.GetCompressionLevel()
 
            if (
                isRNTuple(md:=current_file.Get(PoolOpts.RNTupleNames.MetaData))
                and mode != "tiny"
            ):
                meta_dict[filename]["metadata_items"] = {}
 
                # opening without a model only reads the header and footer,
                # the descriptor is used to decide which fields need to be read
                descriptor_reader = ROOT.RNTupleReader.Open(md)
                descriptor = descriptor_reader.GetDescriptor()
                auxes = {}
                classes_with_aux = {
                    "xAOD::FileMetaData_v1",
                    "xAOD::FileMetaDataAuxInfo_v1",
                    "xAOD::TriggerMenuAuxContainer_v1",
                    "DataVector<xAOD::TriggerMenu_v1>",
                    "xAOD::TriggerMenuJsonAuxContainer_v1",
                    "DataVector<xAOD::TriggerMenuJson_v1>",
                    "xAOD::TruthMetaDataAuxContainer_v1",
                    "DataVector<xAOD::TruthMetaData_v1>",
                    "xAOD::CutBookkeeperContainer_v1",
                    "xAOD::CutBookkeeperAuxContainer_v1",
                    "xAOD::LumiBlockRangeAuxContainer_v1",
                    "DataVector<xAOD::LumiBlockRange_v1>",
                }
 
                # field names to read, keyed by the dynamic attribute name
                dynamic_fmd_fields = {}
                # field names and types to read, keyed by the normalized name
                selected_fields = {}
 
                meta_filter = get_meta_filter(mode, meta_key_filter)
 
                for field in descriptor.GetTopLevelFields():
                    normalizedName = field.GetFieldName()
                    if "index_ref" in normalizedName:
                        continue
                    if regexIOVMetaDataContainer.match(field.GetTypeName()):
                        # if field name is e.g. IOVMetaDataContainer_p1__Digitization_Parameters,
                        # strip the prefix and change underscore to slash to slash
                        normalizedName = (
                            field.GetFieldName()
                            .replace("IOVMetaDataContainer_p1_", "")
                            .replace("_", "/")
                        )
                        meta_dict[filename]["metadata_items"][normalizedName] = (
                            "IOVMetaDataContainer"
                        )
                    elif regexByteStreamMetadataContainer.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][field.GetFieldName()] = (
                            "ByteStreamMetadataContainer"
                        )
                    elif regexEventStreamInfo.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][field.GetFieldName()] = (
                            "EventStreamInfo"
                        )
                    elif regexXAODFileMetaData.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName().replace("xAOD__", "xAOD::")
                        ] = field.GetTypeName()
                    elif regexXAODFileMetaDataAuxDyn.match(
                        normalizedName := field.GetFieldName()
                        .replace("xAOD__", "xAOD::")
                        .replace("AuxDyn:", "AuxDyn.")
                    ):
                        dynamic_fmd_fields[normalizedName.split(".")[1]] = (
                            field.GetFieldName()
                        )
                        meta_dict[filename]["metadata_items"][normalizedName] = (
                            field.GetTypeName()
                        )
                        continue
                    elif regexXAODFileMetaDataAux.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("Aux:", "Aux.")
                        ] = field.GetTypeName()
                    elif regexXAODTruthMetaData.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("DataVector_", "DataVector<")
                            .replace("__Truth", ">_Truth")
                        ] = "TruthMetaData"
                    elif regexXAODTruthMetaDataAux.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("Aux:", "Aux.")
                        ] = field.GetTypeName()
                    elif regexXAODEventFormat.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName().replace("xAOD__", "xAOD::")
                        ] = field.GetTypeName()
                    elif regexXAODTriggerMenuJson.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("DataVector_", "DataVector<")
                            .replace("__Trigger", ">_Trigger")
                        ] = field.GetTypeName()
                    elif regexXAODTriggerMenuJsonAux.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("Aux:", "Aux.")
                        ] = field.GetTypeName()
                    elif regexXAODCutBookkeeperContainer.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("DataVector_", "DataVector<")
                            .replace("__CutBookkeeper", ">_CutBookkeeper")
                        ] = field.GetTypeName()
                    elif regexXAODCutBookkeeperContainerAux.match(field.GetTypeName()):
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName()
                            .replace("xAOD__", "xAOD::")
                            .replace("Aux:", "Aux.")
                        ] = field.GetTypeName()
                    else:
                        meta_dict[filename]["metadata_items"][
                            field.GetFieldName().replace("Aux:", "Aux.")
                        ] = field.GetTypeName()
 
                    if field.GetTypeName() in classes_with_aux:
                        # handle aux classes later
                        auxes[field.GetFieldName()] = field.GetTypeName()
                        continue
 
                    if not should_keep_meta(
                        normalizedName, field.GetTypeName(), meta_filter
                    ):
                        continue
 
                    selected_fields[normalizedName] = (
                        field.GetFieldName(),
                        field.GetTypeName(),
                    )
 
                meta_dict[filename]["metadata_items"] = denormalize_metadata_types(
                    meta_dict[filename]["metadata_items"]
                )
 
                def _get_aux_base(aux_key: str) -> str:
                    # Remove known prefixes
                    key = aux_key
                    key = key.replace("xAOD__TriggerMenuJsonAuxContainer_v1_", "")
                    key = key.replace("xAOD__TriggerMenuAuxContainer_v1_", "")
                    key = key.replace("xAOD__FileMetaDataAuxInfo_v1_", "")
                    key = key.replace("xAOD__TruthMetaDataAuxContainer_v1_", "")
                    # Remove known suffixes
                    if key.endswith("Aux:"):
                        key = key[:-4]
                    elif key.endswith("Aux"):
                        key = key[:-3]
                    # Remove any trailing ':' or '_'
                    key = key.strip("_:")
                    return key
 
                def _get_main_base(main_key: str) -> str:
                    main_base = main_key
                    # For DataVectors
                    if main_key.startswith("DataVector_xAOD__TriggerMenuJson_v1__"):
                        main_base = main_key.replace(
                            "DataVector_xAOD__TriggerMenuJson_v1__", ""
                        )
                    elif main_key.startswith("DataVector_xAOD__TriggerMenu_v1__"):
                        main_base = main_key.replace(
                            "DataVector_xAOD__TriggerMenu_v1__", ""
                        )
                    # For FileMetaData
                    elif main_key.startswith("xAOD__FileMetaData_v1_"):
                        main_base = main_key.replace("xAOD__FileMetaData_v1_", "")
                    # For TruthMetaData
                    elif main_key.startswith("DataVector_xAOD__TruthMetaData_v1__"):
                        main_base = main_key.replace(
                            "DataVector_xAOD__TruthMetaData_v1__", ""
                        )
                    return main_base
 
                def _find_associated_pairs(auxes: dict) -> list[tuple[str, str]]:
                    # Build lookup tables
                    aux_map = {}
                    for k in auxes:
                        if "Aux" in k:
                            aux_map[_get_aux_base(k)] = k
 
                    main_map = {}
                    for k in auxes:
                        base = _get_main_base(k)
                        if base:
                            main_map[base] = k
 
                    # Find pairs
                    pairs = []
                    for base, aux_key in aux_map.items():
                        if base in main_map:
                            pairs.append((aux_key, main_map[base]))
                    return pairs
 
                def _keep_pair(
                    pair: tuple[str, str], auxes: dict, meta_filter: dict
                ) -> bool:
                    key = next(
                        (
                            k
                            for k, v in trigger_menu_json_map.items()
                            if v
                            == pair[1]
                            .replace("xAOD__", "xAOD::")
                            .replace("DataVector_", "DataVector<")
                            .replace("__Trigger", ">_Trigger")
                        ),
                        auxes[pair[0]],
                    )
 
                    try:
                        key = (
                            key.replace("xAOD__", "xAOD::")
                            if key.count("_") <= 1
                            else key.replace("xAOD__", "xAOD::").rsplit("_", 2)[0]
                        )
                    except IndexError:
                        pass
 
                    return should_keep_meta(
                        pair[0]
                        .replace("xAOD__", "xAOD::")
                        .replace("DataVector_", "DataVector<")
                        .replace("__Trigger", ">_Trigger")
                        .replace("Aux:", "Aux."),
                        key,
                        meta_filter,
                    )
 
                selected_pairs = [
                    pair
                    for pair in _find_associated_pairs(auxes)
                    if _keep_pair(pair, auxes, meta_filter)
                ]
                # Some transition AODs contain both the Run2 and Run3 trigger metadata formats.
                # We only wish to read the Run3 format if such a file is encountered.
                if any(
                    regexXAODTriggerMenuJsonAux.match(auxes[pair[0]])
                    for pair in selected_pairs
                ):
                    selected_pairs = [
                        pair
                        for pair in selected_pairs
                        if not regexXAODTriggerMenuAux.match(auxes[pair[0]])
                    ]
                # dynamic attributes are only used together with the FileMetaData
                read_dynamic_fmd = any(
                    "FileMetaData" in pair[0] for pair in selected_pairs
                )
 
                fields_to_read = {name for name, _ in selected_fields.values()}
                fields_to_read.update(name for pair in selected_pairs for name in pair)
                if read_dynamic_fmd:
                    fields_to_read.update(dynamic_fmd_fields.values())
 
                # only the fields present in the model are read from the file
                model = ROOT.RNTupleModel.Create()
                for field in descriptor.GetTopLevelFields():
                    if field.GetFieldName() in fields_to_read:
                        model.AddField(ROOT.std.move(field.CreateField(descriptor)))
 
                reader = ROOT.RNTupleReader.Open(ROOT.std.move(model), md)
                entry = reader.CreateEntry()
                reader.LoadEntry(0, entry)
 
                # read the EventStreamInfo first, so only the IOVs of its run are read later
                for normalizedName, (field_name, type_name) in sorted(
                    selected_fields.items(),
                    key=lambda item: regexEventStreamInfo.match(item[1][1]) is None,
                ):
                    try:
                        value = _convert_value(
                            _get_entry_value(entry, field_name, type_name)
                        )
                    except KeyError:
                        msg.warning(f"missing type {type_name}")
                        continue
                    meta_dict[filename][normalizedName] = value
                    if (
                        regexEventStreamInfo.match(type_name)
                        and value
                        and len(value.get("runNumbers", [])) == 1
                    ):
                        _gbl_run = value["runNumbers"][0]
                        msg.debug("Found RunNumber in EventStreamInfo: %d", _gbl_run)
 
                dynamic_fmd_items = {}
                if read_dynamic_fmd:
                    for name, field_name in dynamic_fmd_fields.items():
                        value = _convert_value(entry[field_name])
                        if name.startswith("is"):
                            # this is probably a boolean
                            value = value != "\x00"
                        dynamic_fmd_items[name] = value
 
                for pair in selected_pairs:
                    return_obj = _convert_value(
                        _get_entry_value(entry, pair[1], auxes[pair[1]]),
                        _get_entry_value(entry, pair[0], auxes[pair[0]]),
                    )
 
                    # both the Run3 TriggerMenuJson and the legacy Run2 TriggerMenu
                    if "TriggerMenu" in pair[0]:
                        if "RAWTriggerMenuJson" in return_obj:
                            key = (
                                pair[1]
                                if pair[0].startswith("Trigger")
                                else trigger_menu_json_map[pair[0]]
                            )
                            meta_dict[filename][key] = return_obj["RAWTriggerMenuJson"]
                            del return_obj["RAWTriggerMenuJson"]
                        if "TriggerConfigInfo" not in meta_dict[filename]:
                            meta_dict[filename]["TriggerConfigInfo"] = {}
                        if "dbkey" in return_obj:
                            meta_dict[filename]["TriggerConfigInfo"][
                                pair[0].split("_")[-1].replace("Aux:", "")
                            ] = {"key": return_obj["dbkey"], "name": return_obj["name"]}
                            del return_obj["dbkey"]
                            del return_obj["name"]
                        if "TriggerMenu" not in meta_dict[filename]:
                            meta_dict[filename]["TriggerMenu"] = {}
                        meta_dict[filename]["TriggerMenu"].update(return_obj)
                    elif "FileMetaData" in pair[0]:
                        if "FileMetaData" not in meta_dict[filename]:
                            meta_dict[filename]["FileMetaData"] = dynamic_fmd_items
                        meta_dict[filename]["FileMetaData"].update(return_obj)
                    elif "TruthMetaData" in pair[0]:
                        if pair == ("TruthMetaDataAux:", "TruthMetaData"):
                            if "TruthMetaData" not in meta_dict[filename]:
                                meta_dict[filename]["TruthMetaData"] = {}
                            meta_dict[filename]["TruthMetaData"].update(return_obj)
                        else:
                            # for backward compatibility
                            meta_dict[filename][
                                pair[1]
                                .replace("xAOD__", "xAOD::")
                                .replace("DataVector_", "DataVector<")
                                .replace("__Truth", ">_Truth")
                            ] = {}
                            meta_dict[filename][
                                pair[0]
                                .replace("xAOD__", "xAOD::")
                                .replace("Aux:", "Aux.")
                            ] = {}
                    elif pair == ("CutBookkeepersAux:", "CutBookkeepers"):
                        meta_dict[filename]["CutBookkeepers"] = return_obj
 
                msg.debug(f"Read metadata from RNTuple: {meta_dict[filename]}")
 
            else:
                # ----- read extra metadata required for 'lite' and 'full' modes ----------------------------------------#
                if mode != 'tiny':
                    # selecting from all tree the only one which contains metadata, respectively "MetaData"
                    metadata_tree = current_file.Get('MetaData')
                    # read all list of branches stored in "MetaData" tree
                    metadata_branches = metadata_tree.GetListOfBranches()
                    nr_of_branches = metadata_branches.GetEntriesFast()
 
                    # object to store the names of metadata containers and their corresponding class name.
                    meta_dict[filename]['metadata_items'] = {}
 
                    meta_filter = get_meta_filter(mode, meta_key_filter)
 
                    # store all persistent classes for metadata container existing in a POOL/ROOT file.
                    persistent_instances = {}
                    dynamic_fmd_items = {}
 
                    # Protect non-Gaudi environments from meta-data classes it doesn't know about
                    if not isGaudiEnv():
                        metadata_tree.SetBranchStatus("*", False)
 
                    for i in range(0, nr_of_branches):
                        branch = metadata_branches.At(i)
                        name = branch.GetName()
                        if name == 'index_ref':
                            # skip the index branch
                            continue
 
                        class_name = branch.GetClassName()
 
                        if regexIOVMetaDataContainer.match(class_name):
                            name = name.replace('IOVMetaDataContainer_p1_', '').replace('_', '/')
 
                        if regexIOVMetaDataContainer.match(class_name):
                            meta_dict[filename]['metadata_items'][name] = 'IOVMetaDataContainer'
                        elif regexByteStreamMetadataContainer.match(class_name):
                            meta_dict[filename]['metadata_items'][name] = 'ByteStreamMetadataContainer'
                        elif regexEventStreamInfo.match(class_name):
                            meta_dict[filename]['metadata_items'][name] = 'EventStreamInfo'
                        elif regexXAODFileMetaData.match(class_name):
                            meta_dict[filename]['metadata_items'][name] = 'FileMetaData'
                        elif regexXAODTruthMetaData.match(class_name):
                            meta_dict[filename]['metadata_items'][name] = 'TruthMetaData'
                        else:
                            type_name = class_name
                            if not type_name:
                                try:
                                    type_name = branch.GetListOfLeaves()[0].GetTypeName()
                                except IndexError:
                                    pass
                            meta_dict[filename]['metadata_items'][name] = type_name
 
                        if len(meta_filter) > 0:
                            keep = False
                            for filter_key, filter_class in meta_filter.items():
                                if (filter_key.replace('/', '_') in name.replace('/', '_') or filter_key == '*') and fnmatchcase(class_name, filter_class):
                                    if 'CutBookkeepers' in filter_key:
                                        keep = filter_key == name
                                        if keep:
                                            break
                                    else:
                                        keep = True
                                        break
 
                            if not keep:
                                continue
                        else:
                            # CutBookkeepers should always be filtered:
                            if 'CutBookkeepers' in name and name not in ['CutBookkeepers', 'CutBookkeepersAux.']:
                                continue
 
                        if not isGaudiEnv():
                            metadata_tree.SetBranchStatus(f"{name}*", True)
 
                        # assign the corresponding persistent class based of the name of the metadata container
                        if regexEventStreamInfo.match(class_name):
                            if class_name.endswith('_p1'):
                                persistent_instances[name] = ROOT.EventStreamInfo_p1()
                            elif class_name.endswith('_p2'):
                                persistent_instances[name] = ROOT.EventStreamInfo_p2()
                            else:
                                persistent_instances[name] = ROOT.EventStreamInfo_p3()
                        elif regexIOVMetaDataContainer.match(class_name):
                            persistent_instances[name] = ROOT.IOVMetaDataContainer_p1()
                        elif regexByteStreamMetadataContainer.match(class_name):
                            persistent_instances[name] = ROOT.ByteStreamMetadataContainer_p1()
                        elif regexXAODEventFormat.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.EventFormat_v1()
                        elif regexXAODTriggerMenu.match(class_name) and _check_project() not in ['AthGeneration']:
                            persistent_instances[name] = ROOT.xAOD.TriggerMenuContainer_v1()
                        elif regexXAODTriggerMenuAux.match(class_name) and _check_project() not in ['AthGeneration']:
                            persistent_instances[name] = ROOT.xAOD.TriggerMenuAuxContainer_v1()
                        elif regexXAODTriggerMenuJson.match(class_name) and _check_project() not in ['AthGeneration']:
                            persistent_instances[name] = ROOT.xAOD.TriggerMenuJsonContainer_v1()
                        elif regexXAODTriggerMenuJsonAux.match(class_name) and _check_project() not in ['AthGeneration']:
                            persistent_instances[name] = ROOT.xAOD.TriggerMenuJsonAuxContainer_v1()
                        elif regexXAODCutBookkeeperContainer.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.CutBookkeeperContainer_v1()
                        elif regexXAODCutBookkeeperContainerAux.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.CutBookkeeperAuxContainer_v1()
                        elif regexXAODFileMetaData.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.FileMetaData_v1()
                        elif regexXAODFileMetaDataAux.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.FileMetaDataAuxInfo_v1()
                        elif regexXAODTruthMetaData.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.TruthMetaDataContainer_v1()
                        elif regexXAODTruthMetaDataAux.match(class_name):
                            persistent_instances[name] = ROOT.xAOD.TruthMetaDataAuxContainer_v1()
 
                        if name in persistent_instances:
                            branch.SetAddress(ROOT.AddressOf(persistent_instances[name]))
 
                        # This creates a dict to store the dynamic attributes of the xAOD::FileMetaData
                        dynamicFMD = regexXAODFileMetaDataAuxDyn.match(name)
                        if dynamicFMD:
                            dynamicName = dynamicFMD.group().split('.')[-1]
                            dynamicType = regex_cppname.match(class_name)
                            if dynamicType:
                                # this should be a string
                                dynamic_fmd_items[dynamicName] = ROOT.std.string()
                                branch.SetAddress(ROOT.AddressOf(dynamic_fmd_items[dynamicName]))
                            else:
                                dynamic_fmd_items[dynamicName] = None
 
 
                    metadata_tree.GetEntry(0)
 
                    # This loads the dynamic attributes of the xAOD::FileMetaData from the TTree
                    for key in dynamic_fmd_items:
                        if dynamic_fmd_items[key] is None:
                            try:
                                if key.startswith("is"):
                                    # this is probably a boolean
                                    dynamic_fmd_items[key] = getattr(metadata_tree, key) != '\x00'
                                else:
                                    # this should be a float
                                    dynamic_fmd_items[key] = getattr(metadata_tree, key)
                            except AttributeError:
                                # should not happen, but just ignore missing attributes
                                pass
                        else:
                            # convert ROOT.std.string objects to python equivalent
                            dynamic_fmd_items[key] = str(dynamic_fmd_items[key])
 
                    # clean the meta-dict if the meta_key_filter flag is used, to return only the key of interest
                    if meta_key_filter:
                        meta_dict[filename] = {}
 
                    # Find the RunNumber from EventStreamInfo, so only the relevant IOVs are read later
                    for key, obj in persistent_instances.items():
                        if key.startswith('EventStreamInfo'):
                            data = _convert_value(obj, None)
                            if data and 'runNumbers' in data and len(data['runNumbers']) == 1:
                                _gbl_run = data['runNumbers'][0]
                                msg.debug("Found RunNumber in EventStreamInfo: %d", _gbl_run)
 
                    # read the metadata
                    for name, content in persistent_instances.items():
                        key = name
                        if hasattr(content, 'm_folderName'):
                            key = content.m_folderName
 
                        # Some transition AODs contain both the Run2 and Run3 metadata formats. We only wish to read the Run3 format if such a file is encountered.
                        has_r3_trig_meta = ('TriggerMenuJson_HLT' in persistent_instances or 'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLT' in persistent_instances)
                        aux = None
                        if key.startswith('TriggerMenuJson_') and not key.endswith('Aux.'): # interface container for the menu (AOD)
                            aux = persistent_instances[key+'Aux.']
                        elif key.startswith('DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_') and not key.endswith('Aux.'): # interface container for the menu (ESD)
                            menuPart = key.split('_')[-1]
                            aux = persistent_instances['xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_'+menuPart+'Aux.']    
                        elif key == 'TriggerMenu' and 'TriggerMenuAux.' in persistent_instances and not has_r3_trig_meta: # AOD case (legacy support, HLT and L1 menus)
                            aux = persistent_instances['TriggerMenuAux.']
                        elif key == 'DataVector<xAOD::TriggerMenu_v1>_TriggerMenu' and 'xAOD::TriggerMenuAuxContainer_v1_TriggerMenuAux.' in persistent_instances and not has_r3_trig_meta: # ESD case (legacy support, HLT and L1 menus)
                            aux = persistent_instances['xAOD::TriggerMenuAuxContainer_v1_TriggerMenuAux.']
                        elif (key == 'CutBookkeepers'
                            and 'CutBookkeepersAux.' in persistent_instances):
                            aux = persistent_instances['CutBookkeepersAux.']
                        elif key == 'CutBookkeepersAux.':
                            continue   # Extracted using the interface object
                        elif (key == 'FileMetaData'
                            and 'FileMetaDataAux.' in persistent_instances):
                            aux = persistent_instances['FileMetaDataAux.']
                        elif (key == 'xAOD::FileMetaData_v1_FileMetaData'
                            and 'xAOD::FileMetaDataAuxInfo_v1_FileMetaDataAux.' in persistent_instances):
                            aux = persistent_instances['xAOD::FileMetaDataAuxInfo_v1_FileMetaDataAux.']
                        elif (key == 'TruthMetaData'
                            and 'TruthMetaDataAux.' in persistent_instances):
                            aux = persistent_instances['TruthMetaDataAux.']
                        elif key == 'TruthMetaDataAux.':
                            continue   # Extracted using the interface object
                        elif 'Menu' in key and key.endswith('Aux.'):
                            continue   # Extracted using the interface object
 
                        return_obj = _convert_value(content, aux)
 
                        if 'TriggerMenuJson' in key  or ('TriggerMenu' in key and not has_r3_trig_meta):
                            if 'RAWTriggerMenuJson' in return_obj:
                                meta_dict[filename][key] = return_obj['RAWTriggerMenuJson']
                                del return_obj['RAWTriggerMenuJson']
                            if 'TriggerConfigInfo' not in meta_dict[filename]:
                                meta_dict[filename]['TriggerConfigInfo'] = {}
                            if 'dbkey' in return_obj:
                                meta_dict[filename]['TriggerConfigInfo'][key.split('_')[-1]] = {
                                    'key' : return_obj['dbkey'],
                                    'name': return_obj['name']
                                    }
                                del return_obj['dbkey']
                                del return_obj['name']
                            if 'TriggerMenu' not in meta_dict[filename]:
                                meta_dict[filename]['TriggerMenu'] = {}
                            meta_dict[filename]['TriggerMenu'].update(return_obj)
                        elif "FileMetaData" in key:
                            if "FileMetaData" not in meta_dict[filename]:
                                meta_dict[filename]["FileMetaData"] = dynamic_fmd_items
                            meta_dict[filename]["FileMetaData"].update(return_obj)
                        else:
                            meta_dict[filename][key] = return_obj
 
            try:
                # get the number of events from EventStreamInfo
                esi_dict = next(key for key, value in meta_dict[filename].items()
                                if isinstance(value, dict) and "numberOfEvents" in value and
                                meta_dict[filename]["metadata_items"][key] == "EventStreamInfo")
                msg.debug(f"{esi_dict=}")
                meta_dict[filename]["nentries"] = meta_dict[filename][esi_dict]["numberOfEvents"]
            except StopIteration as err:
                msg.debug(f"Caught {err=}, {type(err)=}, falling back on opening the DataHeader"
                          " Container to read the number of entries")
                meta_dict[filename]['nentries'] = dataheader_nentries(current_file)
                msg.debug(f"{meta_dict[filename]['nentries']=}")
 
            if unique_tag_info_values and mode=='iov':
                unique_tag_info_values = False
                msg.info('disabling "unique_tag_info_values" option for "iov" mode')
 
            # This is a required workaround which will temporarily be fixing ATEAM-560 originated from  ATEAM-531
            # ATEAM-560: https://its.cern.ch/jira/browse/ATEAM-560
            # ATEAM-531: https://its.cern.ch/jira/browse/ATEAM-531
            # This changes will remove all duplicates values presented in some files due
            # to the improper merging of two IOVMetaDataContainers.
            if unique_tag_info_values:
                msg.info('MetaReader is called with the parameter "unique_tag_info_values" set to True. '
                         'This is a workaround to remove all duplicate values from "/TagInfo" key')
                if '/TagInfo' in meta_dict[filename]:
                    for key, value in meta_dict[filename]['/TagInfo'].items():
                        if isinstance(value, list) and value:
                            if len(unique_values := set(value)) > 1:
                                msg.warning(
                                    f"Found multiple values for {key}: {value}. "
                                    "Looking for possible duplicates."
                                )
                                maybe_ok = False
                                if key == "AMITag":
                                    # curate duplicates like: ['s3681_q453', 's3681_q453_'] or ["s3681_q453", "q453_s3681"]
                                    unique_amitags = set()
                                    for amitags in unique_values:
                                        unique_amitags.add(
                                            "_".join([tag for tag in amitags.split("_") if tag])
                                        )
                                    # Remove tags that are parents - are contained in other tags
                                    # Such that ["s3681","s3681_d1485"] keeps only the latter
                                    parent_tags = []
                                    for atag in unique_amitags:
                                        if any(atag+'_' in x for x in unique_amitags if x != atag):
                                            parent_tags += [atag]
                                    for atag in parent_tags:
                                        # Do not remove the last tag!
                                        if len(unique_amitags)>1:
                                            msg.warning(f"Removing parent AMI tag {atag}")
                                            unique_amitags.remove(atag)
                                    if len(unique_amitags) == 1:
                                        maybe_ok = True
                                        # Make sure we keep the one we want to keep
                                        value.insert(0,list(unique_amitags)[0])
                                elif key == "beam_energy":
                                    # handle duplicates like: ['6500000', '6500000.0'] or [3, "3"]
                                    unique_energies = set()
                                    for energy in unique_values:
                                        try:
                                            energy = int(energy)
                                        except ValueError:
                                            try:
                                                energy = float(energy)
                                            except ValueError:
                                                pass
                                        unique_energies.add(energy)
                                    if len(unique_energies) == 1:
                                        maybe_ok = True
                                # randomSeed is maybe_ok due to mc16 corruption test failures (see ATLASG-3193)
                                elif key in ["AtlasRelease", "IOVDbGlobalTag", "AODFixVersion", "randomSeed"]:
                                    maybe_ok = True
                                if maybe_ok:
                                    msg.warning(
                                        f"Multiple values for {key} may mean the same, or "
                                        "the input file was produced in multi-step job. "
                                        f"Ignoring all but the first entry: {key} = {value[0]}"
                                    )
                                else:
                                    raise ValueError(
                                        f"{key} from /TagInfo contains more than 1 unique value: {value}"
                                    )
 
                            meta_dict[filename]['/TagInfo'][key] = value[0]
 
            if promote is None:
                promote = mode == 'lite' or mode == 'peeker'
 
            # Filter the data and create a prettier output for the 'lite' mode
            if mode == 'lite':
                meta_dict = make_lite(meta_dict)
 
            if mode == 'peeker':
                meta_dict = make_peeker(meta_dict)
 
            if promote:
                meta_dict = promote_keys(meta_dict, mode)
 
            # If AnalysisBase the itemList must be grabbed another way
            if not isGaudiEnv():
                if isinstance(collectionTree, ROOT.TTree):
                    item_list = [
                        (str(b.GetClassName()), str(b.GetName())) for b in collectionTree.GetListOfBranches()
                    ]
                    if normalize_type_names:
                        item_list = [
                            (type_name, _branch_name_to_field_name(name)) for type_name, name in item_list
                        ]
                elif isRNTuple(eventData := current_file.Get(PoolOpts.RNTupleNames.EventData)):
                    # keep the reader alive while iterating, as the descriptor is owned by it
                    eventDataReader = ROOT.RNTupleReader.Open(eventData)
                    # RNTuple fields use "Aux:"/"AuxDyn:" instead of "Aux."/"AuxDyn.",
                    # convert them to the branch names used for TTree above
                    item_list = [
                        (str(f.GetTypeName()), re.sub(r'Aux(Dyn)?:', r'Aux\1.', str(f.GetFieldName()), count=1))
                        for f in eventDataReader.GetDescriptor().GetTopLevelFields()
                        if not str(f.GetTypeName()).startswith('DataHeader')
                    ]
                    if normalize_type_names:
                        item_list = [
                            (_rntuple_type_to_class_name(type_name), name) for type_name, name in item_list
                        ]
                if normalize_type_names:
                    # the order of the TTree branches and RNTuple fields is not related
                    item_list.sort()
                meta_dict[filename]['itemList'] = item_list
 
        # ----- retrieves metadata from bytestream (BS) files (RAW, DRAW) ------------------------------------------#
        elif current_file_type == 'BS':
            
            if ignoreNonExistingLocalFiles and not regex_URI_scheme.match(filename) and not os.path.isfile(filename): 
                msg.warning('Ignoring not accessible file: {}'.format(filename))
                continue
            
            import eformat
 
            # store the number of entries
            bs = eformat.istream(filename)
            meta_dict[filename]['nentries'] = bs.total_events
 
            # store the 'guid' value
            data_reader = eformat.EventStorage.pickDataReader(filename)
            assert data_reader, 'problem picking a data reader for file [%s]' % filename
 
            # set auto flush equivalent, which for BS is always 1
            meta_dict[filename]['auto_flush'] = 1
 
            if hasattr(data_reader, 'GUID'):
                meta_dict[filename]['file_guid'] = data_reader.GUID()
 
            # compression level and algorithm, for BS always ZLIB
            meta_dict[filename]['file_comp_alg'] = 1
            meta_dict[filename]['file_comp_level'] = 1
 
 
            # if the flag full is set to true then grab all metadata
            # ------------------------------------------------------------------------------------------------------#
            if mode != "tiny":
                bs_metadata = {}
 
                for md in data_reader.freeMetaDataStrings():
                    if md.startswith('Event type:'):
                        k = 'eventTypes'
                        v = []
                        if 'is sim' in md:
                            v.append('IS_SIMULATION')
                        else:
                            v.append('IS_DATA')
 
                        if 'is atlas' in md:
                            v.append('IS_ATLAS')
                        else:
                            v.append('IS_TESTBEAM')
 
                        if 'is physics' in md:
                            v.append('IS_PHYSICS')
                        else:
                            v.append('IS_CALIBRATION')
 
                        bs_metadata[k] = tuple(v)
 
                    elif md.startswith('GeoAtlas:'):
                        k = 'geometry'
                        v = md.split('GeoAtlas:')[1].strip()
                        bs_metadata[k] = v
 
                    elif md.startswith('IOVDbGlobalTag:'):
                        k = 'conditions_tag'
                        v = md.split('IOVDbGlobalTag:')[1].strip()
                        bs_metadata[k] = v
 
                    elif '=' in md:
                        k, v = md.split('=', 1)  # Split on first '=' only
                        bs_metadata[k] = v
 
                bs_metadata['detectorMask'] = data_reader.detectorMask()
                bs_metadata['runNumbers'] = data_reader.runNumber()
                bs_metadata['lumiBlockNumbers'] = data_reader.lumiblockNumber()
                bs_metadata['projectTag'] = data_reader.projectTag()
                bs_metadata['stream'] = data_reader.stream()
                #bs_metadata['beamType'] = getattr(data_reader, 'beamType')()
                beamTypeNbr= data_reader.beamType()
                #According to info from Rainer and Guiseppe the beam type is
                #O: no beam
                #1: protons
                #2: ions
                if (beamTypeNbr==0): bs_metadata['beamType'] = 'cosmics'
                elif (beamTypeNbr==1 or beamTypeNbr==2):  bs_metadata['beamType'] = 'collisions'
                else: bs_metadata['beamType'] = 'unknown'
 
                bs_metadata['beamEnergy'] = data_reader.beamEnergy()
 
                meta_dict[filename]['eventTypes'] = bs_metadata.get('eventTypes', [])
                meta_dict[filename]['GeoAtlas'] = bs_metadata.get('geometry', None)
                meta_dict[filename]['conditions_tag'] = bs_metadata.get('conditions_tag', None)
                meta_dict[filename]['project_name'] = bs_metadata.get('projectTag', None)
 
                # Promote up one level
                meta_dict[filename]['detectorMask'] = [bs_metadata.get('detectorMask', None)]
                meta_dict[filename]['runNumbers'] = [bs_metadata.get('runNumbers', None)]
                meta_dict[filename]['lumiBlockNumbers'] = [bs_metadata.get('lumiBlockNumbers', None)]
                meta_dict[filename]['beam_type'] = bs_metadata.get('beamType', None)
                meta_dict[filename]['beam_energy'] = bs_metadata.get('beamEnergy', None)
                meta_dict[filename]['stream'] = bs_metadata.get('stream', None)
 
                if not data_reader.good():
                    # event-less file...
                    meta_dict[filename]['runNumbers'].append(bs_metadata.get('run_number', 0))
                    meta_dict[filename]['lumiBlockNumbers'].append(bs_metadata.get('LumiBlock', 0))
 
                msg.debug(f"{meta_dict[filename]=}")
                msg.debug(f"{len(bs)=}")
                if len(bs):
                    evt = bs[0]
                    try:
                        evt.check()
                        meta_dict[filename]['processingTags'] = [tag.name for tag in evt.stream_tag()]
                        meta_dict[filename]['evt_number'] = [evt.global_id()]
                        meta_dict[filename]['run_type'] = [eformat.helper.run_type2string(evt.run_type())]
                        # ATLASRECTS-7126: If there is no valid lumiblock information
                        # in the ByteStream header, get the info from the first event.
                        if meta_dict[filename]['lumiBlockNumbers'] == [0]:
                            msg.debug('Taking the luminosity block info from the first event (%i)', evt.lumi_block())
                            meta_dict[filename]['lumiBlockNumbers'] = [evt.lumi_block()]
                        # ATLASRECTS-7126: If there is no valid run number information
                        # in the ByteStream header, get the info from the first event.
                        if meta_dict[filename]['runNumbers'] == [0]:
                            msg.debug('Taking the run number info from the first event (%i)', evt.run_no())
                            meta_dict[filename]['runNumbers'] = [evt.run_no()]
                    except RuntimeError as err:
                        msg.error("Issue while reading the first event of BS file %r: %r", filename, err)
                    else:
                        msg.debug(f"{meta_dict[filename]=}")
                else:
                    msg.warning(f"Event-less BS {filename=}, will not read metadata information from the first event")
 
                # fix for ATEAM-122
                if len(bs_metadata.get('eventTypes', '')) == 0:  # see: ATMETADATA-6
                    evt_type = ['IS_DATA', 'IS_ATLAS']
                    if bs_metadata.get('stream', '').startswith('physics_'):
                        evt_type.append('IS_PHYSICS')
                    elif bs_metadata.get('stream', '').startswith('calibration_'):
                        evt_type.append('IS_CALIBRATION')
                    elif bs_metadata.get('projectTag', '').endswith('_calib'):
                        evt_type.append('IS_CALIBRATION')
                    else:
                        evt_type.append('Unknown')
 
                    meta_dict[filename]['eventTypes'] = evt_type
 
                if mode == 'full':
                    meta_dict[filename]['bs_metadata'] = bs_metadata
 
        # ------ Throw an error if the user provide other file types -------------------------------------------------#
        else:
            msg.error('Unknown filetype for {0} - there is no metadata interface for type {1}'.format(filename, current_file_type))
            return None
 
    return meta_dict
 
 
def _check_project():
    import os
    if 'AthSimulation_DIR' in os.environ:
        return 'AthSimulation'
    if 'AthGeneration_DIR' in os.environ:
        return 'AthGeneration'
    return 'Athena'
 
 
def _get_pfn(filename):
    """
    Extract the actual filename if LFN or PFN notation is used
    """
    pfx = filename[0:4]
    if pfx == 'PFN:':
        return filename[4:]
    if pfx == 'LFN:':
        import subprocess, os
        os.environ['POOL_OUTMSG_LEVEL'] = 'Error'
        output = subprocess.check_output(['FClistPFN','-l',filename[4:]],text=True).split('\n')
        if len(output) == 2:
            return output[0]
        msg.error( 'FClistPFN({0}) returned unexpected number of lines:'.format(filename) )
        msg.error( '\n'.join(output) )
    return filename
 
 
def _read_guid(filename):
    """
    Extracts the "guid" (Globally Unique Identifier) in POOL files and Grid catalogs) value from a POOL file.
    :param filename: the input file
    :return: the guid value, None if unavailable
    """
    import ROOT
    root_file = ROOT.TFile.Open( _get_pfn(filename) )
    params = root_file.Get('##Params')
    if not params:
        return
    if not isinstance(params, ROOT.TTree) and not isinstance(params, ROOT.RNTuple) and not isinstance(params, ROOT.TDirectory):
        raise NotImplementedError(f"Cannot extract GUID from object {params!r} of type {type(params)!r}")
 
    regex = re.compile(r'\‍[NAME=(\w+)\‍]\‍[VALUE=(.*)\‍]', re.ASCII)
    fid = None
 
    if isinstance(params, ROOT.TTree):
        for entry in params:
            param = entry.GetLeaf('db_string').GetValueString()
            result = regex.match(param)
            if result and result.group(1) == 'FID' :
                # don't exit yet, it's the last FID entry that counts
                fid = result.group(2)
    elif isinstance(params, ROOT.RNTuple):
        reader = ROOT.RNTupleReader.Open(params)
        try:
            entry = reader.CreateEntry()
        except AttributeError:
            entry = reader.GetModel().CreateEntry()
        for idx in range(reader.GetNEntries()):
            reader.LoadEntry(idx, entry)
            try:
                result = regex.match(str(entry['db_string']))
            except (AttributeError, TypeError) as err:
                # Early RNTuple implementation doesn't allow reading
                # strings on the python side, might be triggering it...
                msg.error(f"Cannot read FID from ##Params in RNTuple w/ ROOT error: {err}")
                return None
            if result and result.group(1) == 'FID' :
                # don't exit yet, it's the last FID entry that counts
                fid = result.group(2)
    elif isinstance(params, ROOT.TDirectory):
        for key in params.GetListOfKeys():
            param = params.Get(key.GetName())
            result = regex.match(str(param))
            if result and result.group(1) == 'FID' :
                # don't exit yet, it's the last FID entry that counts
                fid = result.group(2)
 
    return fid
 
 
def _get_entry_value(entry, field_name, type_name):
    """Get the value of a field from a loaded RNTuple entry.
 
    Indexing the entry instantiates the typed REntry::GetPtr<T>, which fails for
    some classes (e.g. DataVector<xAOD::TriggerMenu_v1> in AnalysisBase), so
    classes are retrieved type-erased and bound to their type by name instead.
    """
    import ROOT
 
    if not ROOT.TClass.GetClass(type_name):
        return entry[field_name]
    return ROOT.bind_object(entry._GetPtr["void"](field_name).get(), type_name)
 
 
def _rntuple_type_to_class_name(type_name):
    """Convert an RNTuple type name to the class name of the equivalent TTree branch.
 
    RNTuple uses the canonical type names (e.g. std::vector<std::int32_t>), while
    TTree branches report the ROOT class names (e.g. vector<int>). Fundamental types
    are stored as plain leaves in a TTree, which do not have a class name.
    """
    if type_name in _rntuple_fundamental_types:
        return ''
    type_name = re.sub(r'std::u?int(?:8|16|32|64)_t',
                       lambda m: _rntuple_integer_types[m.group(0)], type_name)
    type_name = type_name.replace('std::', '')
    # ROOT class names separate consecutive closing brackets with a space
    while '>>' in type_name:
        type_name = type_name.replace('>>', '> >')
    return type_name
 
 
def _branch_name_to_field_name(branch_name):
    """Convert a TTree branch name to the name of the equivalent RNTuple field.
 
    RNTuple field names have all non-alphanumeric characters replaced by underscores
    (e.g. L1DR-EM15TAU12I-J25 becomes L1DR_EM15TAU12I_J25). The "Aux."/"AuxDyn."
    separator of the TTree branch names is kept.
    """
    parts = re.split(r'(Aux(?:Dyn)?\.)', branch_name, maxsplit=1)
    return ''.join(part if i % 2 else re.sub(r'[^A-Za-z0-9]', '_', part)
                   for i, part in enumerate(parts))
 
 
def _extract_fields(obj):
    result = {}
 
    for meth in dir(obj):
        if not meth.startswith('_'):
            if meth.startswith('m_'):
 
                field_name = str(meth)[2:]
                field_value = getattr(obj, meth)
 
                result[field_name] = _convert_value(field_value)
 
    return result
 
 
def _extract_fields_bsmd(value):
    """One entry per bytestream file the metadata came from."""
    return [{'runNumber': int(md.m_runNumber),
             'lumiBlock': int(md.m_lumiBlock),
             'numEvents': int(md.m_numEvents),
             'guid': str(md.m_guid),
             'stream': str(md.m_stream),
             'project': str(md.m_project),
             'beamType': int(md.m_beamType),
             'beamEnergy': int(md.m_beamEnergy)} for md in value]
 
 
def _convert_value(value, aux = None):
    cl=value.__class__
    if hasattr(cl, '__cpp_name__'):
        result = regex_cppname.match(cl.__cpp_name__)
        if result:
            cpp_type = result.group(1)
            if cpp_type == 'vector' or cpp_type == 'std::vector':
                return [_convert_value(val) for val in value]
            elif cpp_type == 'set' or cpp_type == 'std::set':
                return {_convert_value(val) for val in value}
            elif cpp_type == 'pair' or cpp_type == 'std::pair':
                return _convert_value(value.first), _convert_value(value.second)
            
            # elif cpp_type == 'long':
            #   return int(value)
            
            elif cpp_type == 'string' or cpp_type == 'std::string':
                return str(value)
            
            elif cl.__cpp_name__ == "_Bit_reference":
                return bool(value)
 
            # special case which extracts data in a better format from IOVPayloadContainer_p1 class
            elif cl.__cpp_name__ == 'IOVMetaDataContainer_p1':
                return _extract_fields_iovmdc(value)
 
            elif cl.__cpp_name__ == 'IOVPayloadContainer_p1':
                if _gbl_mode == 'iov':
                    return _extract_iov_detailed(value)
                elif _gbl_run is not None:
                    r =  _extract_iov_for_run(value, _gbl_run)
                    if r is not None:
                        return r
                    # fall through to the default extraction if no IOV found for the run
                return _extract_fields_iov( value, range(value.m_attrIndexes.size()) )
 
            elif cl.__cpp_name__ == 'ByteStreamMetadataContainer_p1':
                return _extract_fields_bsmd(value)
 
            elif cl.__cpp_name__ == 'xAOD::EventFormat_v1':
                return _extract_fields_ef(value)
            elif cl.__cpp_name__ == 'xAOD::CutBookkeeperContainer_v1':
                return _extract_fields_cbk(interface=value, aux=aux)
            elif cl.__cpp_name__ == 'xAOD::FileMetaData_v1':
                return _extract_fields_fmd(interface=value, aux=aux)
            elif cl.__cpp_name__ == 'DataVector<xAOD::TruthMetaData_v1>':
                return _extract_fields_tmd(interface=value, aux=aux)
 
            elif cl.__cpp_name__ == 'DataVector<xAOD::TriggerMenu_v1>' :
                return _extract_fields_triggermenu(interface=value, aux=aux)
 
            elif cl.__cpp_name__ == 'DataVector<xAOD::TriggerMenuJson_v1>' :
                return _extract_fields_triggermenujson(interface=value, aux=aux)
 
            elif (cl.__cpp_name__ == 'EventStreamInfo_p1' or
                  cl.__cpp_name__ == 'EventStreamInfo_p2' or
                  cl.__cpp_name__ == 'EventStreamInfo_p3'):
                return _extract_fields_esi(value)
 
            elif (cl.__cpp_name__ == 'EventType_p1' or
                  cl.__cpp_name__ == 'EventType_p3'):
                fields = _extract_fields(value)
                fields = _convert_event_type_bitmask(fields)
                fields = _convert_event_type_user_type(fields)
                return fields
 
            elif regex_persistent_class.match(cl.__cpp_name__):
                return _extract_fields(value)
 
    return value
 
 
def _get_attribute_val(iov_container, attr_name, attr_idx):
    type_idx = attr_idx.typeIndex()
    obj_idx = attr_idx.objIndex()
 
    attr_value = None
 
    if type_idx == 0:
        attr_value = bool(iov_container.m_bool[obj_idx])
    elif type_idx == 1:
        attr_value = int(iov_container.m_char[obj_idx])
    elif type_idx == 2:
        attr_value = int(iov_container.m_unsignedChar[obj_idx])
    elif type_idx == 3:
        attr_value = int(iov_container.m_short[obj_idx])
    elif type_idx == 4:
        attr_value = int(iov_container.m_unsignedShort[obj_idx])
    elif type_idx == 5:
        attr_value = int(iov_container.m_int[obj_idx])
    elif type_idx == 6:
        attr_value = int(iov_container.m_unsignedInt[obj_idx])
    elif type_idx == 7:
        attr_value = int(iov_container.m_long[obj_idx])
    elif type_idx == 8:
        attr_value = int(iov_container.m_unsignedLong[obj_idx])
    elif type_idx == 9:
        attr_value = int(iov_container.m_longLong[obj_idx])
    elif type_idx == 10:
        attr_value = int(iov_container.m_unsignedLongLong[obj_idx])
    elif type_idx == 11:
        attr_value = float(iov_container.m_float[obj_idx])
    elif type_idx == 12:
        attr_value = float(iov_container.m_double[obj_idx])
    elif type_idx == 13:
        # skipping this type because is file IOVPayloadContainer_p1.h (line 120) is commented and not considered
        pass
    elif type_idx == 14:
        attr_value = str(iov_container.m_string[obj_idx])
        # Cleaning class name from value
        if attr_value.startswith('IOVMetaDataContainer_p1_'):
            attr_value = attr_value.replace('IOVMetaDataContainer_p1_', '')
        if attr_value.startswith('_'):
            attr_value = attr_value.replace('_', '/')
        # Now it is clean
    elif type_idx == 15:
        attr_value = int(iov_container.m_date[obj_idx])
    elif type_idx == 16:
        attr_value = int(iov_container.m_timeStamp[obj_idx])
    else:
        raise ValueError('Unknown type id {0} for attribute {1}'.format(type_idx, attr_name))
 
    return attr_value
 
 
def _extract_fields_iov( iov_container, idx_range ):
     result = {}
     for idx in idx_range:
         attr_idx = iov_container.m_attrIndexes[idx]
         name_idx = attr_idx.nameIndex()
         attr_name = iov_container.m_attrName[name_idx]
         attr_value = _get_attribute_val(iov_container, attr_name, attr_idx)
 
         if attr_name not in result:
             result[attr_name] = [attr_value]
         else:
             result[attr_name].append(attr_value)
 
     max_element_count = 0
     for content in result.values():
         if len(content) > max_element_count:
             max_element_count = len(content)
 
     if max_element_count <= 1:
         for name, content in result.items():
             if len(content) > 0:
                 result[name] = content[0]
             else:
                 result[name] = None
 
     return result
 
 
def _extract_iov_for_run(iov_container, run):
    def extract_payload(iov_container, payload):
        result = {}
        for attrList in payload.m_attrLists:
            r = _extract_fields_iov( iov_container, range(attrList.m_firstIndex, attrList.m_lastIndex) )
            for name, content in r.items():
                if name not in result:
                    result[name] = [content]
                elif content not in result[name]:
                    result[name].append(content)
        # turn single element lists into single values
        for name, content in result.items():
            if len(content) == 1:
                result[name] = content[0]
            elif len(content) == 0:
                # to be consistent with the old code
                result[name] = None
        return result
 
    for payload in iov_container.m_payloadVec:
        if run >= payload.m_start>>32 and run < payload.m_stop>>32:
            return extract_payload(iov_container, payload)
    # IOV for the requested run not found
    if len(iov_container.m_payloadVec) > 0:
        # print a warning only if there was actually some payload in the container
        msg.warning('No IOV found for run %d', run)
    return None
 
 
def _extract_iov_detailed(iov_container):
    def iovtostr(t):
        # break iov time into high and low halves (run number usually in the higher half)
        return "({h}:{l})".format(h=t>>32, l=t&(2**32-1))
 
    def extract_list_collection(iov_container, listCollection ):
        result = {}
        ln = 0
        for list in listCollection.m_attrLists:
            ln = ln + 1
            lname = 'List {ln}: iov=[{s} ,{e}];  Channel#={ch}'.format(
                ln=ln, s=iovtostr(list.m_range.m_start),
                e=iovtostr(list.m_range.m_stop),
                ch=list.m_channelNumber )
            result[ lname ] = _extract_fields_iov( iov_container, range(list.m_firstIndex, list.m_lastIndex) )
        return result
 
    result = {}
    pn = 0
    for listCollection in iov_container.m_payloadVec:
        pn = pn + 1
        pname = 'IOV range {n}: [{s}, {e}]'.format(n=pn, s=iovtostr(listCollection.m_start),
                                                   e=iovtostr(listCollection.m_stop))
        result[ pname ] = extract_list_collection(iov_container, listCollection )
    return result
 
 
def _extract_fields_iovmdc(value):
    return _convert_value(value.m_payload)
 
 
def _extract_fields_esi(value):
    result = {}
 
    result['eventTypes'] = []
    for eventType in value.m_eventTypes:
        result['eventTypes'].append(_convert_value(eventType))
 
    result['numberOfEvents'] = value.m_numberOfEvents
    result['runNumbers'] = list(value.m_runNumbers)
    result['lumiBlockNumbers'] = list(value.m_lumiBlockNumbers)
    result['processingTags'] = [str(v) for v in value.m_processingTags]
    result['itemList'] = []
 
    # Get the class name in the repository with CLID <clid>
    from CLIDComps.clidGenerator import clidGenerator
    cgen = clidGenerator("")
    for clid, sgkey in value.m_itemList:
        if isinstance(sgkey, bytes):
            sgkey = sgkey.decode()
        else:
            sgkey = str(sgkey)
        result['itemList'].append((cgen.getNameFromClid(clid), sgkey))
 
    return result
 
 
def _extract_fields_ef(value):
    result = {}
 
    for ef_element in value:
        result[ef_element.first] = ef_element.second.className()
 
    return result
 
 
def _extract_fields_cbk(interface=None, aux=None):
    """Extract CutBookkeeper content into dictionary
 
    This function takes the CutBookkeeperContainer_v1 and CutBookkeeperAuxContainer_v1 objects.
    It makes sure the the interface object uses the auxiliary object as store.
        Args:
            interface (CutBookkeeperContainer_v1):     the interface class
            aux       (CutBookkeeperAuxContainer_v1):  auxiliary container object
        Returns
            dict: with the cycle number and last stream
    """
    if not interface or not aux:
        return {}
    interface.setStore(aux)
 
    max_cycle = -1
    input_stream = ''
 
    for cbk in interface:
        current_cycle = int(cbk.cycle())
        if current_cycle > max_cycle:
            max_cycle = current_cycle
            input_stream = str(cbk.inputStream())
 
    result = {
        'currentCutCycle': max_cycle,
        'currentCutInputStream': input_stream,
    }
    return result
 
 
def _extract_fields_fmd(interface=None, aux=None):
    """Turn static FileMetaData content into dictionary
 
    This function takes the FileMetaData_v1 and FileMetaDataAuxInfo_v1 objects.
    It makes sure the the interface object uses the auxiliary object as store.
    Next the two static variables of FileMetaDataAuxInfo_v1 are retrieved and
    added to the dictionary that is returned.
        Args:
            interface (FileMetaData_v1):        the interface class
            aux       (FileMetaDataAuxInfo_v1): auxiliary container object
        Returns
            dict: with the production release and dataType
    """
    import ROOT
    if not interface or not aux:
        return {}
    interface.setStore(aux)
    metaContent = {
        "productionRelease": ROOT.std.string(),
        "dataType": ROOT.std.string(),
        "runNumbers": ROOT.std.vector('unsigned int')(),
        "lumiBlocks": ROOT.std.vector('unsigned int')(),
    }
    # Note: using this for dynamic attributes returns empty content
    for k, v in metaContent.items():
        try:
            interface.value(getattr(interface, k), v)
        except AttributeError:
            interface.value(k, v)
    # Now return python objects
    result = {k: str(v) for k, v in metaContent.items() if type(v) is ROOT.std.string}
    result.update({k: list(v) for k, v in metaContent.items() if type(v) is ROOT.std.vector('unsigned int')})
    return result
 
 
def _extract_fields_tmd(interface=None, aux=None):
    import ROOT
    BadAuxVarException = ROOT.SG.ExcBadAuxVar
    """Extract TruthMetaData content into dictionary
 
    This function takes the TruthMetaDataContainer_v1 and TruthMetaDataAuxContainer_v1 objects.
    It makes sure the the interface object uses the auxiliary object as store.
        Args:
            interface (TruthMetaDataContainer_v1):     the interface class
            aux       (TruthMetaDataAuxContainer_v1):  auxiliary container object
        Returns
            dict
    """
    if not interface or not aux:
        return {}
    interface.setStore(aux)
 
    # return the first as we do not really expect more than one
    result = {}
    for tmd in interface:
        result['mcChannelNumber'] = tmd.mcChannelNumber()
 
        try:
            result['weightNames'] = [str(v) for v in tmd.weightNames()]
        except BadAuxVarException:
            result['weightNames'] = []
 
        try:
            result['lhefGenerator'] = str(tmd.lhefGenerator())
        except BadAuxVarException:
            result['lhefGenerator'] = ''
    
        try:
            result['generators'] = str(tmd.generators())
        except BadAuxVarException:
            result['generators'] = ''
    
        try:
            result['evgenProcess'] = str(tmd.evgenProcess())
        except BadAuxVarException:
            result['evgenProcess'] = ''
    
        try:
            result['evgenTune'] = str(tmd.evgenTune())
        except BadAuxVarException:
            result['evgenTune'] = ''
 
        try:
            result['hadronizationModel'] = str(tmd.hadronizationModel())
        except BadAuxVarException:
            result['hadronizationModel'] = ''
 
        try:
            result['partonShowerModel'] = str(tmd.partonShowerModel())
        except BadAuxVarException:
            result['partonShowerModel'] = ''
 
        try:
            result['hardPDF'] = str(tmd.hardPDF())
        except BadAuxVarException:
            result['hardPDF'] = ''
    
        try:
            result['softPDF'] = str(tmd.softPDF())
        except BadAuxVarException:
            result['softPDF'] = ''
 
    return result
 
 
""" Note: Deprecated. Legacy support for Run 2 AODs produced in release 21 or in release 22 prior to April 2021
"""
def _extract_fields_triggermenu(interface, aux):
    if aux is None:
        return {}
 
    L1Items = []
    HLTChains = []
 
    try:
        interface.setStore( aux )
        if interface.size() > 0:
            # We make the assumption that the first stored SMK is
            # representative of all events in the input collection.
            firstMenu = interface.at(0)
            L1Items   = [ _convert_value(item) for item in firstMenu.itemNames() ]
            HLTChains = [ _convert_value(chain) for chain in firstMenu.chainNames() ]
    except Exception as err: # noqa: F841
        msg.warning('Problem reading xAOD::TriggerMenu:')
 
    result = {}
    result['L1Items'] = L1Items
    result['HLTChains'] = HLTChains
 
    return result
 
def _extract_fields_triggermenujson(interface, aux):
    result = {}
 
    try:
        interface.setStore( aux )
        if interface.size() > 0:
            # We make the assumption that the first stored SMK is
            # representative of all events in the input collection.
            firstMenu = interface.at(0)
            import json
            decoded = json.loads(firstMenu.payload())
            result['RAWTriggerMenuJson'] = firstMenu.payload()
            result['name'] = firstMenu.name()
            result['dbkey'] = firstMenu.key()
            if decoded['filetype'] == 'hltmenu':
                result['HLTChains'] = [ _convert_value(chain) for chain in decoded['chains'] ] 
            elif decoded['filetype'] == 'l1menu':
                result['L1Items'] = [ _convert_value(item) for item in decoded['items'] ] 
            elif decoded['filetype'] in ['bunchgroupset', 'hltprescale', 'l1prescale', 'hltmonitoringsummary']:
                return result
 
            else:
                msg.warning('Got an xAOD::TriggerMenuJson called {0} but only expecting hltmenu or l1menu'.format(decoded['filetype']))
                return {}
                
    except Exception as err: # noqa: F841
        msg.warning('Problem reading xAOD::TriggerMenuJson')
 
    return result
 
def _convert_event_type_user_type(value):
    if 'user_type' in value:
        items = value['user_type'].split('#')[3:]
        for i in range(0, len(items), 2):
            value[items[i]] = _convert_value(items[i+1])
    return value
 
def _convert_event_type_bitmask(value):
 
    types = None
    for key in value:
        if key == 'bit_mask':
            val = value[key]
 
            bitmask_length = len(val)
 
            is_simulation = False
            is_testbeam = False
            is_calibration = False
 
            if bitmask_length > 0:  # ROOT.EventType.IS_SIMULATION
                is_simulation = val[0]
 
            if bitmask_length > 1:  # ROOT.EventType.IS_TESTBEAM
                is_testbeam = val[1]
 
            if bitmask_length > 2:  # ROOT.EventType.IS_CALIBRATION:
                is_calibration = val[2]
 
            types = [
                'IS_SIMULATION' if is_simulation else 'IS_DATA',
                'IS_TESTBEAM' if is_testbeam else 'IS_ATLAS',
                'IS_CALIBRATION' if is_calibration else 'IS_PHYSICS'
            ]
 
    value['type'] = types
    return value
 
 
def make_lite(meta_dict):
    for filename, file_content in meta_dict.items():
        for key in file_content:
            if key in meta_dict[filename]['metadata_items'] and regexEventStreamInfo.match(meta_dict[filename]['metadata_items'][key]):
                for item in list(meta_dict[filename][key]):
                    if item not in lite_primary_keys_to_keep:
                        meta_dict[filename][key].pop(item)
 
        if '/TagInfo' in file_content:
            
 
            for item in list(meta_dict[filename]['/TagInfo']):
                if item not in lite_TagInfo_keys_to_keep:
                    meta_dict[filename]['/TagInfo'].pop(item)
    return meta_dict
 
 
def make_peeker(meta_dict):
    for filename, file_content in meta_dict.items():
        for key in file_content:
            if key in meta_dict[filename]['metadata_items'] and regexEventStreamInfo.match(meta_dict[filename]['metadata_items'][key]):
                keys_to_keep = [
                    'lumiBlockNumbers',
                    'runNumbers',
                    'mc_event_number',
                    'mc_channel_number',
                    'eventTypes',
                    'processingTags',
                    'itemList'
                ]
                for item in list(meta_dict[filename][key]):
                    if item not in keys_to_keep:
                        meta_dict[filename][key].pop(item)
 
        if '/TagInfo' in file_content:
            keys_to_keep = [
                'beam_energy',
                'beam_type',
                'GeoAtlas',
                'IOVDbGlobalTag',
                'AODFixVersion',
                'AMITag',
                'project_name',
                'triggerStreamOfFile',
                'AtlasRelease',
                'specialConfiguration',
                'mc_campaign',
                'hepmc_version',
                'generators',
                'keywords',
                'data_year',
            ]
            for item in list(meta_dict[filename]['/TagInfo']):
                if item not in keys_to_keep:
                    meta_dict[filename]['/TagInfo'].pop(item)
 
        if '/Simulation/Parameters' in file_content:
            keys_to_keep = [
                'G4Version',
                'TruthStrategy',
                'SimBarcodeOffset',
                'RegenerationIncrement',
                'TRTRangeCut',
                'SimulationFlavour',
                'Simulator',
                'PhysicsList',
                'SimulatedDetectors',
                'IsDataOverlay',
            ]
            for item in list(meta_dict[filename]['/Simulation/Parameters']):
                if item not in keys_to_keep:
                    meta_dict[filename]['/Simulation/Parameters'].pop(item)
 
        if '/Digitization/Parameters' in file_content:
            keys_to_keep = [
                'numberOfCollisions',
                'intraTrainBunchSpacing',
                'BeamIntensityPattern'
                'physicsList',
                'digiSteeringConf',
                'pileUp',
                'DigitizedDetectors',
            ]
            for item in list(meta_dict[filename]['/Digitization/Parameters']):
                if item not in keys_to_keep:
                    meta_dict[filename]['/Digitization/Parameters'].pop(item)
 
        if 'CutBookkeepers' in file_content:
            keys_to_keep = [
                'currentCutCycle',
                'currentCutInputStream',
            ]
            for item in list(meta_dict[filename]['CutBookkeepers']):
                if item not in keys_to_keep:
                    meta_dict[filename]['CutBookkeepers'].pop(item)
 
        if 'TruthMetaData' in file_content:
            keys_to_keep = [
                'mcChannelNumber',
                'weightNames',
            ]
            for item in list(meta_dict[filename]['TruthMetaData']):
                if item not in keys_to_keep:
                    meta_dict[filename]['TruthMetaData'].pop(item)
 
    return meta_dict
 
 
def promote_keys(meta_dict, mode):
    for filename, file_content in meta_dict.items():
        md = meta_dict[filename]
        for key in file_content:
            if key in md['metadata_items'] and regexEventStreamInfo.match(md['metadata_items'][key]):
                md.update(md[key])
 
                if 'eventTypes' in md and len(md['eventTypes']):
                    et = md['eventTypes'][0]
                    md['mc_event_number'] = et.get('mc_event_number', md['runNumbers'][0])
                    if 'mc_channel_number' in et:
                        md['mc_channel_number'] = et.get('mc_channel_number', None)
                    md['eventTypes'] = et['type']
 
                    # For very old files
                    if 'GeoAtlas' in et:
                        md['GeoAtlas']       = et.get('GeoAtlas', None)
                    if 'IOVDbGlobalTag' in et:
                        md['IOVDbGlobalTag'] = et.get('IOVDbGlobalTag', None)
 
                if 'lumiBlockNumbers' in md[key]:
                    md['lumiBlockNumbers'] = md[key]['lumiBlockNumbers']
                
                if 'processingTags' in md[key]:
                    md['processingTags'] = md[key]['processingTags']
 
                meta_dict[filename].pop(key)
                break
 
            if not isGaudiEnv() and key in md['metadata_items'] and 'FileMetaData' in key:
                if 'beamType' in md[key]:
                    md['beam_type'] = md[key]['beamType']
 
                if 'runNumbers' in md[key]:
                    md['runNumbers'] = md[key]['runNumbers']
 
                if 'mcProcID' in md[key]:
                    md['mc_channel_number'] = int(md[key]['mcProcID'])
 
                if 'mcCampaign' in md[key]:
                    md['mc_campaign'] = md[key]['mcCampaign']
 
                if 'dataYear' in md[key]:
                    md['data_year'] = int(md[key]['dataYear'])
 
                if 'lumiBlocks' in md[key]:
                    md['lumiBlockNumbers'] = md[key]['lumiBlocks']
                
                if mode == 'peeker' and 'amiTag' in md[key]:
                    md['AMITag'] = md[key]['amiTag']
 
                if 'beamEnergy' in md[key]:
                    md['beam_energy'] = int(md[key]['beamEnergy'])
 
                if 'geometryVersion' in md[key]:
                    md['GeoAtlas'] = md[key]['geometryVersion']
 
                if 'AODFixVersion' in md[key]:
                    md['AODFixVersion'] = md[key]['AODFixVersion']
 
                # EventType checks
                md['eventTypes'] = []
                if mode == 'peeker' and 'simFlavour' in md[key]:
                    md['SimulationFlavour'] = md[key]['simFlavour']
 
                if mode == 'peeker' and 'isDataOverlay' in md[key]:
                    md['IsDataOverlay'] = md[key]['isDataOverlay']
 
                if 'dataType' in md[key]:
                    md['processingTags'] = [md[key]['dataType']]
 
                if (
                    ('simFlavour' in md[key] and ('FullG4' in md[key]['simFlavour'] or 'ATLFAST' in md[key]['simFlavour']))
                    or 'DAOD_TRUTH' in md[key]['dataType']
                ):
                    md['eventTypes'].append('IS_SIMULATION')
                else:
                    md['eventTypes'].append('IS_DATA')
 
                if (
                    'GeoAtlas' in md and 'ATLAS' in md['GeoAtlas']
                    or 'DAOD_TRUTH' in md[key]['dataType']
                ):
                    md['eventTypes'].append('IS_ATLAS')
                    # this is probably safe to assume for all files used in AnalysisBase
                    md['eventTypes'].append('IS_PHYSICS')
                else:
                    md['eventTypes'].append('IS_TESTBEAM')
 
                if mode == 'peeker':
                    if 'productionRelease' in md[key]:
                        md['AtlasRelease'] = md[key]['productionRelease']
                    
                    if 'generatorsInfo' in md[key]:
                        md['generators']  = md[key]['generatorsInfo']
 
                if mode == 'lite':
                    meta_dict[filename].pop(key)
                break
 
        if '/TagInfo' in file_content:
            md.update(md['/TagInfo'])
            md.pop('/TagInfo')
 
        if '/Generation/Parameters' in file_content:
            md.update(md['/Generation/Parameters'])
            md.pop('/Generation/Parameters')
 
        if '/Simulation/Parameters' in file_content:
            md.update(md['/Simulation/Parameters'])
            md.pop('/Simulation/Parameters')
 
        if '/Digitization/Parameters' in file_content:
            md.update(md['/Digitization/Parameters'])
            md.pop('/Digitization/Parameters')
 
        if 'CutBookkeepers' in file_content:
            md.update(md['CutBookkeepers'])
            md.pop('CutBookkeepers')
 
    return meta_dict
 
 
def convert_itemList(metadata, layout):
    """
    This function will rearrange the itemList values to match the format of 'eventdata_items', 'eventdata_itemsList'
    or 'eventdata_itemsDic' generated with the legacy file peeker tool
    :param metadata: a dictionary obtained using read_metadata method.
                     The mode for read_metadata must be 'peeker of 'full'
    :param layout: the mode in which the data will be converted:
                * for 'eventdata_items' use: layout= None
                * for 'eventdata_itemsList' use: layout= '#join'
                * for 'eventdata_itemsDic' use: layout= 'dict'
    """
 
    # Find the itemsList:
    item_list = None
 
    if 'itemList' in metadata:
        item_list = metadata['itemList']
    else:
 
        current_key = None
 
        for key in metadata:
            if 'metadata_items' in metadata and key in metadata['metadata_items'] and metadata['metadata_items'][key] == 'EventStreamInfo_p3':
                current_key = key
                break
        if current_key is not None:
            item_list = metadata[current_key]['itemList']
 
    if item_list is not None:
 
        if layout is None:
            return item_list
 
        elif layout == '#join':
            return [k + '#' + v for k, v in item_list if k]
 
 
        elif layout == 'dict':
            from collections import defaultdict
            dic = defaultdict(list)
 
            for k, v in item_list:
                dic[k].append(v)
 
            return dict(dic)
 
 
def dataheader_nentries(infile):
    """Extract number of entries from DataHeader.
 
    infile  ROOT TFile object or filename string
    return  Number of entries as returned by DataHeader object in infile,
            None in absence of DataHeader object
    """
    import ROOT
    from PyUtils.PoolFile import PoolOpts
    if not isinstance(infile, ROOT.TFile):
        infile = ROOT.TFile.Open(infile)
 
    # with the canonical naming scheme the DataHeader is stored in the EventData RNTuple
    for name in (PoolOpts.TTreeNames.DataHeader, PoolOpts.RNTupleNames.DataHeader,
                 PoolOpts.RNTupleNames.EventData):
        obj = infile.Get(name)
        msg.debug(f"dataheader_nentries: {name=}, {obj=}, {type(obj)=}")
        if not obj:
            continue
        if isinstance(obj, ROOT.TTree):
            return obj.GetEntriesFast()
        else:
            # check early to avoid scary ROOT read errors
            if ROOT.gROOT.GetVersionInt() < 63100:
                raise RuntimeError("ROOT ver. 6.31/01 or greater needed to read RNTuple files")
            if isRNTuple(obj):
                return ROOT.RNTupleReader.Open(obj).GetNEntries()
            else:
                raise NotImplementedError(f"Keys of type {type(obj)!r} not supported")
 
def get_meta_filter(mode="lite", meta_key_filter=None) -> dict:
    """Return a dictionary of metadata filters based on the mode and
    optional meta_key_filter.
    """
 
    if meta_key_filter is None:
        meta_key_filter = []
 
    # create a container for the list of filters used for the lite version
    meta_filter = {}
 
    # set the filters for name
    if mode == 'lite':
        if isGaudiEnv():
            meta_filter = {
                '/TagInfo': 'IOVMetaDataContainer_p1',
                'IOVMetaDataContainer_p1__TagInfo': 'IOVMetaDataContainer_p1',
                '*': 'EventStreamInfo_p*'
            }
        else:
            meta_filter = {
                'FileMetaData': '*',
                'FileMetaDataAux.': 'xAOD::FileMetaDataAuxInfo_v1',
            }
 
    # set the filters for name
    if mode == 'peeker':
        meta_filter.update({
            'TriggerMenu': 'DataVector<xAOD::TriggerMenu_v1>', # R2 trigger metadata format AOD (deprecated)
            'TriggerMenuAux.': 'xAOD::TriggerMenuAuxContainer_v1',
            'DataVector<xAOD::TriggerMenu_v1>_TriggerMenu': 'DataVector<xAOD::TriggerMenu_v1>', # R2 trigger metadata format ESD (deprecated)
            'xAOD::TriggerMenuAuxContainer_v1_TriggerMenuAux.': 'xAOD::TriggerMenuAuxContainer_v1',
            'TriggerMenuJson_HLT': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format AOD
            'TriggerMenuJson_HLTAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'TriggerMenuJson_HLTMonitoring': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format AOD
            'TriggerMenuJson_HLTMonitoringAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'TriggerMenuJson_HLTPS': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format AOD
            'TriggerMenuJson_HLTPSAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'TriggerMenuJson_L1': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format AOD
            'TriggerMenuJson_L1Aux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'TriggerMenuJson_L1PS': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format AOD
            'TriggerMenuJson_L1PSAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'CutBookkeepers': 'xAOD::CutBookkeeperContainer_v1',
            'CutBookkeepersAux.': 'xAOD::CutBookkeeperAuxContainer_v1',
            'FileMetaData': '*',
            'FileMetaDataAux.': 'xAOD::FileMetaDataAuxInfo_v1',
            'TruthMetaData': '*',
            'TruthMetaDataAux.': 'xAOD::TruthMetaDataAuxContainer_v1',
            'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLT': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format ESD
            'xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTMonitoring': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format ESD
            'xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTMonitoringAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_HLTPS': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format ESD
            'xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_HLTPSAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format ESD
            'xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_L1Aux.': 'xAOD::TriggerMenuJsonAuxContainer_v1',
            'DataVector<xAOD::TriggerMenuJson_v1>_TriggerMenuJson_L1PS': 'DataVector<xAOD::TriggerMenuJson_v1>', # R3 trigger metadata format ESD
            'xAOD::TriggerMenuJsonAuxContainer_v1_TriggerMenuJson_L1PSAux.': 'xAOD::TriggerMenuJsonAuxContainer_v1'
        })
 
        if isGaudiEnv():
            meta_filter.update({
                '/TagInfo': 'IOVMetaDataContainer_p1',
                'IOVMetaDataContainer_p1__TagInfo': 'IOVMetaDataContainer_p1',
                '/Simulation/Parameters': 'IOVMetaDataContainer_p1',
                '/Digitization/Parameters': 'IOVMetaDataContainer_p1',
                '/EXT/DCS/MAGNETS/SENSORDATA': 'IOVMetaDataContainer_p1',
                '*': 'EventStreamInfo_p*'
            })
 
    if (mode == 'full' or mode == 'iov') and meta_key_filter:
        meta_filter = {f: '*' for f in meta_key_filter}
 
    return meta_filter
 
def denormalize_metadata_types(metadata_dict):
    """
    Convert canonical/C++ STL types in the metadata_items dictionary back to their
    ROOT equivalents for backward compatibility.
    - 'float' => 'Float_t'
    - 'char' => 'Char_t'
    - 'std::string' => 'string'
    - 'xAOD::FileMetaData_v1' => 'FileMetaData'
    - 'xAOD::FileMetaDataAuxInfo_v1' => 'FileMetaDataAux'
    - template types, e.g. 'std::vector<std::uint32_t>' => 'vector<unsigned int>'
    (add more as needed)
    """
    type_map = {
        "float": "Float_t",
        "char": "Char_t",
        "std::string": "string",
        "std::uint32_t": "UInt_t",
        "xAOD::FileMetaData_v1": "FileMetaData",
    }
    denormalized = {}
    for k, v in metadata_dict.items():
        new_v = v
        for old, new in type_map.items():
            if new_v == old:
                new_v = new
            elif new_v.endswith("." + old):
                new_v = new_v.rsplit(".", 1)[0] + "." + new
        if "<" in new_v:
            new_v = _rntuple_type_to_class_name(new_v)
        denormalized[k] = new_v
    return denormalized
 
 
def should_keep_meta(normalizedName, typeName, meta_filter):
    """
    Helper function to determine if metadata should be kept based on meta_filter.
    """
    if len(meta_filter) == 0:
        return True
 
    for filter_key, filter_class in meta_filter.items():
        if (
            filter_key.replace("/", "_") in normalizedName.replace("/", "_")
            or filter_key == "*"
        ) and fnmatchcase(typeName, filter_class):
            if "CutBookkeepers" in filter_key:
                keep = filter_key == normalizedName
                if keep:
                    return True
            else:
                return True
    return False