8#ifndef COLUMNAR_TEST_FIXTURES_COLUMN_READER_BRANCH_H
9#define COLUMNAR_TEST_FIXTURES_COLUMN_READER_BRANCH_H
13#include <AthLinks/ElementLink.h>
44 std::cout <<
"found unknown keys for " <<
m_columnName <<
":";
47 std::cout <<
" " << std::hex << key << std::dec <<
" (allowed targets:";
48 for (
auto index : allowedSet)
55 [[nodiscard]] std::vector<std::string>
connect (
const ColumnInfo& columnInfo,
const std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& offsetColumns,
const std::unordered_map<std::string,ColumnInfo>& requestedColumns)
58 std::vector<std::string> keyColumnNames;
64 for (
auto& [requestedName, requestedInfo] : requestedColumns)
66 if (requestedInfo.keyColumnForVariantLink ==
m_columnName)
68 keyColumnNames.push_back (requestedName);
70 for (
const auto& targetName : requestedInfo.variantLinkTargetNames)
75 throw std::runtime_error (
"no key column found for variant link: " +
m_columnName);
77 return keyColumnNames;
90 if (eventIndex + 1 >= targetOffsetColumn.size())
91 throw std::runtime_error (
"target offset column not yet filled for: " +
m_targetNames.at(i) +
" when checking link column " +
m_columnName);
114 if (linkIndex == 0 && linkKey == 0)
120 unsigned targetIndex = 0u;
140 allowedSet.insert(i);
144 for (
auto iter = allowedSet.begin(); iter != allowedSet.end();)
148 if (eventIndex + 1 >= targetOffsetColumn.size())
149 throw std::runtime_error (
"target offset column not yet filled for: " +
m_targetNames.at(
index));
150 if (targetOffsetColumn.at(eventIndex) + linkIndex >= targetOffsetColumn.at(eventIndex + 1))
151 iter = allowedSet.erase(iter);
166 targetIndex = *allowedSet.begin();
169 std::ostringstream
error;
170 error <<
"target key mismatch: read sgkey " << std::hex << linkKey << std::dec;
177 error <<
", alternate key found for non-variant link:";
181 error <<
" " << std::hex << key << std::dec;
187 throw std::runtime_error (std::move (
error).
str());
201 if (eventIndex + 1 >= targetOffsetColumn.size())
202 throw std::runtime_error (
"target offset column not yet filled for: " +
m_targetNames.at(targetIndex));
203 auto myLinkIndex = linkIndex + targetOffsetColumn.at(eventIndex);
204 if (myLinkIndex >= targetOffsetColumn.at(eventIndex + 1))
205 throw std::runtime_error (
"index out of range for link: " +
m_columnName +
" with element index " + std::to_string(linkIndex) +
" targeting " +
m_targetNames.at(targetIndex) +
" with offset " + std::to_string(targetOffsetColumn.at(eventIndex)) +
" and next offset " + std::to_string(targetOffsetColumn.at(eventIndex + 1)));
223 [[nodiscard]]
const std::vector<typename CM::LinkKeyType>&
keysColumn (std::size_t
index)
const
250 void addTarget (
const std::string& name,
const std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& offsetColumns, std::uint32_t clid = 0)
252 unsigned targetIndex = 0;
258 if (
auto offsetIter = offsetColumns.find (name); offsetIter != offsetColumns.end())
261 throw std::runtime_error (
"missing offset column: " + name);
276 template <
typename T,
template <
typename>
class Reader>
291 virtual bool connect(
Backend source, std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& ,std::unordered_map<std::string, ColumnInfo>& requestedColumns)
override
293 auto iter = requestedColumns.find (
outputColumns.at(0).name);
294 if (iter == requestedColumns.end())
297 requestedColumns.erase (iter);
314 outData.push_back (branchData);
328 result.timeRead =
benchmark.getEntryTime(emptyTime);
333 result.uncompressedSize =
branchReader.uncompressedSize();
345 template <
typename T,
template <
typename>
class Reader>
350 std::vector<ColumnarOffsetType>
offsets = {0};
363 virtual bool connect(
Backend source, std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& offsetColumns, std::unordered_map<std::string, ColumnInfo>& requestedColumns)
override
365 auto iter = requestedColumns.find (
outputColumns.at(0).name);
366 if (iter == requestedColumns.end())
373 throw std::runtime_error (
"offset name mismatch: " + iter->second.offsetName +
" != " +
outputColumns.at(1).name);
375 requestedColumns.erase (iter);
377 if (
auto offsetIter = offsetColumns.find (
outputColumns.at(1).name); offsetIter != offsetColumns.end())
383 if (iter != requestedColumns.end())
385 requestedColumns.erase (iter);
405 outData.insert (
outData.end(), branchData.begin(), branchData.end());
419 throw std::runtime_error (
"offset column not filled yet: " +
outputColumns.at(1).name);
421 throw std::runtime_error (
"offset column does not match: " +
outputColumns.at(1).name);
429 result.timeRead =
benchmark.getEntryTime(emptyTime);
434 result.uncompressedSize =
branchReader.uncompressedSize();
446 template <
typename T,
template <
typename>
class Reader>
450 std::vector<ColumnarOffsetType>
offsets = {0};
463 virtual bool connect(
Backend source, std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& , std::unordered_map<std::string, ColumnInfo>& requestedColumns)
override
465 auto iter = requestedColumns.find (
outputColumns.at(0).name);
466 if (iter == requestedColumns.end())
473 throw std::runtime_error (
"offset name mismatch: " + iter->second.offsetName +
" != " +
outputColumns.at(1).name);
475 requestedColumns.erase (iter);
478 if (iter == requestedColumns.end())
480 requestedColumns.erase (iter);
498 for (
auto& data : branchData)
518 result.timeRead =
benchmark.getEntryTime(emptyTime);
523 result.uncompressedSize =
branchReader.uncompressedSize();
535 template <
typename T,
template <
typename>
class Reader>
540 std::vector<ColumnarOffsetType>
offsets = {0};
555 virtual bool connect(
Backend source, std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& offsetColumns, std::unordered_map<std::string, ColumnInfo>& requestedColumns)
override
557 auto iter = requestedColumns.find (
outputColumns.at(0).name);
558 if (iter == requestedColumns.end())
565 throw std::runtime_error (
"offset name mismatch: " + iter->second.offsetName +
" != " +
outputColumns.at(1).name);
566 for (
const auto & keyColumn :
columnData.connect (iter->second, offsetColumns, requestedColumns))
568 outputColumns.push_back ({.name = keyColumn, .primary =
false, .enabled =
true});
569 requestedColumns.erase (keyColumn);
572 requestedColumns.erase (iter);
575 if (iter == requestedColumns.end())
577 requestedColumns.erase (iter);
598 for (
auto& data : branchData)
600 for (
auto& element : data)
627 result.timeRead =
benchmark.getEntryTime(emptyTime);
632 result.uncompressedSize =
branchReader.uncompressedSize();
650 template <
typename T,
template <
typename>
class Reader>
670 virtual bool connect(
Backend source, std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& , std::unordered_map<std::string, ColumnInfo>& requestedColumns)
override
672 auto iter = requestedColumns.find (
outputColumns.at(0).name);
673 if (iter == requestedColumns.end())
680 throw std::runtime_error (
"offset name mismatch: " + iter->second.offsetName +
" != " +
outputColumns.at(1).name);
682 requestedColumns.erase (iter);
685 if (iter == requestedColumns.end())
690 throw std::runtime_error (
"offset name mismatch: " + iter->second.offsetName +
" != " +
outputColumns.at(2).name);
692 requestedColumns.erase (iter);
695 if (iter == requestedColumns.end())
698 requestedColumns.erase (iter);
717 for (
auto& outerData : branchData)
719 for (
auto& innerData : outerData)
743 result.timeRead =
benchmark.getEntryTime(emptyTime);
748 result.uncompressedSize =
branchReader.uncompressedSize();
760 template <
typename T,
template <
typename>
class Reader>
766 std::vector<ColumnarOffsetType>
offsets = {0};
781 virtual bool connect(
Backend source, std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& offsetColumns, std::unordered_map<std::string, ColumnInfo>& requestedColumns)
override
783 auto iter = requestedColumns.find (
outputColumns.at(0).name);
784 if (iter == requestedColumns.end())
791 throw std::runtime_error (
"offset name mismatch: " + iter->second.offsetName +
" != " +
outputColumns.at(1).name);
792 for (
const auto & keyColumn :
columnData.connect (iter->second, offsetColumns, requestedColumns))
794 outputColumns.push_back ({.name = keyColumn, .primary =
false, .enabled =
true});
795 requestedColumns.erase (keyColumn);
798 requestedColumns.erase (iter);
800 if (
auto offsetIter = offsetColumns.find (
outputColumns.at(1).name); offsetIter != offsetColumns.end())
806 if (iter != requestedColumns.end())
809 requestedColumns.erase (iter);
829 for (
auto& element : branchData)
835 throw std::runtime_error (
"offset column not filled yet: " +
outputColumns.at(1).name);
837 throw std::runtime_error (
"offset column does not match: " +
outputColumns.at(1).name);
859 result.timeRead =
benchmark.getEntryTime(emptyTime);
864 result.uncompressedSize =
branchReader.uncompressedSize();
882 template <
typename T,
template <
typename>
class Reader>
888 std::vector<ColumnarOffsetType>
offsets = {0};
902 virtual bool connect(
Backend source, std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& offsetColumns, std::unordered_map<std::string, ColumnInfo>& requestedColumns)
override
905 if (iter == requestedColumns.end())
911 throw std::runtime_error (
"offset name mismatch: " + iter->second.offsetName +
" != " +
outputColumns.at(1).name);
913 for (
const auto & keyColumn :
columnData.connect (iter->second, offsetColumns, requestedColumns))
915 outputColumns.push_back ({.name = keyColumn, .primary =
false, .enabled =
true});
916 requestedColumns.erase (keyColumn);
918 requestedColumns.erase (iter);
920 if (
auto offsetIter = offsetColumns.find (
outputColumns.at(1).name); offsetIter != offsetColumns.end())
926 if (iter != requestedColumns.end())
929 requestedColumns.erase (iter);
950 for (
const auto& element : branchData)
952 if (element.isDefault() || element.index() ==
static_cast<unsigned int>(-1))
963 throw std::runtime_error(
"offset column not filled yet: " +
outputColumns.at(1).name);
967 throw std::runtime_error(
"offset column does not match: " +
outputColumns.at(1).name);
991 result.timeRead =
benchmark.getEntryTime(emptyTime);
996 result.uncompressedSize =
branchReader.uncompressedSize();
1014 template <
typename T,
template <
typename>
class Reader>
1034 virtual bool connect(
Backend source, std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& offsetColumns, std::unordered_map<std::string, ColumnInfo>& requestedColumns)
override
1036 auto iter = requestedColumns.find (
outputColumns.at(0).name);
1037 if (iter == requestedColumns.end())
1044 throw std::runtime_error (
"offset name mismatch: " + iter->second.offsetName +
" != " +
outputColumns.at(1).name);
1046 for (
const auto & keyColumn :
columnData.connect (iter->second, offsetColumns, requestedColumns))
1048 outputColumns.push_back ({.name = keyColumn, .primary =
false, .enabled =
true});
1049 requestedColumns.erase (keyColumn);
1052 requestedColumns.erase (iter);
1055 if (iter != requestedColumns.end())
1058 requestedColumns.erase (iter);
1062 if (iter != requestedColumns.end())
1065 requestedColumns.erase (iter);
1082 const auto& branchData =
branchReader.getEntry (entry);
1086 for (
auto& data : branchData)
1088 for (
auto& element : data)
1115 result.timeRead =
benchmark.getEntryTime(emptyTime);
1120 result.uncompressedSize =
branchReader.uncompressedSize();
1138 template <
template <
typename>
class Reader>
1156 virtual bool connect(
Backend source, std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& , std::unordered_map<std::string, ColumnInfo>& requestedColumns)
override
1158 auto iter = requestedColumns.find (
outputColumns.at(0).name);
1159 if (iter == requestedColumns.end())
1166 throw std::runtime_error (
"offset name mismatch: " + iter->second.offsetName +
" != " +
outputColumns.at(1).name);
1168 requestedColumns.erase (iter);
1171 if (iter == requestedColumns.end())
1176 requestedColumns.erase (iter);
1179 if (iter != requestedColumns.end())
1182 requestedColumns.erase (iter);
1198 const auto& branchData =
branchReader.getEntry (entry);
1201 for (
auto& data : branchData)
1224 result.timeRead =
benchmark.getEntryTime(emptyTime);
1229 result.uncompressedSize =
branchReader.uncompressedSize();
1238 template <
template <
typename>
class Reader>
1255 virtual bool connect(
Backend source, std::unordered_map<std::string,
const std::vector<ColumnarOffsetType>*>& , std::unordered_map<std::string, ColumnInfo>& requestedColumns)
override
1257 auto iter = requestedColumns.find (
outputColumns.at(0).name);
1258 if (iter == requestedColumns.end())
1264 throw std::runtime_error (
"offset name mismatch: " + iter->second.offsetName +
" != " +
outputColumns.at(1).name);
1266 requestedColumns.erase (iter);
1269 if (iter == requestedColumns.end())
1274 requestedColumns.erase (iter);
1288 const auto& branchData =
branchReader.getEntry (entry);
1291 for (
auto data : branchData)
1293 columnData.push_back (data->samplingPattern());
1310 result.name =
branchReader.columnName() +
"(fallback)";
1311 result.timeRead =
benchmark.getEntryTime(emptyTime);
1316 result.uncompressedSize =
branchReader.uncompressedSize();
bool isDefault() const
Test to see if this link is in the default state.
ElementLink implementation for ROOT usage.
index_type index() const
Get the index of the element inside of its container.
sgkey_t key() const
Get the key that we reference, as a hash.
a class that holds the columnar data for a single call
void setColumn(std::size_t columnIndex, std::size_t size, CT *dataPtr)
set the data for the given column
static constexpr std::size_t nullIndex
the index used for an invalid index (always has to be 0)
this is a simple benchmarking helper class wrapping timers from std::chrono
std::vector< OutputColumnInfo > outputColumns
void checkOffsets(unsigned eventIndex)
std::vector< SG::sgkey_t > m_targetKeys
std::vector< typename CM::LinkIndexType > m_columnData
const CM::LinkIndexType * data() const noexcept
std::vector< std::string > m_targetNames
void addLink(const ElementLink< T > &element, unsigned eventIndex)
std::vector< const std::vector< ColumnarOffsetType > * > m_targetOffsetColumns
void addTarget(const std::string &name, const std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &offsetColumns, std::uint32_t clid=0)
~LinkColumnVector() noexcept
std::vector< std::vector< typename CM::LinkKeyType > > m_keysColumns
void addSplitLink(std::size_t linkIndex, SG::sgkey_t linkKey, unsigned eventIndex)
auto begin() const noexcept
std::unordered_map< SG::sgkey_t, std::unordered_set< std::size_t > > m_unknownKeysAllowedTargets
auto end() const noexcept
std::size_t size() const noexcept
const std::vector< typename CM::LinkKeyType > & keysColumn(std::size_t index) const
std::vector< std::string > connect(const ColumnInfo &columnInfo, const std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &offsetColumns, const std::unordered_map< std::string, ColumnInfo > &requestedColumns)
uint32_t sgkey_t
Type used for hashed StoreGate key+CLID pairs.
std::variant< TTree *, RNTupleBackend * > Backend
const std::unordered_map< std::string, SG::sgkey_t > knownSgKeys
lookup table from container name to its sgkey hash
SG::sgkey_t computeSgKey(const std::string &name, std::uint32_t clid)
compute the StoreGate hashed key for a container
constexpr ColumnarOffsetType invalidObjectIndex
the value for an invalid element index
a struct that contains meta-information about each column that's needed to interface the column with ...
std::string soleLinkTargetName
for simple link columns: the name of the target container
std::uint32_t soleLinkTargetClid
for simple link columns: the CLID of the target container
std::string name
the name of the column
std::size_t LinkIndexType
the type used for columns that represent element links
static LinkIndexType mergeLinkKeyIndex(LinkIndexType key, LinkIndexType index)
merge a key and index value into a link value
the performance data for reading a single branch/column
virtual void getEntry(Long64_t entry) override
virtual void setData(ColumnVectorData &colData) override
virtual void clearColumns() override
std::vector< ColumnarOffsetType > offsets
std::vector< char > columnData
Reader< std::vector< std::string > > branchReader
std::vector< std::size_t > columnHashData
virtual BranchPerfData getPerfData(float emptyTime) override
virtual bool connect(Backend source, std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &, std::unordered_map< std::string, ColumnInfo > &requestedColumns) override
virtual void collectColumnData() override
Benchmark benchmarkUnpack
ColumnReaderMetNames(const std::string &val_branchName)
virtual void clearColumns() override
virtual BranchPerfData getPerfData(float emptyTime) override
virtual void setData(ColumnVectorData &colData) override
Benchmark benchmarkUnpack
ColumnReaderSamplingPattern(const std::string &val_branchName)
Reader< xAOD::CaloClusterContainer > branchReader
std::vector< std::uint32_t > columnData
std::vector< ColumnarOffsetType > offsets
virtual void collectColumnData() override
virtual void getEntry(Long64_t entry) override
virtual bool connect(Backend source, std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &, std::unordered_map< std::string, ColumnInfo > &requestedColumns) override
virtual void setData(ColumnVectorData &columnData) override
virtual void collectColumnData() override
virtual void clearColumns() override
virtual void getEntry(Long64_t entry) override
virtual BranchPerfData getPerfData(float emptyTime) override
ColumnReaderScalar(const std::string &val_branchName)
Benchmark benchmarkUnpack
virtual bool connect(Backend source, std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &, std::unordered_map< std::string, ColumnInfo > &requestedColumns) override
std::vector< ColumnarOffsetType > offsets
const std::vector< ColumnarOffsetType > * offsetColumn
Reader< std::vector< ElementLink< T > > > branchReader
virtual void collectColumnData() override
Benchmark benchmarkUnpack
virtual void clearColumns() override
virtual void setData(ColumnVectorData &colData) override
LinkColumnVector columnData
virtual BranchPerfData getPerfData(float emptyTime) override
ColumnReaderVectorLink(const std::string &val_branchName)
virtual void getEntry(Long64_t entry) override
virtual bool connect(Backend source, std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &offsetColumns, std::unordered_map< std::string, ColumnInfo > &requestedColumns) override
ColumnReaderVectorRLink(const std::string &val_branchName)
virtual void setData(ColumnVectorData &colData) override
virtual BranchPerfData getPerfData(float emptyTime) override
virtual void clearColumns() override
const std::vector< ColumnarOffsetType > * offsetColumn
std::vector< ColumnarOffsetType > offsets
Benchmark benchmarkUnpack
virtual void getEntry(Long64_t entry) override
virtual bool connect(Backend source, std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &offsetColumns, std::unordered_map< std::string, ColumnInfo > &requestedColumns) override
Reader< std::vector< ElementLink< T > > > branchReader
LinkColumnVector columnData
virtual void collectColumnData() override
std::vector< ColumnarOffsetType > offsets
LinkColumnVector columnData
virtual void clearColumns() override
virtual BranchPerfData getPerfData(float emptyTime) override
std::vector< ColumnarOffsetType > eventOffsets
Reader< std::vector< std::vector< ElementLink< T > > > > branchReader
Benchmark benchmarkUnpack
virtual void collectColumnData() override
virtual void setData(ColumnVectorData &colData) override
ColumnReaderVectorVectorLink(const std::string &val_branchName)
virtual bool connect(Backend source, std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &offsetColumns, std::unordered_map< std::string, ColumnInfo > &requestedColumns) override
virtual void getEntry(Long64_t entry) override
virtual void getEntry(Long64_t entry) override
virtual bool connect(Backend source, std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &offsetColumns, std::unordered_map< std::string, ColumnInfo > &requestedColumns) override
LinkColumnVector columnData
std::vector< ColumnarOffsetType > eventOffsets
virtual BranchPerfData getPerfData(float emptyTime) override
Benchmark benchmarkUnpack
ColumnReaderVectorVectorVariantLink(const std::string &val_branchName)
virtual void clearColumns() override
virtual void setData(ColumnVectorData &colData) override
virtual void collectColumnData() override
std::vector< ColumnarOffsetType > offsets
Reader< std::vector< std::vector< ElementLink< T > > > > branchReader
std::vector< ColumnarOffsetType > innerOffsets
virtual void setData(ColumnVectorData &colData) override
std::vector< ColumnarOffsetType > outerOffsets
Benchmark benchmarkUnpack
virtual void clearColumns() override
ColumnReaderVectorVectorVector(const std::string &val_branchName)
virtual void getEntry(Long64_t entry) override
virtual bool connect(Backend source, std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &, std::unordered_map< std::string, ColumnInfo > &requestedColumns) override
virtual void collectColumnData() override
Reader< std::vector< std::vector< std::vector< T > > > > branchReader
std::vector< T > columnData
virtual BranchPerfData getPerfData(float emptyTime) override
virtual void collectColumnData() override
ColumnReaderVectorVector(const std::string &val_branchName)
Benchmark benchmarkUnpack
Reader< std::vector< std::vector< T > > > branchReader
virtual void getEntry(Long64_t entry) override
virtual BranchPerfData getPerfData(float emptyTime) override
std::vector< ColumnarOffsetType > offsets
virtual void clearColumns() override
std::vector< T > columnData
virtual void setData(ColumnVectorData &colData) override
virtual bool connect(Backend source, std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &, std::unordered_map< std::string, ColumnInfo > &requestedColumns) override
virtual void getEntry(Long64_t entry) override
ColumnReaderVector(const std::string &val_branchName)
virtual void setData(ColumnVectorData &columnData) override
virtual bool connect(Backend source, std::unordered_map< std::string, const std::vector< ColumnarOffsetType > * > &offsetColumns, std::unordered_map< std::string, ColumnInfo > &requestedColumns) override
virtual void collectColumnData() override
virtual void clearColumns() override
virtual BranchPerfData getPerfData(float emptyTime) override
const std::vector< ColumnarOffsetType > * offsetColumn
Reader< std::vector< T > > branchReader
Benchmark benchmarkUnpack
std::vector< ColumnarOffsetType > offsets