54 virtual std::string
name()
const = 0;
59 template <
typename T,
typename I>
64 const std::function<T(
I)>&,
65 const T default_value = T(),
71 std::string
name()
const override;
78 template <
typename T,
typename I>
80 const std::function<T(I)>& func,
81 const T default_value,
85 m_default_value(default_value),
89 template <
typename T,
typename I>
95 template <
typename T,
typename I>
101 template <
typename T,
typename I>
105 template <
typename T,
typename I>
109 template <
typename T,
typename I>
124 template <
typename I>
127 template <
typename I>
133 template <
typename T>
134 void add(
const std::string&
name,
const std::function<T(
I)>&,
135 const T& default_value = T(),
142 typename R=decltype(std::declval<F>()(std::declval<I>()))
144 void add(
const std::string&
name,
const F func,
const T& def = T(),
146 add<R>(
name, std::function<R(
I)>(func), R(def),
comp);
150 const std::vector<SharedConsumer<I> > &
getConsumers()
const;
153 template <
typename T>
162 template <
typename I>
163 template <
typename T>
165 const std::function<T(I)>&
fun,
169 if (m_used.count(
name)) {
170 throw std::logic_error(
"tried to insert '" +
name +
"' twice");
172 m_consumers.push_back(
177 template <
typename I>
194 template <
size_t N,
typename T,
typename I,
typename =
void>
200 template <
size_t N,
typename T,
typename I>
203 static const bool ok_type = subtype::ok_type;
204 static const bool any_type = (
208 template <
typename T,
typename I>
212 static const bool any_type = subtype::any_type || ok_type;
223 template <
size_t N,
typename F,
typename T,
size_t M = N>
229 const std::array<hsize_t,M>& extent):
233 const size_t this_dim_max = extent.at(extent.size() -
N);
234 if (
offset >= this_dim_max)
return;
236 buffer.insert(
buffer.end(), in.buffer.begin(), in.buffer.end());
237 for (
const auto& in_ele: in.element_offsets){
238 std::array<hsize_t, N> element_pos;
239 element_pos.at(0) =
offset;
240 std::copy(in_ele.begin(), in_ele.end(), element_pos.begin() + 1);
247 template <
typename F,
typename T,
size_t M>
252 const std::array<hsize_t,M>& ):
255 for (
const auto& filler:
f) {
265 if (consumers.size() < 1) {
266 throw std::logic_error(
267 "you must specify at least one consumer when initializing the HDF5"
272 size_t dt_offset = 0;
274 type.insertMember(filler->name(), dt_offset, filler->getType());
282 size_t dt_offset = 0;
284 type.insertMember(filler->name(), dt_offset, filler->getWriteType());
292 template <
typename I,
size_t N>
295 const std::array<hsize_t,N>&
extent,
303 template <
typename I,
size_t N>
305 const std::array<hsize_t,N>& extent_,
306 hsize_t batch_size_):
316 std::vector<data_buffer_t> def;
318 def.push_back(filler->getDefault());
325 std::vector<hsize_t>
vec(std::array<hsize_t,N>
a) {
326 return std::vector<hsize_t>(
a.begin(),
a.end());
332 std::array<hsize_t, N> ar;
348 template <
size_t N,
typename I>
353 const std::array<hsize_t, N>& extent = internal::uniform<N>(5),
362 template <
typename T>
365 size_t index()
const;
368 template <
typename T>
381 template <
size_t N,
typename I>
384 const std::array<hsize_t,N>& extent,
396 template <
size_t N,
typename I>
405 m_file_space(H5S_SIMPLE)
408 if (
m_par.batch_size < 1) {
409 throw std::logic_error(
"batch size must be > 0");
419 params.setFillValue(
m_par.type, default_value.data());
429 template <
size_t N,
typename I>
434 }
catch (H5::Exception&
err) {
441 template <
size_t N,
typename I>
442 template <
typename T>
444 if (m_buffer_rows == m_par.batch_size) {
454 " ** H5 Writer rank is greater than the depth of fill(...) input! **"
465 !(checkType::any_type && !checkType::ok_type),
467 " ** H5 Writer input type matches fill(...), but rank is incorrect! **"
472 " ** H5 Writer input type doesn't match input for fill(...)! **"
476 m_consumers,
arg, m_par.extent);
477 hsize_t n_el = buf.element_offsets.size();
478 std::vector<hsize_t> elements;
479 for (
const auto& el_local: buf.element_offsets) {
480 std::array<hsize_t, N+1> el_global;
481 el_global[0] = m_offset + m_buffer_rows;
482 std::copy(el_local.begin(), el_local.end(), el_global.begin() + 1);
483 elements.insert(elements.end(), el_global.begin(), el_global.end());
486 m_file_space.selectElements(H5S_SELECT_APPEND, n_el, elements.data());
492 template <
size_t N,
typename I>
494 const hsize_t buffer_size = m_buffer_rows;
495 if (buffer_size == 0)
return;
498 std::vector<hsize_t> total_dims{buffer_size + m_offset};
499 total_dims.insert(total_dims.end(),
500 m_par.extent.begin(),
502 m_ds.extend(total_dims.data());
503 m_file_space.setExtentSimple(total_dims.size(), total_dims.data());
506 hsize_t n_buffer_pts =
m_buffer.size() / m_consumers.size();
507 assert(m_file_space.getSelectNpoints() >= 0);
508 assert(
static_cast<hsize_t
>(m_file_space.getSelectNpoints())
510 H5::DataSpace mem_space(1, &n_buffer_pts);
511 m_ds.write(
m_buffer.data(), m_par.type, mem_space, m_file_space);
512 m_offset += buffer_size;
515 m_file_space.selectNone();
518 template <
size_t N,
typename I>
520 return m_buffer_rows + m_offset;
533 template <
size_t N,
class I>
537 const std::array<hsize_t, N>& extent = internal::uniform<N>(5),
548 template <
typename T>
558 template <
size_t N,
typename T>
568 template <
typename T>