From 98376f8c0a29167a3f4f7fac212fef09be49b628 Mon Sep 17 00:00:00 2001 From: Luc Grosheintz Date: Thu, 13 Jul 2023 10:55:28 +0200 Subject: [PATCH 1/2] Allow Buffers to depend on the file datatype. (#802) The upcoming `StringBuffer` will need to know the file datatype, because in HDF5 the expect datalayout, and therefore the way the buffer is populated, is different for fixed and variable length strings. --- include/highfive/bits/H5Attribute_misc.hpp | 11 ++++++---- include/highfive/bits/H5Converter_misc.hpp | 21 +++++++++++-------- include/highfive/bits/H5Slice_traits_misc.hpp | 12 +++++++---- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/include/highfive/bits/H5Attribute_misc.hpp b/include/highfive/bits/H5Attribute_misc.hpp index ed920eb3f..c53f4cf49 100644 --- a/include/highfive/bits/H5Attribute_misc.hpp +++ b/include/highfive/bits/H5Attribute_misc.hpp @@ -61,8 +61,9 @@ inline T Attribute::read() const { template inline void Attribute::read(T& array) const { const DataSpace& mem_space = getMemSpace(); + auto file_datatype = getDataType(); const details::BufferInfo buffer_info( - getDataType(), + file_datatype, [this]() -> std::string { return this->getName(); }, details::BufferInfo::read); @@ -82,7 +83,7 @@ inline void Attribute::read(T& array) const { return; } - auto r = details::data_converter::get_reader(dims, array); + auto r = details::data_converter::get_reader(dims, array, file_datatype); read(r.get_pointer(), buffer_info.data_type); // re-arrange results r.unserialize(array); @@ -125,8 +126,10 @@ inline void Attribute::write(const T& buffer) { return; } + auto file_datatype = getDataType(); + const details::BufferInfo buffer_info( - getDataType(), + file_datatype, [this]() -> std::string { return this->getName(); }, details::BufferInfo::write); @@ -136,7 +139,7 @@ inline void Attribute::write(const T& buffer) { << " into dataset of dimensions " << mem_space.getNumberDimensions(); throw DataSpaceException(ss.str()); } - auto w = details::data_converter::serialize(buffer); + auto w = details::data_converter::serialize(buffer, file_datatype); write_raw(w.get_pointer(), buffer_info.data_type); } diff --git a/include/highfive/bits/H5Converter_misc.hpp b/include/highfive/bits/H5Converter_misc.hpp index cfb3e2420..27c822e05 100644 --- a/include/highfive/bits/H5Converter_misc.hpp +++ b/include/highfive/bits/H5Converter_misc.hpp @@ -22,6 +22,7 @@ struct enable_shallow_copy: public std::enable_if::is_trivially_cop template struct enable_deep_copy: public std::enable_if::is_trivially_copyable, V> {}; + template struct ShallowCopyBuffer { using type = unqualified_t; @@ -83,19 +84,18 @@ struct Writer::type>: public ShallowCopyBuffe using super = ShallowCopyBuffer; public: - explicit Writer(const T& val) + explicit Writer(const T& val, const DataType& file_datatype) : super(val){}; }; template struct Writer::type>: public DeepCopyBuffer { - explicit Writer(const T& val) + explicit Writer(const T& val, const DataType& file_datatype) : DeepCopyBuffer(inspector::getDimensions(val)) { inspector::serialize(val, this->get_pointer()); } }; - template struct Reader; @@ -106,7 +106,7 @@ struct Reader::type>: public ShallowCopyBuffe using type = typename super::type; public: - Reader(const std::vector&, type& val) + Reader(const std::vector&, type& val, const DataType&) : super(val) {} }; @@ -117,23 +117,26 @@ struct Reader::type>: public DeepCopyBuffer { using type = typename super::type; public: - Reader(const std::vector& _dims, type&) + Reader(const std::vector& _dims, type&, const DataType&) : super(_dims) {} }; struct data_converter { template - static Writer serialize(const typename inspector::type& val) { - return Writer(val); + static Writer serialize(const typename inspector::type& val, + const DataType& file_datatype) { + return Writer(val, file_datatype); } template - static Reader get_reader(const std::vector& dims, T& val) { + static Reader get_reader(const std::vector& dims, + T& val, + const DataType& file_datatype) { // TODO Use bufferinfo for recursive_ndim auto effective_dims = details::squeezeDimensions(dims, inspector::recursive_ndim); inspector::prepare(val, effective_dims); - return Reader(effective_dims, val); + return Reader(effective_dims, val, file_datatype); } }; diff --git a/include/highfive/bits/H5Slice_traits_misc.hpp b/include/highfive/bits/H5Slice_traits_misc.hpp index 662c8d03f..0dc9f9a7d 100644 --- a/include/highfive/bits/H5Slice_traits_misc.hpp +++ b/include/highfive/bits/H5Slice_traits_misc.hpp @@ -172,8 +172,10 @@ inline void SliceTraits::read(T& array, const DataTransferProps& xfer_ const auto& slice = static_cast(*this); const DataSpace& mem_space = slice.getMemSpace(); + auto file_datatype = slice.getDataType(); + const details::BufferInfo buffer_info( - slice.getDataType(), + file_datatype, [&slice]() -> std::string { return details::get_dataset(slice).getPath(); }, details::BufferInfo::Operation::read); @@ -193,7 +195,7 @@ inline void SliceTraits::read(T& array, const DataTransferProps& xfer_ return; } - auto r = details::data_converter::get_reader(dims, array); + auto r = details::data_converter::get_reader(dims, array, file_datatype); read(r.get_pointer(), buffer_info.data_type, xfer_props); // re-arrange results r.unserialize(array); @@ -251,8 +253,10 @@ inline void SliceTraits::write(const T& buffer, const DataTransferProp return; } + auto file_datatype = slice.getDataType(); + const details::BufferInfo buffer_info( - slice.getDataType(), + file_datatype, [&slice]() -> std::string { return details::get_dataset(slice).getPath(); }, details::BufferInfo::Operation::write); @@ -263,7 +267,7 @@ inline void SliceTraits::write(const T& buffer, const DataTransferProp << " into dataset with n = " << buffer_info.n_dimensions << " dimensions."; throw DataSpaceException(ss.str()); } - auto w = details::data_converter::serialize(buffer); + auto w = details::data_converter::serialize(buffer, file_datatype); write_raw(w.get_pointer(), buffer_info.data_type, xfer_props); } From 528e6fe7690e065b595fc6434191415fa065e2c0 Mon Sep 17 00:00:00 2001 From: Luc Grosheintz Date: Thu, 13 Jul 2023 10:58:54 +0200 Subject: [PATCH 2/2] Clang-format CI should show what's wrong. (#804) --- .github/workflows/clang_format.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/clang_format.yml b/.github/workflows/clang_format.yml index 99cb6e1b3..56f2fd8d5 100644 --- a/.github/workflows/clang_format.yml +++ b/.github/workflows/clang_format.yml @@ -32,5 +32,8 @@ jobs: then echo "Some files are not well formatted:" echo $modified_files + echo "" + echo "The diff is:" + git diff exit 1 fi