Implement flexible strings.

This commit implements serializing `std::string` to both variable (the
default) and fixed length HDF5 strings.

include/highfive/bits/H5Attribute_misc.hpp

@@ -87,8 +87,10 @@ inline void Attribute::read(T& array) const {
     read(r.getPointer(), buffer_info.data_type);
     // re-arrange results
     r.unserialize(array);
-    auto t = create_datatype<typename details::inspector<T>::base_type>();
+
+    auto t = buffer_info.data_type;
     auto c = t.getClass();
+
     if (c == DataTypeClass::VarLen || t.isVariableStr()) {
 #if H5_VERSION_GE(1, 12, 0)
         // This one have been created in 1.12.0

include/highfive/bits/H5Converter_misc.hpp

@@ -16,11 +16,23 @@
 namespace HighFive {
 namespace details {
 
+template <class T>
+struct is_std_string {
+    static constexpr bool value =
+        std::is_same<typename inspector<T>::base_type, std::string>::value;
+};
+
 template <class T, class V = void>
-struct enable_shallow_copy: public std::enable_if<inspector<T>::is_trivially_copyable, V> {};
+struct enable_shallow_copy
+    : public std::enable_if<!is_std_string<T>::value && inspector<T>::is_trivially_copyable, V> {};
 
 template <class T, class V = void>
-struct enable_deep_copy: public std::enable_if<!inspector<T>::is_trivially_copyable, V> {};
+struct enable_deep_copy
+    : public std::enable_if<!is_std_string<T>::value && !inspector<T>::is_trivially_copyable, V> {};
+
+template <class T, class V = void>
+struct enable_string_copy: public std::enable_if<is_std_string<T>::value, V> {};
+
 
 template <typename T, bool IsReadOnly>
 struct ShallowCopyBuffer {
@@ -85,6 +97,234 @@ struct DeepCopyBuffer {
     std::vector<size_t> dims;
 };
 
+enum class BufferMode { Read, Write };
+
+
+///
+/// \brief String length in bytes excluding the `\0`.
+///
+inline size_t char_buffer_size(char const* const str, size_t max_string_length) {
+    for (size_t i = 0; i <= max_string_length; ++i) {
+        if (str[i] == '\0') {
+            return i;
+        }
+    }
+
+    return max_string_length;
+}
+
+
+///
+/// \brief A buffer for reading/writing strings.
+///
+/// A string in HDF5 can be represented as a fixed or variable length string.
+/// The important difference for this buffer is that `H5D{read,write}` expects
+/// different input depending on if the strings are fixed or variable length.
+/// For fixed length strings, it expects an array of chars, i.e. one string
+/// packed after the other contiguously.  While for variable length strings it
+/// expects a list of pointer to the beginning of each string. Variable length
+/// string must be null-terminated; otherwise their length is unknown.
+///
+/// This buffer hides the difference between fixed and variable length strings
+/// by having internal datastructures available for both cases at compile time.
+/// The choice which internal buffer to use is made at runtime.
+///
+/// In a dataset with N fixed-length strings which each are M characters long,
+/// the in-memory strings are copied into an internal buffer of size N*M. If
+/// null- or space-padded the buffer should be filled with the appropriate
+/// character. This is important if the in-memory strings are less than M
+/// characters long.
+///
+/// A dataset with N variable-length strings (all null-terminated) uses uses
+/// the internal list of pointers to the beginning of each string. Those
+/// pointers can either point to the in-memory string themselves, if those
+/// strings are known to be null-terminated. Otherwise the in-memory string are
+/// copied to an internal buffer of null-terminated strings; and the pointer
+/// point to the start of the string in the internal buffer.
+///
+/// This class is responsible for arranging the string properly before passing
+/// the buffers to HDF5. To keep this class generic, it provides a generic
+/// read/write interface to the internal strings, i.e. a pointer with a size.
+/// For reading from the buffer the proxy is called `StringConstView`. This
+/// proxy object is to be used by the `inspector` to copy from the buffer into
+/// the final destination, e.g. an `std::string`.  Similarly, there's a proxy
+/// object for serializing into the buffer, i.e. the `StringView`. Again the
+/// `inspector` is responsible for obtaining the pointer, size and padding of
+/// the string.
+///
+/// Nomenclature:
+///   - size of a string is the number of bytes required to store the string,
+///     including the null character for null-terminated strings.
+///
+///   - length of a string is the number of bytes without the null character.
+///
+/// Note: both 'length' and 'size' are counted in number of bytes, not number
+///   of symbols or characters. Even for UTF8 strings.
+template <typename T, BufferMode buffer_mode>
+struct StringBuffer {
+    using type = unqualified_t<T>;
+    using hdf5_type = typename inspector<type>::hdf5_type;
+
+    class StringView {
+      public:
+        StringView(StringBuffer<T, buffer_mode>& _buffer, size_t _i)
+            : buffer(_buffer)
+            , i(_i) {}
+
+        ///
+        /// \brief Assign the in-memory string to the buffer.
+        ///
+        /// This method copies the in-memory string to the appropriate
+        /// internal buffer as needed.
+        ///
+        /// The `length` is the length of the string in bytes.
+        void assign(char const* data, size_t length, StringPadding padding) {
+            if (buffer.isVariableLengthString()) {
+                if (padding == StringPadding::NullTerminated) {
+                    buffer.variable_length_pointers[i] = data;
+                } else {
+                    buffer.variable_length_buffer[i] = std::string(data, length);
+                    buffer.variable_length_pointers[i] = buffer.variable_length_buffer[i].data();
+                }
+            } else if (buffer.isFixedLengthString()) {
+                // If the buffer is fixed-length and null-terminated, then
+                // `buffer.string_length` doesn't include the null-character.
+                if (length > buffer.string_length) {
+                    throw std::invalid_argument("String length too big.");
+                }
+
+                memcpy(&buffer.fixed_length_buffer[i * buffer.string_size], data, length);
+            }
+        }
+
+      private:
+        StringBuffer<T, buffer_mode>& buffer;
+        size_t i;
+    };
+
+
+    class StringConstView {
+      public:
+        StringConstView(const StringBuffer<T, buffer_mode>& _buffer, size_t _i)
+            : buffer(_buffer)
+            , i(_i) {}
+
+        char const* data() const {
+            if (buffer.isVariableLengthString()) {
+                return buffer.variable_length_pointers[i];
+            } else {
+                return &buffer.fixed_length_buffer[i * buffer.string_size];
+            }
+        }
+
+        // Length of the string in bytes without null for null-terminated strings.
+        size_t length() const {
+            if (buffer.isNullTerminated()) {
+                return char_buffer_size(data(), buffer.string_length);
+            } else {
+                return buffer.string_length;
+            }
+        }
+
+      private:
+        const StringBuffer<T, buffer_mode>& buffer;
+        size_t i;
+    };
+
+
+    class Iterator {
+      public:
+        Iterator(StringBuffer<T, buffer_mode>& _buffer, size_t _pos)
+            : buffer(_buffer)
+            , pos(_pos) {}
+
+        Iterator operator+(size_t n_strings) const {
+            return Iterator(buffer, pos + n_strings);
+        }
+
+        void operator+=(size_t n_strings) {
+            pos += n_strings;
+        }
+
+        StringView operator*() {
+            return StringView(buffer, pos);
+        }
+
+        StringConstView operator*() const {
+            return StringConstView(buffer, pos);
+        }
+
+      private:
+        StringBuffer<T, buffer_mode>& buffer;
+        size_t pos;
+    };
+
+    StringBuffer(std::vector<size_t> _dims, const DataType& _file_datatype)
+        : file_datatype(_file_datatype.asStringType())
+        , padding(file_datatype.getPadding())
+        , string_size(file_datatype.isVariableStr() ? size_t(-1) : file_datatype.getSize())
+        , string_length(string_size - size_t(isNullTerminated()))
+        , dims(_dims) {
+        if (string_size == 0 && isNullTerminated()) {
+            throw DataTypeException(
+                "Fixed-length, null-terminated need at least one byte to store the "
+                "null-character.");
+        }
+
+        auto n_strings = compute_total_size(dims);
+        if (isVariableLengthString()) {
+            variable_length_buffer.resize(n_strings);
+            variable_length_pointers.resize(n_strings);
+        } else {
+            char pad = padding == StringPadding::SpacePadded ? ' ' : '\0';
+            fixed_length_buffer.assign(n_strings * string_size, pad);
+        }
+    }
+
+    bool isVariableLengthString() const {
+        return file_datatype.isVariableStr();
+    }
+
+    bool isFixedLengthString() const {
+        return file_datatype.isFixedLenStr();
+    }
+
+    bool isNullTerminated() const {
+        return file_datatype.getPadding() == StringPadding::NullTerminated;
+    }
+
+
+    void* getPointer() {
+        if (file_datatype.isVariableStr()) {
+            return variable_length_pointers.data();
+        } else {
+            return fixed_length_buffer.data();
+        }
+    }
+
+    Iterator begin() {
+        return Iterator(*this, 0ul);
+    }
+
+    void unserialize(T& val) {
+        inspector<type>::unserialize(begin(), dims, val);
+    }
+
+  private:
+    StringType file_datatype;
+    StringPadding padding;
+    size_t string_size;    // Size of buffer required to store the string.
+                           // Meaningful for fixed length strings only.
+    size_t string_length;  // Semantic length of string.
+    std::vector<size_t> dims;
+
+    std::vector<char> fixed_length_buffer;
+    std::vector<std::string> variable_length_buffer;
+    std::vector<
+        typename std::conditional<buffer_mode == BufferMode::Write, const char, char>::type*>
+        variable_length_pointers;
+};
+
 
 template <typename T, typename Enable = void>
 struct Writer;
@@ -107,6 +347,14 @@ struct Writer<T, typename enable_deep_copy<T>::type>: public DeepCopyBuffer<T> {
     }
 };
 
+template <typename T>
+struct Writer<T, typename enable_string_copy<T>::type>: public StringBuffer<T, BufferMode::Write> {
+    explicit Writer(const T& val, const DataType& _file_datatype)
+        : StringBuffer<T, BufferMode::Write>(inspector<T>::getDimensions(val), _file_datatype) {
+        inspector<T>::serialize(val, this->begin());
+    }
+};
+
 template <typename T, typename Enable = void>
 struct Reader;
 
@@ -133,6 +381,13 @@ struct Reader<T, typename enable_deep_copy<T>::type>: public DeepCopyBuffer<T> {
 };
 
 
+template <typename T>
+struct Reader<T, typename enable_string_copy<T>::type>: public StringBuffer<T, BufferMode::Write> {
+  public:
+    explicit Reader(const std::vector<size_t>& _dims, const T& val, const DataType& _file_datatype)
+        : StringBuffer<T, BufferMode::Write>(_dims, _file_datatype) {}
+};
+
 struct data_converter {
     template <typename T>
     static Writer<T> serialize(const typename inspector<T>::type& val,

include/highfive/bits/H5DataType_misc.hpp

@@ -22,7 +22,7 @@
 #include <half.hpp>
 #endif
 
-#include "H5Converter_misc.hpp"
+#include "H5Inspector_misc.hpp"
 
 namespace HighFive {
 

include/highfive/bits/H5Inspector_misc.hpp

@@ -1,16 +1,10 @@
-/*
- *  Copyright (c) 2022 Blue Brain Project
- *
- *  Distributed under the Boost Software License, Version 1.0.
- *    (See accompanying file LICENSE_1_0.txt or copy at
- *          http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
 #pragma once
 
 #include <type_traits>
 #include <cstring>
+#include <vector>
+#include <array>
+#include <string>
 #include <numeric>
 
 #include "../H5Reference.hpp"
@@ -27,7 +21,9 @@
 #include <Eigen/Eigen>
 #endif
 
+
 namespace HighFive {
+
 namespace details {
 
 inline bool checkDimensions(const std::vector<size_t>& dims, size_t n_dim_requested) {
@@ -259,14 +255,15 @@ struct inspector<std::string>: type_helper<std::string> {
         throw DataSpaceException("A std::string cannot be written directly.");
     }
 
-    static void serialize(const type& val, hdf5_type* m) {
-        *m = val.c_str();
+    template <class It>
+    static void serialize(const type& val, It m) {
+        (*m).assign(val.data(), val.size(), StringPadding::NullTerminated);
     }
 
-    static void unserialize(const hdf5_type* vec,
-                            const std::vector<size_t>& /* dims */,
-                            type& val) {
-        val = vec[0];
+    template <class It>
+    static void unserialize(const It& vec, const std::vector<size_t>& /* dims */, type& val) {
+        const auto& view = *vec;
+        val.assign(view.data(), view.length());
     }
 };