diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 340297f4..3387756e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -3,7 +3,7 @@ name: CI Test on: [push, pull_request] jobs: - bst1-78-0: + bst1-79-0: runs-on: ubuntu-latest env: METALL_LIMIT_MAKE_PARALLELS: 8 @@ -12,9 +12,9 @@ jobs: - name: Test run: | pushd /dev/shm - wget -q https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz + wget -q https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.gz mkdir boost - tar xf boost_1_78_0.tar.gz -C boost --strip-components 1 + tar xf boost_1_79_0.tar.gz -C boost --strip-components 1 export BOOST_ROOT=${PWD}/boost popd export METALL_TEST_DIR=${GITHUB_JOB} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e31a8a80..dcf44bc8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,6 +27,8 @@ install_boost: script: - hostname - pwd +# - spack install boost@1.79.0 +# - spack install boost@1.78.0 # - spack install boost@1.77.0 # - spack install boost@1.76.0 # - spack install boost@1.75.0 @@ -58,6 +60,18 @@ install_boost: - export METALL_TEST_DIR="/dev/shm/metall_test-${CI_CONCURRENT_ID}-${CI_PIPELINE_IID}" - srun -N1 -ppdebug bash ./scripts/CI/build_and_test.sh +build_gcc10.2.1_bst1.79.0: + extends: .build + variables: + GCC_VERSION: "10.2.1" + BOOST_VERSION: "1.79.0" + +build_gcc10.2.1_bst1.78.0: + extends: .build + variables: + GCC_VERSION: "10.2.1" + BOOST_VERSION: "1.78.0" + build_gcc10.2.1_bst1.77.0: extends: .build variables: diff --git a/CMakeLists.txt b/CMakeLists.txt index fe2e6df8..b10cee8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ cmake_policy(SET CMP0077 NEW) # Metall general configuration # -------------------------------------------------------------------------------- # project(Metall - VERSION 0.20 + VERSION 0.21 DESCRIPTION "A persistent memory allocator for data-centric analytics" HOMEPAGE_URL "https://github.com/LLNL/metall") diff --git a/README.md b/README.md index ed8764b5..b52cd4c7 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,11 @@ All core files exist under ## Required -- GCC 8.1 or more (8.3 or more is recommended due to early implementation of the Filesystem library). -- Boost C++ Libraries 1.64 or more (build is not required; needs only their header files). +- Boost C++ Libraries 1.64 or more. + - Build is not required; needs only their header files. - To use JSON containers in Metall, Boost C++ Libraries 1.75 or more is required. +- C++17 compiler + - Tested with GCC 8.1 or more; however, 8.3 or more is recommended due to early implementation of the C++ Filesystem library. ## Build diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 66f4302d..389a693e 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -38,7 +38,7 @@ PROJECT_NAME = "Metall" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = v0.20 +PROJECT_NUMBER = v0.21 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index b00bdccf..5c847020 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -54,4 +54,5 @@ else() message(STATUS "Will skip building the MPI examples") endif() -add_subdirectory(json) \ No newline at end of file +add_subdirectory(json) +add_subdirectory(container) \ No newline at end of file diff --git a/example/container/CMakeLists.txt b/example/container/CMakeLists.txt new file mode 100644 index 00000000..0ba68c7b --- /dev/null +++ b/example/container/CMakeLists.txt @@ -0,0 +1,3 @@ +if (Boost_VERSION_STRING VERSION_GREATER_EQUAL "1.75") + add_metall_executable(string_key_store string_key_store.cpp) +endif() \ No newline at end of file diff --git a/example/container/string_key_store.cpp b/example/container/string_key_store.cpp new file mode 100644 index 00000000..69e83bb2 --- /dev/null +++ b/example/container/string_key_store.cpp @@ -0,0 +1,75 @@ +// Copyright 2022 Lawrence Livermore National Security, LLC and other Metall Project Developers. +// See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#include +#include +#include + +namespace mc = metall::container; +namespace mj = mc::experimental::json; + +// Example of a string-key-store with int value. +void int_store_example(); + +// Example of a string-key-store with JSON value. +void json_store_example(); + +int main() { + int_store_example(); + json_store_example(); +} + +void int_store_example() { + using int_store_type = mc::string_key_store; + { + metall::manager manager(metall::create_only, "./string_key_store_obj"); + + // Allocate an instance of the int-store which accept duplicate keys by default. + auto *store = manager.construct("int-store")(manager.get_allocator()); + + store->insert("a"); // Insert with the default value + store->insert("b", 0); // Insert with a value + store->insert("b", 1); // Insert another element with an existing key + } + + { + metall::manager manager(metall::open_read_only, "./string_key_store_obj"); + auto *store = manager.find("int-store").first; + + // Iterate over all elements + for (auto loc = store->begin(); loc != store->end(); ++loc) { + std::cout << store->key(loc) << " : " << store->value(loc) << std::endl; + } + } +} + +void json_store_example() { + using json_type = mj::value>; + using json_store_type = mc::string_key_store; + { + metall::manager manager(metall::open_only, "./string_key_store_obj"); + + const bool unique = true; + const uint64_t hash_seed = 123; + auto *store = manager.construct("json-store")(unique, hash_seed, manager.get_allocator()); + + store->insert("a", + mj::parse(R"({"name":"Alice"})", manager.get_allocator())); + + store->insert("b", + mj::parse(R"({"name":"N/A"})", manager.get_allocator())); + store->insert("b", + mj::parse(R"({"name":"Bob"})", manager.get_allocator())); // Overwrite + } + + { + metall::manager manager(metall::open_read_only, "./string_key_store_obj"); + auto *store = manager.find("json-store").first; + + // Use find() to locate elements. + std::cout << "a : " << mj::serialize(store->value(store->find("a"))) << std::endl; + std::cout << "b : " << mj::serialize(store->value(store->find("b"))) << std::endl; + } +} diff --git a/include/metall/container/experimental/json/detail/string_view.hpp b/include/metall/container/experimental/json/detail/string_view.hpp deleted file mode 100644 index 218d0f79..00000000 --- a/include/metall/container/experimental/json/detail/string_view.hpp +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2021 Lawrence Livermore National Security, LLC and other Metall Project Developers. -// See the top-level COPYRIGHT file for details. -// -// SPDX-License-Identifier: (Apache-2.0 OR MIT) - -#ifndef METALL_CONTAINER_EXPERIMENT_JSON_DETAIL_STRING_VIEW_HPP -#define METALL_CONTAINER_EXPERIMENT_JSON_DETAIL_STRING_VIEW_HPP - -#include -#include - -namespace metall::container::experiment::json::jsndtl { - -template , - typename pointer_type = char_type *> -class basic_string_view { - private: - using internal_pointer_type = typename std::pointer_traits::template rebind; - - public: - using value_type = char_type; - using traits_type = traits; - - using pointer = char_type *; - using const_pointer = const char_type *; - using reference = char_type &; - using const_reference = const char &; - - using const_iterator = const_pointer; - using iterator = const_iterator; - using const_reverse_iterator = std::reverse_iterator; - using reverse_iterator = const_reverse_iterator; - - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - - basic_string_view() noexcept = default; - - explicit basic_string_view(const char *const s) - : m_ptr_string(s), - m_length(traits_type::length(s)) {} - - explicit basic_string_view(std::string_view s) - : m_ptr_string(s.data()), - m_length(s.length()) {} - - basic_string_view(const char *const s, const size_type len) - : m_ptr_string(s), - m_length(len) {} - - ~basic_string_view() noexcept { - m_ptr_string = nullptr; - m_length = 0; - } - - constexpr int compare(const basic_string_view &v) const noexcept { - std::cout << data() << " <> " << v.data() << " " - << traits::compare(to_raw_pointer(data()), to_raw_pointer(v.data()), m_length) << std::endl; - return traits::compare(to_raw_pointer(data()), to_raw_pointer(v.data()), m_length); - } - - size_type size() const noexcept { - return m_length; - } - - size_type length() const noexcept { - return m_length; - } - - const_pointer data() const noexcept { - return to_raw_pointer(m_ptr_string); - } - - const_reference operator[](const size_type pos) const { - return m_ptr_string[pos]; - } - - private: - internal_pointer_type m_ptr_string{nullptr}; - std::size_t m_length{0}; -}; - -template -inline std::size_t hash_value(const basic_string_view &value) { - // std::cout << value.data() << " -> " << metall::mtlldetail::MurmurHash64A(value.data(), value.length(), 563466) << std::endl; - return jsndtl::MurmurHash64A(value.data(), value.length(), 563466); -} - -template -constexpr bool operator==(const basic_string_view &lhs, - const basic_string_view &rhs) noexcept { - return (lhs.compare(rhs) == 0); -} - -template -constexpr bool operator!=(const basic_string_view &lhs, - const basic_string_view &rhs) noexcept { - return !(lhs == rhs); -} - -template -constexpr bool operator<(const basic_string_view &lhs, - const basic_string_view &rhs) noexcept { - return lhs.compare(rhs) < 0; -} - -template -constexpr bool operator<=(const basic_string_view &lhs, - const basic_string_view &rhs) noexcept { - return !(lhs > rhs); -} - -template -constexpr bool operator>(const basic_string_view &lhs, - const basic_string_view &rhs) noexcept { - return rhs < lhs; -} - -template -constexpr bool operator>=(const basic_string_view &lhs, - const basic_string_view &rhs) noexcept { - return !(lhs < rhs); -} - -template -std::ostream &operator<<(std::ostream &os, const basic_string_view &obj) { - os << obj.data(); - return os; -} - -template -using string_view = basic_string_view, allocator_type>; - -} // namespace json::jsndtl - -#endif //METALL_CONTAINER_EXPERIMENT_JSON_DETAIL_STRING_VIEW_HPP diff --git a/include/metall/container/experimental/json/equal.hpp b/include/metall/container/experimental/json/equal.hpp index 3704f396..33378a94 100644 --- a/include/metall/container/experimental/json/equal.hpp +++ b/include/metall/container/experimental/json/equal.hpp @@ -67,7 +67,7 @@ inline bool operator==(const array &array, const bj::array &bj_a template inline bool operator==(const bj::array &bj_array, const array &array) { - return bj_array == array; + return array == bj_array; } template @@ -77,7 +77,7 @@ inline bool operator!=(const array &array, const bj::array &bj_a template inline bool operator!=(const bj::array &bj_array, const array &array) { - return bj_array != array; + return array != bj_array; } template @@ -87,7 +87,7 @@ inline bool operator==(const compact_object &object, const bj::o template inline bool operator==(const bj::object &bj_object, const compact_object &object) { - return bj_object == object; + return object == bj_object; } template @@ -97,7 +97,7 @@ inline bool operator!=(const compact_object &object, const bj::o template inline bool operator!=(const bj::object &bj_object, const compact_object &object) { - return bj_object != object; + return object != bj_object; } template @@ -107,7 +107,7 @@ inline bool operator==(const indexed_object &object, const bj::o template inline bool operator==(const bj::object &bj_object, const indexed_object &object) { - return bj_object == object; + return object == bj_object; } template @@ -117,7 +117,27 @@ inline bool operator!=(const indexed_object &object, const bj::o template inline bool operator!=(const bj::object &bj_object, const indexed_object &object) { - return bj_object != object; + return object != bj_object; +} + +template +inline bool operator==(const string &string, const bj::string &bj_string) { + return jsndtl::general_string_equal(string, bj_string); +} + +template +inline bool operator==(const bj::string &bj_string, const string &string) { + return string == bj_string; +} + +template +inline bool operator!=(const string &string, const bj::string &bj_string) { + return !(string == bj_string); +} + +template +inline bool operator!=(const bj::string &bj_string, const string &string) { + return string != bj_string; } } // namespace metall::container::experimental::json #endif //METALL_CONTAINER_EXPERIMENT_JSON_EQUAL_HPP diff --git a/include/metall/container/experimental/json/json.hpp b/include/metall/container/experimental/json/json.hpp index b2f26cd9..c8aef2ee 100644 --- a/include/metall/container/experimental/json/json.hpp +++ b/include/metall/container/experimental/json/json.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/include/metall/container/experimental/json/json_fwd.hpp b/include/metall/container/experimental/json/json_fwd.hpp index ea1da9d9..db41faa2 100644 --- a/include/metall/container/experimental/json/json_fwd.hpp +++ b/include/metall/container/experimental/json/json_fwd.hpp @@ -7,6 +7,8 @@ #define METALL_CONTAINER_EXPERIMENT_JSON_JSON_FWD_HPP #include +#include +#include /// \namespace metall::container::experimental /// \brief Namespace for Metall containers in an experimental phase. @@ -42,6 +44,11 @@ class indexed_object; template class compact_object; +template , + typename allocator = std::allocator> +using string = metall::container::basic_string; + namespace jsndtl { template inline bool general_key_value_pair_equal(const key_value_pair &, @@ -55,6 +62,9 @@ inline bool general_indexed_object_equal(const indexed_object &, template inline bool general_array_equal(const array &, const other_array_type &) noexcept; + +template +inline bool general_string_equal(const string &, const other_string_type &) noexcept; } #endif // DOXYGEN_SKIP diff --git a/include/metall/container/experimental/json/string.hpp b/include/metall/container/experimental/json/string.hpp new file mode 100644 index 00000000..a296f2a7 --- /dev/null +++ b/include/metall/container/experimental/json/string.hpp @@ -0,0 +1,23 @@ +// Copyright 2022 Lawrence Livermore National Security, LLC and other Metall Project Developers. +// See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#ifndef METALL_CONTAINER_EXPERIMENT_STRING_HPP +#define METALL_CONTAINER_EXPERIMENT_STRING_HPP + +#include + +namespace metall::container::experimental::json { +namespace jsndtl { + +template +inline bool general_string_equal(const string &string, + const other_string_type &other_string) noexcept { + return std::strcmp(string.c_str(), other_string.c_str()) == 0; +} + +} // namespace jsndtl +} // namespace metall::container::experimental::json { + +#endif //METALL_CONTAINER_EXPERIMENT_STRING_HPP diff --git a/include/metall/container/experimental/json/value.hpp b/include/metall/container/experimental/json/value.hpp index dd4a9a40..eb119dec 100644 --- a/include/metall/container/experimental/json/value.hpp +++ b/include/metall/container/experimental/json/value.hpp @@ -11,7 +11,6 @@ #include #include -#include #include namespace metall::container::experimental::json { @@ -24,7 +23,7 @@ namespace jsndtl { /// \brief Provides 'equal' calculation for other value types that have the same interface as the value class. template -inline bool general_value_equal(const value& value, const other_value_type& other_value) noexcept { +inline bool general_value_equal(const value &value, const other_value_type &other_value) noexcept { if (other_value.is_null()) { return value.is_null(); } else if (other_value.is_bool()) { @@ -34,7 +33,8 @@ inline bool general_value_equal(const value& value, const other_ return value.as_int64() == other_value.as_int64(); } if (value.is_uint64()) { - return (other_value.as_int64() < 0) ? false : value.as_uint64() == static_cast(other_value.as_int64()); + return (other_value.as_int64() < 0) ? false : value.as_uint64() + == static_cast(other_value.as_int64()); } return false; } else if (other_value.is_uint64()) { @@ -66,9 +66,9 @@ template > class value { public: using allocator_type = _allocator_type; - using string_type = mc::basic_string, - typename std::allocator_traits::template rebind_alloc>; + using string_type = string, + typename std::allocator_traits::template rebind_alloc>; using object_type = object<_allocator_type>; using array_type = array<_allocator_type>; diff --git a/include/metall/container/experimental/json/value_to.hpp b/include/metall/container/experimental/json/value_to.hpp index 58bb46de..6565322a 100644 --- a/include/metall/container/experimental/json/value_to.hpp +++ b/include/metall/container/experimental/json/value_to.hpp @@ -28,7 +28,7 @@ inline void value_to_impl_helper(const mj::value &input_value, b } else if (input_value.is_double()) { *out_bj_value = input_value.as_double(); } else if (input_value.is_string()) { - *out_bj_value = input_value.as_string(); + *out_bj_value = input_value.as_string().c_str(); } else if (input_value.is_array()) { bj::array bj_array; for (const auto &elem : input_value.as_array()) { diff --git a/include/metall/container/string_key_store.hpp b/include/metall/container/string_key_store.hpp index 99842d9f..dbe417f4 100644 --- a/include/metall/container/string_key_store.hpp +++ b/include/metall/container/string_key_store.hpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace metall::container { @@ -26,7 +27,7 @@ namespace mc = metall::container; /// i.e., it does not work if used inside another container. /// \tparam _value_type A value type. /// \tparam allocator_type An allocator type. -template > +template > class string_key_store { private: template @@ -54,15 +55,20 @@ class string_key_store { using value_type = _value_type; using locator_type = string_key_store_locator; + /// \brief Constructor. + /// \param allocator An allocator object. + explicit string_key_store(const allocator_type &allocator = allocator_type()) + : m_map(allocator) {} + /// \brief Constructor. /// \param unique Accept duplicate keys if false is specified. - /// \param seed Hash function seed. + /// \param hash_seed Hash function seed. /// \param allocator An allocator object. - explicit string_key_store(const bool unique = false, - const uint64_t seed = 123, - const allocator_type &allocator = allocator_type()) + string_key_store(const bool unique, + const uint64_t hash_seed, + const allocator_type &allocator = allocator_type()) : m_unique(unique), - m_seed(seed), + m_hash_seed(hash_seed), m_map(allocator) {} /// \brief Copy constructor @@ -71,7 +77,7 @@ class string_key_store { /// \brief Allocator-extended copy constructor string_key_store(const string_key_store &other, const allocator_type &alloc) : m_unique(other.m_unique), - m_seed(other.m_seed), + m_hash_seed(other.m_hash_seed), m_map(other.m_map, alloc) {} /// \brief Move constructor @@ -80,7 +86,7 @@ class string_key_store { /// \brief Allocator-extended move constructor string_key_store(string_key_store &&other, const allocator_type &alloc) noexcept : m_unique(other.m_unique), - m_seed(other.m_seed), + m_hash_seed(other.m_hash_seed), m_map(std::move(other.m_map), alloc) {} /// \brief Copy assignment operator @@ -263,10 +269,22 @@ class string_key_store { return m_map.get_allocator(); } + /// \brief Returns if this container inserts keys uniquely. + /// \return True if this container inserts key avoiding duplicates; otherwise false. + bool unique() const { + return m_unique; + } + + /// \brief Returns the hash seed. + /// \return Hash seed. + bool hash_seed() const { + return m_hash_seed; + } + private: /// \brief Generates a new internal ID for 'key'. internal_id_type priv_generate_internal_id(const key_type &key) { - auto internal_id = priv_hash_key(key, m_seed); + auto internal_id = priv_hash_key(key, m_hash_seed); std::size_t distance = 0; while (m_map.count(internal_id) > 0) { @@ -282,7 +300,7 @@ class string_key_store { /// If this container does not have an element with 'key', /// returns k_max_internal_id. internal_id_type priv_find_internal_id(const key_type &key) const { - auto internal_id = priv_hash_key(key, m_seed); + auto internal_id = priv_hash_key(key, m_hash_seed); for (std::size_t d = 0; d <= m_max_id_probe_distance; ++d) { const auto itr = m_map.find(internal_id); @@ -330,9 +348,9 @@ class string_key_store { return new_id; } - bool m_unique; - uint64_t m_seed; - map_type m_map; + bool m_unique{false}; + uint64_t m_hash_seed{123}; + map_type m_map{allocator_type{}}; std::size_t m_max_id_probe_distance{0}; }; diff --git a/include/metall/detail/file.hpp b/include/metall/detail/file.hpp index 0e864a80..e5290580 100644 --- a/include/metall/detail/file.hpp +++ b/include/metall/detail/file.hpp @@ -331,8 +331,10 @@ inline bool free_file_space([[maybe_unused]] const int fd, #endif } +namespace file_copy_detail { + #if defined(__cpp_lib_filesystem) && !defined(METALL_DISABLE_CXX17_FILESYSTEM_LIB) -inline bool copy_file(const std::string &source_path, const std::string &destination_path) { +inline bool copy_file_dense(const std::string &source_path, const std::string &destination_path) { bool success = true; try { if (!fs::copy_file(source_path, destination_path, fs::copy_options::overwrite_existing)) { @@ -355,7 +357,7 @@ inline bool copy_file(const std::string &source_path, const std::string &destina #else -inline bool copy_file(const std::string &source_path, const std::string &destination_path) { +inline bool copy_file_dense(const std::string &source_path, const std::string &destination_path) { { const ssize_t source_file_size = get_file_size(source_path); const ssize_t actual_source_file_size = get_actual_file_size(source_path); @@ -418,6 +420,41 @@ inline bool copy_file(const std::string &source_path, const std::string &destina #endif +#ifdef __linux__ +inline bool copy_file_sparse_linux(const std::string &source_path, const std::string &destination_path) { + std::string command("cp --sparse=auto " + source_path + " " + destination_path); + const int status = std::system(command.c_str()); + const bool success = (status != -1) && !!(WIFEXITED(status)); + if (!success) { + std::stringstream ss; + ss << "Failed copying file: " << source_path << " -> " << destination_path; + logger::out(logger::level::error, __FILE__, __LINE__, ss.str().c_str()); + return false; + } + return success; +} +#endif + +}// namespace file_copy_detail + +/// \brief Copy a file. +/// \param source_path A source file path. +/// \param destination_path A destination path. +/// \param sparse_copy If true is specified, tries to perform sparse file copy. +/// \return On success, returns true. On error, returns false. +inline bool copy_file(const std::string &source_path, + const std::string &destination_path, + const bool sparse_copy = true) { + if (sparse_copy) { +#ifdef __linux__ + return file_copy_detail::copy_file_sparse_linux(source_path, destination_path); +#else + logger::out(logger::level::info, __FILE__, __LINE__, "Sparse file copy is not available"); +#endif + } + return file_copy_detail::copy_file_dense(source_path, destination_path); +} + /// \brief Get the file names in a directory. /// This function does not list files recursively. /// Only regular files are returned. @@ -487,25 +524,26 @@ copy_files_in_directory_in_parallel_helper(const std::string &source_dir_path, std::atomic_uint_fast64_t num_successes = 0; std::atomic_uint_fast64_t file_no_cnt = 0; - auto copy_lambda = [&file_no_cnt, &num_successes, &source_dir_path, &src_file_names, &destination_dir_path, ©_func]() { - while (true) { - const auto file_no = file_no_cnt.fetch_add(1); - if (file_no >= src_file_names.size()) break; - const std::string &src_file_path = source_dir_path + "/" + src_file_names[file_no]; - const std::string &dst_file_path = destination_dir_path + "/" + src_file_names[file_no]; - num_successes.fetch_add(copy_func(src_file_path, dst_file_path) ? 1 : 0); - } - }; + auto copy_lambda + = [&file_no_cnt, &num_successes, &source_dir_path, &src_file_names, &destination_dir_path, ©_func]() { + while (true) { + const auto file_no = file_no_cnt.fetch_add(1); + if (file_no >= src_file_names.size()) break; + const std::string &src_file_path = source_dir_path + "/" + src_file_names[file_no]; + const std::string &dst_file_path = destination_dir_path + "/" + src_file_names[file_no]; + num_successes.fetch_add(copy_func(src_file_path, dst_file_path) ? 1 : 0); + } + }; const auto num_threads = (int)std::min(src_file_names.size(), (std::size_t)(max_num_threads > 0 ? max_num_threads : std::thread::hardware_concurrency())); std::vector threads(num_threads, nullptr); - for (auto &th : threads) { + for (auto &th: threads) { th = new std::thread(copy_lambda); } - for (auto &th : threads) { + for (auto &th: threads) { th->join(); } @@ -518,11 +556,19 @@ copy_files_in_directory_in_parallel_helper(const std::string &source_dir_path, /// \param destination_dir_path A path to destination directory. /// \param max_num_threads The maximum number of threads to use. /// If <= 0 is given, it is automatically determined. +/// \param sparse_copy Performs sparse file copy. /// \return On success, returns true. On error, returns false. inline bool copy_files_in_directory_in_parallel(const std::string &source_dir_path, const std::string &destination_dir_path, - const int max_num_threads) { - return copy_files_in_directory_in_parallel_helper(source_dir_path, destination_dir_path, max_num_threads, copy_file); + const int max_num_threads, + const bool sparse_copy = true) { + return copy_files_in_directory_in_parallel_helper(source_dir_path, + destination_dir_path, + max_num_threads, + [&sparse_copy](const std::string &src, + const std::string &dst) -> bool { + return copy_file(src, dst, sparse_copy); + }); } } // namespace metall::mtlldetail diff --git a/include/metall/version.hpp b/include/metall/version.hpp index d401da67..d02c9277 100644 --- a/include/metall/version.hpp +++ b/include/metall/version.hpp @@ -14,7 +14,7 @@ /// METALL_VERSION / 100 % 1000 // the minor version. /// METALL_VERSION % 100 // the patch level. /// \endcode -#define METALL_VERSION 2000 +#define METALL_VERSION 2100 namespace metall { /// \brief Variable type to handle a version data. diff --git a/test/container/CMakeLists.txt b/test/container/CMakeLists.txt index 6253ea99..4715b596 100644 --- a/test/container/CMakeLists.txt +++ b/test/container/CMakeLists.txt @@ -4,6 +4,6 @@ add_metall_test_executable(stl_allocator_test stl_allocator_test.cpp) add_metall_test_executable(fallback_allocator_adaptor_test fallback_allocator_adaptor_test.cpp) -add_metall_test_executable(string_key_store string_key_store.cpp) +add_metall_test_executable(string_key_store_test string_key_store_test.cpp) add_subdirectory(json) \ No newline at end of file diff --git a/test/container/string_key_store.cpp b/test/container/string_key_store_test.cpp similarity index 81% rename from test/container/string_key_store.cpp rename to test/container/string_key_store_test.cpp index ec44cabd..b076c578 100644 --- a/test/container/string_key_store.cpp +++ b/test/container/string_key_store_test.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include "../test_utility.hpp" #define METALL_CONTAINER_STRING_KEY_STORE_USE_SIMPLE_HASH @@ -19,8 +18,9 @@ namespace { namespace bip = boost::interprocess; TEST (StringKeyStoreTest, DuplicateInsert) { - metall::container::string_key_store store(false); + metall::container::string_key_store> store(false, 111); + ASSERT_FALSE(store.unique()); ASSERT_EQ(store.count("a"), 0); ASSERT_EQ(store.size(), 0); @@ -47,8 +47,9 @@ TEST (StringKeyStoreTest, DuplicateInsert) { } TEST (StringKeyStoreTest, UniqueInsert) { - metall::container::string_key_store store(true); + metall::container::string_key_store> store(true, 111); + ASSERT_TRUE(store.unique()); ASSERT_EQ(store.count("a"), 0); ASSERT_EQ(store.size(), 0); @@ -75,7 +76,7 @@ TEST (StringKeyStoreTest, UniqueInsert) { } TEST (StringKeyStoreTest, CopyConstructorDuplicate) { - metall::container::string_key_store store(false); + metall::container::string_key_store> store(false, 111); store.insert("a"); store.insert("b"); store.insert("b"); @@ -85,6 +86,10 @@ TEST (StringKeyStoreTest, CopyConstructorDuplicate) { ASSERT_EQ(store.count("b"), 2); ASSERT_EQ(store.size(), 3); + ASSERT_EQ(store.get_allocator(), store_copy.get_allocator()); + ASSERT_EQ(store.unique(), store_copy.unique()); + ASSERT_EQ(store.hash_seed(), store_copy.hash_seed()); + ASSERT_EQ(store_copy.count("a"), 1); ASSERT_EQ(store_copy.count("b"), 2); ASSERT_EQ(store_copy.size(), 3); @@ -95,7 +100,7 @@ TEST (StringKeyStoreTest, CopyConstructorDuplicate) { } TEST (StringKeyStoreTest, CopyConstructorUnique) { - metall::container::string_key_store store(true); + metall::container::string_key_store> store(true, 111); store.insert("a"); store.insert("b"); @@ -104,6 +109,10 @@ TEST (StringKeyStoreTest, CopyConstructorUnique) { ASSERT_EQ(store.count("b"), 1); ASSERT_EQ(store.size(), 2); + ASSERT_EQ(store.get_allocator(), store_copy.get_allocator()); + ASSERT_EQ(store.unique(), store_copy.unique()); + ASSERT_EQ(store.hash_seed(), store_copy.hash_seed()); + ASSERT_EQ(store_copy.count("a"), 1); ASSERT_EQ(store_copy.count("b"), 1); ASSERT_EQ(store_copy.size(), 2); @@ -114,7 +123,7 @@ TEST (StringKeyStoreTest, CopyConstructorUnique) { } TEST (StringKeyStoreTest, CopyAsignmentDuplicate) { - metall::container::string_key_store store(false); + metall::container::string_key_store> store(false, 111); store.insert("a"); store.insert("b"); store.insert("b"); @@ -124,6 +133,10 @@ TEST (StringKeyStoreTest, CopyAsignmentDuplicate) { ASSERT_EQ(store.count("b"), 2); ASSERT_EQ(store.size(), 3); + ASSERT_EQ(store.get_allocator(), store_copy.get_allocator()); + ASSERT_EQ(store.unique(), store_copy.unique()); + ASSERT_EQ(store.hash_seed(), store_copy.hash_seed()); + ASSERT_EQ(store_copy.count("a"), 1); ASSERT_EQ(store_copy.count("b"), 2); ASSERT_EQ(store_copy.size(), 3); @@ -134,7 +147,7 @@ TEST (StringKeyStoreTest, CopyAsignmentDuplicate) { } TEST (StringKeyStoreTest, CopyAsignmentUnique) { - metall::container::string_key_store store(true); + metall::container::string_key_store> store(true, 111); store.insert("a"); store.insert("b"); @@ -143,6 +156,10 @@ TEST (StringKeyStoreTest, CopyAsignmentUnique) { ASSERT_EQ(store.count("b"), 1); ASSERT_EQ(store.size(), 2); + ASSERT_EQ(store.get_allocator(), store_copy.get_allocator()); + ASSERT_EQ(store.unique(), store_copy.unique()); + ASSERT_EQ(store.hash_seed(), store_copy.hash_seed()); + ASSERT_EQ(store_copy.count("a"), 1); ASSERT_EQ(store_copy.count("b"), 1); ASSERT_EQ(store_copy.size(), 2); @@ -153,7 +170,7 @@ TEST (StringKeyStoreTest, CopyAsignmentUnique) { } TEST (StringKeyStoreTest, MoveConstructorDuplicate) { - metall::container::string_key_store store(false); + metall::container::string_key_store> store(false, 111); store.insert("a"); store.insert("b"); store.insert("b"); @@ -169,7 +186,7 @@ TEST (StringKeyStoreTest, MoveConstructorDuplicate) { } TEST (StringKeyStoreTest, MoveConstructorUnique) { - metall::container::string_key_store store(true); + metall::container::string_key_store> store(true, 111); store.insert("a"); store.insert("b"); @@ -184,7 +201,7 @@ TEST (StringKeyStoreTest, MoveConstructorUnique) { } TEST (StringKeyStoreTest, MoveAsignmentDuplicate) { - metall::container::string_key_store store(false); + metall::container::string_key_store> store(false, 111); store.insert("a"); store.insert("b"); store.insert("b"); @@ -200,7 +217,7 @@ TEST (StringKeyStoreTest, MoveAsignmentDuplicate) { } TEST (StringKeyStoreTest, MoveAsignmentUnique) { - metall::container::string_key_store store(true); + metall::container::string_key_store> store(true, 111); store.insert("a"); store.insert("b"); @@ -215,7 +232,7 @@ TEST (StringKeyStoreTest, MoveAsignmentUnique) { } TEST (StringKeyStoreTest, Clear) { - metall::container::string_key_store store(true); + metall::container::string_key_store> store(true, 111); store.insert("a"); store.insert("b", "0"); store.clear(); @@ -223,7 +240,7 @@ TEST (StringKeyStoreTest, Clear) { } TEST (StringKeyStoreTest, EraseMultipleWithKey) { - metall::container::string_key_store store(false); + metall::container::string_key_store> store(false, 111); ASSERT_EQ(store.erase("a"), 0); store.insert("a"); store.insert("b"); @@ -236,7 +253,7 @@ TEST (StringKeyStoreTest, EraseMultipleWithKey) { } TEST (StringKeyStoreTest, EraseSingleWithKey) { - metall::container::string_key_store store(true); + metall::container::string_key_store> store(true, 111); ASSERT_EQ(store.erase("a"), 0); store.insert("a"); store.insert("b"); @@ -249,7 +266,7 @@ TEST (StringKeyStoreTest, EraseSingleWithKey) { } TEST (StringKeyStoreTest, EraseMultipleWithLocator) { - metall::container::string_key_store store(false); + metall::container::string_key_store> store(false, 111); ASSERT_EQ(store.erase(store.find("a")), store.end()); store.insert("a"); store.insert("b"); @@ -263,7 +280,7 @@ TEST (StringKeyStoreTest, EraseMultipleWithLocator) { } TEST (StringKeyStoreTest, EraseSingleWithLocator) { - metall::container::string_key_store store(true); + metall::container::string_key_store> store(true, 111); ASSERT_EQ(store.erase(store.find("a")), store.end()); store.insert("a"); store.insert("b"); @@ -276,7 +293,7 @@ TEST (StringKeyStoreTest, EraseSingleWithLocator) { } TEST (StringKeyStoreTest, LocatorDuplicate) { - metall::container::string_key_store store(false); + metall::container::string_key_store> store(false, 111); ASSERT_EQ(store.begin(), store.end()); ASSERT_EQ(store.find("a"), store.end()); ASSERT_EQ(store.equal_range("a").first, store.end()); @@ -335,7 +352,7 @@ TEST (StringKeyStoreTest, LocatorDuplicate) { } TEST (StringKeyStoreTest, LocatorUnique) { - metall::container::string_key_store store(true); + metall::container::string_key_store> store(true, 111); ASSERT_EQ(store.begin(), store.end()); ASSERT_EQ(store.find("a"), store.end()); ASSERT_EQ(store.equal_range("a").first, store.end()); @@ -394,7 +411,7 @@ TEST (StringKeyStoreTest, LocatorUnique) { } TEST (StringKeyStoreTest, MaxProbeDistance) { - metall::container::string_key_store store; + metall::container::string_key_store> store; ASSERT_EQ(store.max_id_probe_distance(), 0); store.insert("a"); ASSERT_EQ(store.max_id_probe_distance(), 0); @@ -403,7 +420,7 @@ TEST (StringKeyStoreTest, MaxProbeDistance) { } TEST (StringKeyStoreTest, Rehash) { - metall::container::string_key_store store; + metall::container::string_key_store> store; store.insert("a", "0"); store.insert("b", "1"); store.insert("c", "2"); @@ -429,7 +446,7 @@ TEST (StringKeyStoreTest, Persistence) { // Create { bip::managed_mapped_file mfile(bip::create_only, file_path.c_str(), 1 << 20); - auto* store = mfile.construct(bip::unique_instance)(true, 123, mfile.get_allocator()); + auto *store = mfile.construct(bip::unique_instance)(true, 111, mfile.get_allocator()); store->insert("a"); store->value(store->find("a")).push_back(10); } @@ -437,7 +454,7 @@ TEST (StringKeyStoreTest, Persistence) { // Append { bip::managed_mapped_file mfile(bip::open_only, file_path.c_str()); - auto* store = mfile.find(bip::unique_instance).first; + auto *store = mfile.find(bip::unique_instance).first; ASSERT_EQ(store->size(), 1); ASSERT_EQ(store->value(store->find("a"))[0], 10); @@ -451,11 +468,12 @@ TEST (StringKeyStoreTest, Persistence) { // Read only { bip::managed_mapped_file mfile(bip::open_read_only, file_path.c_str()); - auto* store = mfile.find(bip::unique_instance).first; + auto *store = mfile.find(bip::unique_instance).first; ASSERT_EQ(store->size(), 2); ASSERT_EQ(store->value(store->find("a"))[0], 10); ASSERT_EQ(store->value(store->find("b"))[0], 20); ASSERT_EQ(store->value(store->find("b"))[1], 30); } } + } \ No newline at end of file