From ca5b14bc6b9e238d049cc6302833ba7b2e284daf Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Tue, 1 Mar 2022 17:33:41 -0800 Subject: [PATCH 1/9] (CMake, FetchContent) add alias target (Metall::Metall) --- CMakeLists.txt | 4 ++-- example/cmake/FetchContent/CMakeLists.txt | 18 +++++++++--------- example/cmake/find_package/CMakeLists.txt | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03d853c9..7f458fb1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,7 @@ configure_file(MetallConfig.h.in MetallConfig.h) # ----- Setting up a INTERFACE library to install header files ----- # include(GNUInstallDirs) add_library(${PROJECT_NAME} INTERFACE) +add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) target_include_directories(${PROJECT_NAME} INTERFACE $ $) @@ -56,8 +57,7 @@ write_basic_package_version_file( configure_package_config_file( "${PROJECT_SOURCE_DIR}/cmake/${PROJECT_NAME}Config.cmake.in" "${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" - INSTALL_DESTINATION - ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/cmake) + INSTALL_DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/cmake) # install config files install(FILES diff --git a/example/cmake/FetchContent/CMakeLists.txt b/example/cmake/FetchContent/CMakeLists.txt index 526ff038..612bcbce 100644 --- a/example/cmake/FetchContent/CMakeLists.txt +++ b/example/cmake/FetchContent/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.14) include(FetchContent) -project(myproject) +project(myproject LANGUAGES C CXX) # Metall requires C++ 17 set(CMAKE_CXX_STANDARD 17) @@ -28,11 +28,11 @@ if (NOT BUILD_C) set(JUST_INSTALL_METALL_HEADER TRUE) endif() FetchContent_Declare( - metall + Metall GIT_REPOSITORY https://github.com/LLNL/metall.git GIT_TAG master ) -FetchContent_MakeAvailable(metall) +FetchContent_MakeAvailable(Metall) # ---------------------------------------- # # For using Metall CXX API @@ -44,16 +44,16 @@ add_executable(cpp_example ../src/cpp_example.cpp) # Need Boost header files target_include_directories(cpp_example PRIVATE ${Boost_INCLUDE_DIRS}) -# Link Metall -# Although target_link_libraries() is used, no library file (e.g., *.a file) is linked. -# Only include path will be set here. -target_link_libraries(cpp_example PRIVATE Metall) - # This is required if one uses GCC. target_link_libraries(cpp_example PRIVATE stdc++fs) target_link_libraries(cpp_example PRIVATE Threads::Threads) +# Link Metall +# Although target_link_libraries() is used, no library file (e.g., *.a file) is linked. +# Only include path will be set here. +target_link_libraries(cpp_example PRIVATE Metall::Metall) + # ---------------------------------------- # # For using Metall C API # ---------------------------------------- # @@ -61,5 +61,5 @@ if (BUILD_C) add_executable(c_example ../src/c_example.c) # Link Metall C library (libmetall_c) - target_link_libraries(c_example PRIVATE metall_c) + target_link_libraries(c_example PRIVATE Metall::metall_c) endif () \ No newline at end of file diff --git a/example/cmake/find_package/CMakeLists.txt b/example/cmake/find_package/CMakeLists.txt index 791e0662..200c804d 100644 --- a/example/cmake/find_package/CMakeLists.txt +++ b/example/cmake/find_package/CMakeLists.txt @@ -1,4 +1,4 @@ -project(myproject) +project(myproject LANGUAGES C CXX) # Metall requires C++ 17 set(CMAKE_CXX_STANDARD 17) From 70a89c39585f7dff81677cb1ac8e3373d492a231 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Thu, 10 Mar 2022 12:38:32 -0800 Subject: [PATCH 2/9] Add CITATION file --- CITATION.cff | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..b81eda64 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,34 @@ +# This CITATION.cff file was generated with cffinit. +# Visit https://bit.ly/cffinit to generate yours today! + +cff-version: 1.2.0 +title: >- + Metall: A Persistent Memory Allocator for + Data-Centric Analytics +message: >- + If you use this software, please cite it using the + metadata from this file. +type: software +authors: + - given-names: Keita + family-names: Iwabuchi + email: kiwabuchi@llnl.gov + affiliation: ' Lawrence Livermore National Laboratory' + orcid: 'https://orcid.org/0000-0002-9395-0843' + - given-names: 'Roger ' + family-names: Pearce + affiliation: Lawrence Livermore National Laboratory + - given-names: Maya + family-names: Gokhale + affiliation: Lawrence Livermore National Laboratory +identifiers: + - type: url + value: 'https://doi.org/10.11578/dc.20190410.1' +repository-code: 'https://github.com/LLNL/metall' +abstract: >- + Metall is a persistent memory allocator built on + top of the memory-mapped file mechanism. + + Metall enables applications to transparently + allocate custom C++ data structures into various + types of persistent memories. From 0673178d486182aeca46692271e707cb24dfa762 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Tue, 22 Mar 2022 10:51:42 -0700 Subject: [PATCH 3/9] [skip ci] Add vector of json example --- example/json/CMakeLists.txt | 1 + example/json/vector_of_json.cpp | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 example/json/vector_of_json.cpp diff --git a/example/json/CMakeLists.txt b/example/json/CMakeLists.txt index c9058237..2a60cfaa 100644 --- a/example/json/CMakeLists.txt +++ b/example/json/CMakeLists.txt @@ -2,4 +2,5 @@ if (Boost_VERSION_STRING VERSION_GREATER_EQUAL "1.75") add_metall_executable(json_create json_create.cpp) add_metall_executable(json_open json_open.cpp) add_metall_executable(jgraph jgraph.cpp) + add_metall_executable(vector_of_json vector_of_json.cpp) endif() \ No newline at end of file diff --git a/example/json/vector_of_json.cpp b/example/json/vector_of_json.cpp new file mode 100644 index 00000000..2d38e9e9 --- /dev/null +++ b/example/json/vector_of_json.cpp @@ -0,0 +1,44 @@ +// Copyright 2022 Lawrence Livermore National Security, LLC and other Metall Project Developers. +// See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#include +#include +#include +#include + +using namespace metall::container; +using namespace metall::container::experimental; + +int main() { + // Metall JSON container works like the other containers w.r.t. allocation, i.e., + // it takes an allocator type in its template parameter and an allocator object in its constructors. + using json_value_type = json::value>; + // Need to use scoped_allocator as this is a multi-layer container. + using vector_json_type = vector>; + + // An example input json strings. + std::vector json_string_list{R"({"name": "Alice", "list": [0, 1]})", + R"({"name": "Brad", "list": [2, 3]})"}; + + // Create a vector-of-json object + { + metall::manager manager(metall::create_only, "./test"); + auto *vec = manager.construct(metall::unique_instance)(manager.get_allocator()); + for (const auto &json_string: json_string_list) { + vec->emplace_back(json::parse(json_string, manager.get_allocator())); + } + } + + // Reattach the vector-of-json object created above. + { + metall::manager manager(metall::open_read_only, "./test"); + auto *vec = manager.find(metall::unique_instance).first; + for (const auto &json: *vec) { + json::pretty_print(std::cout, json); // Show contents. + } + } + + return 0; +} From 9f64fb3537c49a647fe2077140ca50a16e0f1986 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Wed, 23 Mar 2022 10:14:34 -0700 Subject: [PATCH 4/9] [skip ci] Update publication info --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index caebdf81..048ad945 100644 --- a/README.md +++ b/README.md @@ -115,15 +115,15 @@ doxygen ../docs/Doxyfile.in # Publication -## Metall: A Persistent Memory Allocator Enabling Graph Processing - -* [Paper PDF](https://www.osti.gov/servlets/purl/1576900) - -* [IEEE Xplore](https://ieeexplore.ieee.org/document/8945094) +``` +Keita Iwabuchi, Karim Youssef, Kaushik Velusamy, Maya Gokhale, Roger Pearce, +Metall: A persistent memory allocator for data-centric analytics, +Parallel Computing, 2022, 102905, ISSN 0167-8191, https://doi.org/10.1016/j.parco.2022.102905. +``` -## Metall: A Persistent Memory Allocator for Data-Centric Analytics (latest publication, preprint) +* [Parallel Computing](https://www.sciencedirect.com/science/article/abs/pii/S0167819122000114) (journal) -* [arXiv](https://arxiv.org/abs/2108.07223) +* [arXiv](https://arxiv.org/abs/2108.07223) (preprint) # About From 15fc504e46f5ee50e4130e1d06d7266e2cc6118d Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Wed, 23 Mar 2022 10:31:54 -0700 Subject: [PATCH 5/9] [skip ci] Update README --- README.md | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 048ad945..64b8c337 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,9 @@ For example, ```bash # Download Boost (Boost C++ Libraries 1.64 or more is required) # One can skip this step if Boost is already available. -wget https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz -tar xvf boost_1_77_0.tar.gz -export BOOST_ROOT=$PWD/boost_1_77_0 +wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz +tar xvf boost_1_78_0.tar.gz +export BOOST_ROOT=$PWD/boost_1_78_0 git clone https://github.com/LLNL/metall export METALL_INCLUDE=$PWD/metall/include @@ -127,13 +127,6 @@ Parallel Computing, 2022, 102905, ISSN 0167-8191, https://doi.org/10.1016/j.parc # About -## Authors - -* Keita Iwabuchi (kiwabuchi at llnl dot gov) -* Roger A Pearce (rpearce at llnl dot gov) -* Maya B Gokhale (gokhale2 at llnl dot gov). - - ## License Metall is distributed under the terms of both the MIT license and the From b7b3824f0182cc21987fd0f9bc7c235012c3c30c Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Wed, 23 Mar 2022 10:38:22 -0700 Subject: [PATCH 6/9] [skip ci] Update README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 64b8c337..53733707 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,10 @@ Parallel Computing, 2022, 102905, ISSN 0167-8191, https://doi.org/10.1016/j.parc # About +## Contact + +[Keita Iwabuchi](https://github.com/KIwabuchi) / [Roger Pearce](https://github.com/rogerpearce) / [Maya Gokhale](https://github.com/mayagokhale) + ## License Metall is distributed under the terms of both the MIT license and the From 99e620a27b439e372723e97465b65395f51717e4 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Wed, 23 Mar 2022 10:44:13 -0700 Subject: [PATCH 7/9] [skip ci] Update README --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 53733707..ed8764b5 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,9 @@ Parallel Computing, 2022, 102905, ISSN 0167-8191, https://doi.org/10.1016/j.parc ## Contact -[Keita Iwabuchi](https://github.com/KIwabuchi) / [Roger Pearce](https://github.com/rogerpearce) / [Maya Gokhale](https://github.com/mayagokhale) +- [GitHub Issues](https://github.com/LLNL/metall/issues) is open. + +- Primary contact: [Keita Iwabuchi (LLNL)](https://github.com/KIwabuchi). ## License From 8e9d33ba5809d6fc90f1b53ecc6381b7328b9ce8 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Wed, 30 Mar 2022 20:08:24 -0700 Subject: [PATCH 8/9] Add string_key_store --- include/metall/container/string_key_store.hpp | 341 +++++++++++++ .../container/string_key_store_locator.hpp | 47 ++ test/container/CMakeLists.txt | 2 + test/container/string_key_store.cpp | 461 ++++++++++++++++++ 4 files changed, 851 insertions(+) create mode 100644 include/metall/container/string_key_store.hpp create mode 100644 include/metall/container/string_key_store_locator.hpp create mode 100644 test/container/string_key_store.cpp diff --git a/include/metall/container/string_key_store.hpp b/include/metall/container/string_key_store.hpp new file mode 100644 index 00000000..99842d9f --- /dev/null +++ b/include/metall/container/string_key_store.hpp @@ -0,0 +1,341 @@ +// Copyright 2022 Lawrence Livermore National Security, LLC and other Metall Project Developers. +// See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#ifndef METALL_CONTAINER_CONCURRENT_STRING_KEY_STORE_HPP +#define METALL_CONTAINER_CONCURRENT_STRING_KEY_STORE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace metall::container { + +namespace { +namespace mc = metall::container; +} + +/// \brief A ke-value store that uses string for its key. +/// \warning This container is designed to work as the top-level container, +/// i.e., it does not work if used inside another container. +/// \tparam _value_type A value type. +/// \tparam allocator_type An allocator type. +template > +class string_key_store { + private: + template + using other_allocator = typename std::allocator_traits::template rebind_alloc; + + template + using other_scoped_allocator = mc::scoped_allocator_adaptor>; + + using internal_id_type = uint64_t; + using internal_string_type = mc::basic_string, other_allocator>; + using internal_value_type = std::tuple; + + using map_type = mc::unordered_multimap, + std::equal_to, + other_scoped_allocator>>; + + /// Used for representing 'invalid key'. + static constexpr internal_id_type k_max_internal_id = std::numeric_limits::max(); + + public: + using key_type = std::string_view; + using value_type = _value_type; + using locator_type = string_key_store_locator; + + /// \brief Constructor. + /// \param unique Accept duplicate keys if false is specified. + /// \param seed Hash function seed. + /// \param allocator An allocator object. + explicit string_key_store(const bool unique = false, + const uint64_t seed = 123, + const allocator_type &allocator = allocator_type()) + : m_unique(unique), + m_seed(seed), + m_map(allocator) {} + + /// \brief Copy constructor + string_key_store(const string_key_store &) = default; + + /// \brief Allocator-extended copy constructor + string_key_store(const string_key_store &other, const allocator_type &alloc) + : m_unique(other.m_unique), + m_seed(other.m_seed), + m_map(other.m_map, alloc) {} + + /// \brief Move constructor + string_key_store(string_key_store &&) noexcept = default; + + /// \brief Allocator-extended move constructor + string_key_store(string_key_store &&other, const allocator_type &alloc) noexcept + : m_unique(other.m_unique), + m_seed(other.m_seed), + m_map(std::move(other.m_map), alloc) {} + + /// \brief Copy assignment operator + string_key_store &operator=(const string_key_store &) = default; + + /// \brief Move assignment operator + string_key_store &operator=(string_key_store &&) noexcept = default; + + /// \brief Inserts a key with the default value. + /// If the unique parameter in the constructor was set to true and a duplicate key item already exists, + /// this function does nothing and returns false. + /// \param key A key to insert. + /// \return True if an item is inserted; otherwise, false. + bool insert(const key_type &key) { + const auto internal_id = priv_find_or_generate_internal_id(key); + if (m_unique && m_map.count(internal_id) == 1) { + return false; + } + + // We delegate tuple to decide if value_type's constructor needs an allocator object. + // By taking this approach, we can avoid the complex logic of uses-allocator construction. + // This is why we use tuple for internal_value_type. + auto internal_value = internal_value_type{std::allocator_arg, m_map.get_allocator()}; + std::get<0>(internal_value) = key; + + m_map.emplace(internal_id, std::move(internal_value)); + + return true; + } + + /// \brief Inserts an item. + /// Suppose the unique parameter was set to true in the constructor and a duplicate key item exists. + /// In that case, this function just updates the value of the existing one. + /// \param key A key to insert. + /// \param value A value to insert. + /// \return Always true. + bool insert(const key_type &key, const value_type &value) { + const auto internal_id = priv_find_or_generate_internal_id(key); + + assert(!m_unique || m_map.count(internal_id) <= 1); + if (m_unique && m_map.count(internal_id) == 1) { + assert(m_map.count(internal_id) == 1); + std::get<1>(m_map.find(internal_id)->second) = value; + } else { + m_map.emplace(internal_id, internal_value_type{std::allocator_arg, m_map.get_allocator(), key.data(), value}); + } + return true; + } + + /// \brief Insert() with move operator version. + /// \param key A key to insert. + /// \param value A value to insert. + /// \return Always true. + bool insert(const key_type &key, value_type &&value) { + const auto internal_id = priv_find_or_generate_internal_id(key); + + assert(!m_unique || m_map.count(internal_id) <= 1); + if (m_unique && m_map.count(internal_id) == 1) { + std::get<1>(m_map.find(internal_id)->second) = std::move(value); + } else { + m_map.emplace(internal_id, + internal_value_type{std::allocator_arg, m_map.get_allocator(), key.data(), std::move(value)}); + } + return true; + } + + /// \brief Clear all contents. This call does not reduce the memory usage. + void clear() { + m_map.clear(); + m_max_id_probe_distance = 0; + } + + /// \brief Counts the number of items associated with the key. + /// \param key A key to count. + /// \return The number of items associated with the key. + std::size_t count(const key_type &key) const { + const auto internal_id = priv_find_internal_id(key); + return m_map.count(internal_id); + } + + /// \brief Returns the number of elements in this container. + /// \return The number of elements in this container. + std::size_t size() const { + return m_map.size(); + } + + /// \brief Returns the key of the element at 'position'. + /// \param position A locator object. + /// \return The key of the element at 'position'. + const key_type key(const locator_type &position) const { + const auto &key = std::get<0>(position.m_iterator->second); + return key_type(key.c_str()); + } + + /// \brief Returns the value of the element at 'position'. + /// \param position A locator object. + /// \return The value of the element at 'position'. + value_type &value(const locator_type &position) { + // Convert to a non-const iterator + auto non_const_iterator = m_map.erase(position.m_iterator, position.m_iterator); + return std::get<1>(non_const_iterator->second); + } + + /// \brief Returns the value of the element at 'position'. + /// \param position A locator object. + /// \return The value of the element at 'position' as const. + const value_type &value(const locator_type &position) const { + return std::get<1>(position.m_iterator->second); + } + + /// \brief Finds an element with key equivalent to 'key'. + /// \param key The key of an element to find. + /// \return An locator object that points the found element. + locator_type find(const key_type &key) const { + const auto internal_id = priv_find_internal_id(key); + return locator_type(m_map.find(internal_id)); + } + + /// \brief Returns a range containing all elements with key key in the container. + /// \param key The key of elements to find. + /// \return A pair of locator objects. + /// The range is defined by two locators. + /// The first points to the first element of the range, + /// and the second points to the element following the last element of the range. + std::pair equal_range(const key_type &key) const { + const auto internal_id = priv_find_internal_id(key); + const auto range = m_map.equal_range(internal_id); + return std::make_pair(locator_type(range.first), locator_type(range.second)); + } + + /// \brief Return an iterator that points the first element in the container. + /// \return An iterator that points the first element in the container. + locator_type begin() const { + return locator_type(m_map.begin()); + } + + /// \brief Returns an iterator to the element following the last element. + /// \return An iterator to the element following the last element. + locator_type end() const { + return locator_type(m_map.end()); + } + + /// \brief Removes all elements with the key equivalent to key. + /// \param key The key of elements to remove. + /// \return The number of elements removed. + std::size_t erase(const key_type &key) { + const auto internal_id = priv_find_internal_id(key); + return m_map.erase(internal_id); + } + + /// \brief Removes the element at 'position'. + /// \param position The position of an element to remove. + /// \return A locator that points to the next element of the removed one. + locator_type erase(const locator_type &position) { + if (position == end()) return end(); + return locator_type(m_map.erase(position.m_iterator)); + } + + /// \brief Returns the maximum ID probe distance. + /// In other words, the maximum number of key pairs that have the same hash value. + /// \return The maximum ID probe distance. + std::size_t max_id_probe_distance() const { + return m_max_id_probe_distance; + } + + /// \brief Rehash elements. + void rehash() { + auto old_map = map_type(get_allocator()); + std::swap(old_map, m_map); + m_map.clear(); + assert(m_map.empty()); + for (auto &elem: old_map) { + insert(std::get<0>(elem.second), std::move(std::get<1>(elem.second))); + } + } + + /// \brief Returns an instance of the internal allocator. + /// \return An instance of the internal allocator. + allocator_type get_allocator() { + return m_map.get_allocator(); + } + + private: + /// \brief Generates a new internal ID for 'key'. + internal_id_type priv_generate_internal_id(const key_type &key) { + auto internal_id = priv_hash_key(key, m_seed); + + std::size_t distance = 0; + while (m_map.count(internal_id) > 0) { + internal_id = priv_increment_internal_id(internal_id); + ++distance; + } + m_max_id_probe_distance = std::max(distance, m_max_id_probe_distance); + + return internal_id; + } + + /// \brief Finds the internal ID that corresponds with 'key'. + /// If this container does not have an element with 'key', + /// returns k_max_internal_id. + internal_id_type priv_find_internal_id(const key_type &key) const { + auto internal_id = priv_hash_key(key, m_seed); + + for (std::size_t d = 0; d <= m_max_id_probe_distance; ++d) { + const auto itr = m_map.find(internal_id); + if (itr == m_map.end()) { + break; + } + + if (std::get<0>(itr->second) == key) { + return internal_id; + } + internal_id = priv_increment_internal_id(internal_id); + } + + return k_max_internal_id; // Couldn't find + } + + /// \brief Finds the internal ID that corresponds to 'key'. + /// If this container does not have an element with 'key', + /// Generate a new internal ID. + internal_id_type priv_find_or_generate_internal_id(const key_type &key) { + auto internal_id = priv_find_internal_id(key); + if (internal_id == k_max_internal_id) { // Couldn't find + // Generate a new one. + internal_id = priv_generate_internal_id(key); + } + return internal_id; + } + + static internal_id_type priv_hash_key(const key_type &key, [[maybe_unused]]const uint64_t seed) { +#ifdef METALL_CONTAINER_STRING_KEY_STORE_USE_SIMPLE_HASH + internal_id_type hash = key.empty() ? 0 : (uint8_t)key[0] % 2; +#else + auto hash = (internal_id_type)metall::mtlldetail::MurmurHash64A(key.data(), (int)key.length(), seed); +#endif + if (hash == k_max_internal_id) { + hash = priv_increment_internal_id(hash); + } + assert(hash != k_max_internal_id); + return hash; + } + + static internal_id_type priv_increment_internal_id(const internal_id_type id) { + const auto new_id = (id + 1) % k_max_internal_id; + assert(new_id != k_max_internal_id); + return new_id; + } + + bool m_unique; + uint64_t m_seed; + map_type m_map; + std::size_t m_max_id_probe_distance{0}; +}; + +} // namespace metalldata + +#endif //METALL_CONTAINER_CONCURRENT_STRING_KEY_STORE_HPP diff --git a/include/metall/container/string_key_store_locator.hpp b/include/metall/container/string_key_store_locator.hpp new file mode 100644 index 00000000..914941f0 --- /dev/null +++ b/include/metall/container/string_key_store_locator.hpp @@ -0,0 +1,47 @@ +// Copyright 2022 Lawrence Livermore National Security, LLC and other Metall Project Developers. +// See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#ifndef METALL_CONTAINER_CONCURRENT__STRING_KEY_STORE_LOCATOR_HPP_ +#define METALL_CONTAINER_CONCURRENT__STRING_KEY_STORE_LOCATOR_HPP_ + +namespace metall::container { + +template +class string_key_store_locator { + private: + template + friend class string_key_store; + + public: + + string_key_store_locator(iterator_type iterator) + : m_iterator(iterator) {} + + string_key_store_locator &operator++() { + ++m_iterator; + return *this; + } + + string_key_store_locator operator++(int) { + auto tmp(*this); + ++m_iterator; + return tmp; + } + + bool operator==(const string_key_store_locator &other) const { + return m_iterator == other.m_iterator; + } + + bool operator!=(const string_key_store_locator &other) const { + return m_iterator != other.m_iterator; + } + + private: + iterator_type m_iterator; +}; + +} // namespace metall::container + +#endif //METALL_CONTAINER_CONCURRENT__STRING_KEY_STORE_LOCATOR_HPP_ diff --git a/test/container/CMakeLists.txt b/test/container/CMakeLists.txt index ed9ce4ef..6253ea99 100644 --- a/test/container/CMakeLists.txt +++ b/test/container/CMakeLists.txt @@ -4,4 +4,6 @@ add_metall_test_executable(stl_allocator_test stl_allocator_test.cpp) add_metall_test_executable(fallback_allocator_adaptor_test fallback_allocator_adaptor_test.cpp) +add_metall_test_executable(string_key_store string_key_store.cpp) + add_subdirectory(json) \ No newline at end of file diff --git a/test/container/string_key_store.cpp b/test/container/string_key_store.cpp new file mode 100644 index 00000000..ec44cabd --- /dev/null +++ b/test/container/string_key_store.cpp @@ -0,0 +1,461 @@ +// Copyright 2019 Lawrence Livermore National Security, LLC and other Metall Project Developers. +// See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#include "gtest/gtest.h" +#include +#include +#include +#include +#include +#include +#include "../test_utility.hpp" + +#define METALL_CONTAINER_STRING_KEY_STORE_USE_SIMPLE_HASH + +namespace { + +namespace bip = boost::interprocess; + +TEST (StringKeyStoreTest, DuplicateInsert) { + metall::container::string_key_store store(false); + + ASSERT_EQ(store.count("a"), 0); + ASSERT_EQ(store.size(), 0); + + ASSERT_TRUE(store.insert("a")); + ASSERT_EQ(store.count("a"), 1); + ASSERT_EQ(store.size(), 1); + + ASSERT_TRUE(store.insert("a")); + ASSERT_EQ(store.count("a"), 2); + ASSERT_EQ(store.size(), 2); + + ASSERT_TRUE(store.insert("b")); + ASSERT_EQ(store.count("b"), 1); + ASSERT_EQ(store.size(), 3); + + std::string val("1"); + ASSERT_TRUE(store.insert("b", val)); + ASSERT_EQ(store.count("b"), 2); + ASSERT_EQ(store.size(), 4); + + ASSERT_TRUE(store.insert("b", std::move(val))); + ASSERT_EQ(store.count("b"), 3); + ASSERT_EQ(store.size(), 5); +} + +TEST (StringKeyStoreTest, UniqueInsert) { + metall::container::string_key_store store(true); + + ASSERT_EQ(store.count("a"), 0); + ASSERT_EQ(store.size(), 0); + + ASSERT_TRUE(store.insert("a")); + ASSERT_EQ(store.count("a"), 1); + ASSERT_EQ(store.size(), 1); + + ASSERT_FALSE(store.insert("a")); + ASSERT_EQ(store.count("a"), 1); + ASSERT_EQ(store.size(), 1); + + ASSERT_TRUE(store.insert("b")); + ASSERT_EQ(store.count("b"), 1); + ASSERT_EQ(store.size(), 2); + + std::string val("1"); + ASSERT_TRUE(store.insert("b", val)); + ASSERT_EQ(store.count("b"), 1); + ASSERT_EQ(store.size(), 2); + + ASSERT_TRUE(store.insert("b", std::move(val))); + ASSERT_EQ(store.count("b"), 1); + ASSERT_EQ(store.size(), 2); +} + +TEST (StringKeyStoreTest, CopyConstructorDuplicate) { + metall::container::string_key_store store(false); + store.insert("a"); + store.insert("b"); + store.insert("b"); + + auto store_copy(store); + ASSERT_EQ(store.count("a"), 1); + ASSERT_EQ(store.count("b"), 2); + ASSERT_EQ(store.size(), 3); + + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.count("b"), 2); + ASSERT_EQ(store_copy.size(), 3); + + ASSERT_TRUE(store_copy.insert("a")); + ASSERT_EQ(store_copy.count("a"), 2); + ASSERT_EQ(store_copy.size(), 4); +} + +TEST (StringKeyStoreTest, CopyConstructorUnique) { + metall::container::string_key_store store(true); + store.insert("a"); + store.insert("b"); + + auto store_copy(store); + ASSERT_EQ(store.count("a"), 1); + ASSERT_EQ(store.count("b"), 1); + ASSERT_EQ(store.size(), 2); + + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.count("b"), 1); + ASSERT_EQ(store_copy.size(), 2); + + ASSERT_FALSE(store_copy.insert("a")); + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.size(), 2); +} + +TEST (StringKeyStoreTest, CopyAsignmentDuplicate) { + metall::container::string_key_store store(false); + store.insert("a"); + store.insert("b"); + store.insert("b"); + + auto store_copy = store; + ASSERT_EQ(store.count("a"), 1); + ASSERT_EQ(store.count("b"), 2); + ASSERT_EQ(store.size(), 3); + + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.count("b"), 2); + ASSERT_EQ(store_copy.size(), 3); + + ASSERT_TRUE(store_copy.insert("a")); + ASSERT_EQ(store_copy.count("a"), 2); + ASSERT_EQ(store_copy.size(), 4); +} + +TEST (StringKeyStoreTest, CopyAsignmentUnique) { + metall::container::string_key_store store(true); + store.insert("a"); + store.insert("b"); + + auto store_copy = store; + ASSERT_EQ(store.count("a"), 1); + ASSERT_EQ(store.count("b"), 1); + ASSERT_EQ(store.size(), 2); + + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.count("b"), 1); + ASSERT_EQ(store_copy.size(), 2); + + ASSERT_FALSE(store_copy.insert("a")); + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.size(), 2); +} + +TEST (StringKeyStoreTest, MoveConstructorDuplicate) { + metall::container::string_key_store store(false); + store.insert("a"); + store.insert("b"); + store.insert("b"); + + auto store_copy(std::move(store)); + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.count("b"), 2); + ASSERT_EQ(store_copy.size(), 3); + + ASSERT_TRUE(store_copy.insert("a")); + ASSERT_EQ(store_copy.count("a"), 2); + ASSERT_EQ(store_copy.size(), 4); +} + +TEST (StringKeyStoreTest, MoveConstructorUnique) { + metall::container::string_key_store store(true); + store.insert("a"); + store.insert("b"); + + auto store_copy(std::move(store)); + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.count("b"), 1); + ASSERT_EQ(store_copy.size(), 2); + + ASSERT_FALSE(store_copy.insert("a")); + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.size(), 2); +} + +TEST (StringKeyStoreTest, MoveAsignmentDuplicate) { + metall::container::string_key_store store(false); + store.insert("a"); + store.insert("b"); + store.insert("b"); + + auto store_copy = std::move(store); + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.count("b"), 2); + ASSERT_EQ(store_copy.size(), 3); + + ASSERT_TRUE(store_copy.insert("a")); + ASSERT_EQ(store_copy.count("a"), 2); + ASSERT_EQ(store_copy.size(), 4); +} + +TEST (StringKeyStoreTest, MoveAsignmentUnique) { + metall::container::string_key_store store(true); + store.insert("a"); + store.insert("b"); + + auto store_copy = std::move(store); + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.count("b"), 1); + ASSERT_EQ(store_copy.size(), 2); + + ASSERT_FALSE(store_copy.insert("a")); + ASSERT_EQ(store_copy.count("a"), 1); + ASSERT_EQ(store_copy.size(), 2); +} + +TEST (StringKeyStoreTest, Clear) { + metall::container::string_key_store store(true); + store.insert("a"); + store.insert("b", "0"); + store.clear(); + ASSERT_EQ(store.size(), 0); +} + +TEST (StringKeyStoreTest, EraseMultipleWithKey) { + metall::container::string_key_store store(false); + ASSERT_EQ(store.erase("a"), 0); + store.insert("a"); + store.insert("b"); + store.insert("b"); + ASSERT_EQ(store.erase("c"), 0); + ASSERT_EQ(store.erase("a"), 1); + ASSERT_EQ(store.erase("a"), 0); + ASSERT_EQ(store.erase("b"), 2); + ASSERT_EQ(store.erase("b"), 0); +} + +TEST (StringKeyStoreTest, EraseSingleWithKey) { + metall::container::string_key_store store(true); + ASSERT_EQ(store.erase("a"), 0); + store.insert("a"); + store.insert("b"); + store.insert("b"); + ASSERT_EQ(store.erase("c"), 0); + ASSERT_EQ(store.erase("a"), 1); + ASSERT_EQ(store.erase("a"), 0); + ASSERT_EQ(store.erase("b"), 1); + ASSERT_EQ(store.erase("b"), 0); +} + +TEST (StringKeyStoreTest, EraseMultipleWithLocator) { + metall::container::string_key_store store(false); + ASSERT_EQ(store.erase(store.find("a")), store.end()); + store.insert("a"); + store.insert("b"); + store.insert("b"); + ASSERT_EQ(store.erase(store.find("c")), store.end()); + + auto itr = store.begin(); + ASSERT_NE(itr = store.erase(itr), store.end()); + ASSERT_NE(itr = store.erase(itr), store.end()); + ASSERT_EQ(itr = store.erase(itr), store.end()); +} + +TEST (StringKeyStoreTest, EraseSingleWithLocator) { + metall::container::string_key_store store(true); + ASSERT_EQ(store.erase(store.find("a")), store.end()); + store.insert("a"); + store.insert("b"); + store.insert("b"); + ASSERT_EQ(store.erase(store.find("c")), store.end()); + + auto itr = store.begin(); + ASSERT_NE(itr = store.erase(itr), store.end()); + ASSERT_EQ(itr = store.erase(itr), store.end()); +} + +TEST (StringKeyStoreTest, LocatorDuplicate) { + metall::container::string_key_store store(false); + ASSERT_EQ(store.begin(), store.end()); + ASSERT_EQ(store.find("a"), store.end()); + ASSERT_EQ(store.equal_range("a").first, store.end()); + ASSERT_EQ(store.equal_range("a").second, store.end()); + store.insert("a"); + store.insert("b"); + store.insert("b", "0"); + ASSERT_NE(store.begin(), store.end()); + + ASSERT_EQ(store.key(store.find("a")), "a"); + ASSERT_EQ(store.value(store.find("a")), std::string()); + + { + auto a_range = store.equal_range("a"); + std::size_t a_count = 0; + for (auto locator = a_range.first; locator != a_range.second; ++locator) { + ASSERT_EQ(store.key(locator), "a"); + ASSERT_EQ(store.value(locator), std::string()); + ++a_count; + } + ASSERT_EQ(a_count, 1); + } + + { + auto b_range = store.equal_range("b"); + std::size_t b_count = 0; + std::size_t b_default_value_count = 0; + std::size_t b_with_value_count = 0; + for (auto locator = b_range.first; locator != b_range.second; ++locator) { + ASSERT_EQ(store.key(locator), "b"); + b_default_value_count += store.value(locator).empty(); + b_with_value_count += store.value(locator) == "0"; + ++b_count; + } + ASSERT_EQ(b_count, 2); + ASSERT_EQ(b_default_value_count, 1); + ASSERT_EQ(b_with_value_count, 1); + } + + { + std::size_t count = 0; + std::size_t a_count = 0; + std::size_t b_default_value_count = 0; + std::size_t b_with_value_count = 0; + for (auto locator = store.begin(); locator != store.end(); ++locator) { + a_count += store.key(locator) == "a" && store.value(locator).empty(); + b_default_value_count += store.key(locator) == "b" && store.value(locator).empty(); + b_with_value_count += store.key(locator) == "b" && store.value(locator) == "0"; + ++count; + } + ASSERT_EQ(count, 3); + ASSERT_EQ(a_count, 1); + ASSERT_EQ(b_default_value_count, 1); + ASSERT_EQ(b_with_value_count, 1); + } +} + +TEST (StringKeyStoreTest, LocatorUnique) { + metall::container::string_key_store store(true); + ASSERT_EQ(store.begin(), store.end()); + ASSERT_EQ(store.find("a"), store.end()); + ASSERT_EQ(store.equal_range("a").first, store.end()); + ASSERT_EQ(store.equal_range("a").second, store.end()); + store.insert("a"); + store.insert("b"); + store.insert("b", "0"); + ASSERT_NE(store.begin(), store.end()); + + ASSERT_EQ(store.key(store.find("a")), "a"); + ASSERT_EQ(store.value(store.find("a")), std::string()); + + { + auto a_range = store.equal_range("a"); + std::size_t a_count = 0; + for (auto locator = a_range.first; locator != a_range.second; ++locator) { + ASSERT_EQ(store.key(locator), "a"); + ASSERT_EQ(store.value(locator), std::string()); + ++a_count; + } + ASSERT_EQ(a_count, 1); + } + + { + auto b_range = store.equal_range("b"); + std::size_t b_count = 0; + std::size_t b_default_value_count = 0; + std::size_t b_with_value_count = 0; + for (auto locator = b_range.first; locator != b_range.second; ++locator) { + ASSERT_EQ(store.key(locator), "b"); + b_default_value_count += store.value(locator).empty(); + b_with_value_count += store.value(locator) == "0"; + ++b_count; + } + ASSERT_EQ(b_count, 1); + ASSERT_EQ(b_default_value_count, 0); + ASSERT_EQ(b_with_value_count, 1); + } + + { + std::size_t count = 0; + std::size_t a_count = 0; + std::size_t b_default_value_count = 0; + std::size_t b_with_value_count = 0; + for (auto locator = store.begin(); locator != store.end(); ++locator) { + a_count += store.key(locator) == "a" && store.value(locator).empty(); + b_default_value_count += store.key(locator) == "b" && store.value(locator).empty(); + b_with_value_count += store.key(locator) == "b" && store.value(locator) == "0"; + ++count; + } + ASSERT_EQ(count, 2); + ASSERT_EQ(a_count, 1); + ASSERT_EQ(b_default_value_count, 0); + ASSERT_EQ(b_with_value_count, 1); + } +} + +TEST (StringKeyStoreTest, MaxProbeDistance) { + metall::container::string_key_store store; + ASSERT_EQ(store.max_id_probe_distance(), 0); + store.insert("a"); + ASSERT_EQ(store.max_id_probe_distance(), 0); + store.insert("b"); + ASSERT_LE(store.max_id_probe_distance(), 1); +} + +TEST (StringKeyStoreTest, Rehash) { + metall::container::string_key_store store; + store.insert("a", "0"); + store.insert("b", "1"); + store.insert("c", "2"); + store.rehash(); + ASSERT_EQ(store.value(store.find("a")), "0"); + ASSERT_EQ(store.value(store.find("b")), "1"); + ASSERT_EQ(store.value(store.find("c")), "2"); + store.erase("b"); + store.rehash(); + ASSERT_EQ(store.value(store.find("a")), "0"); + ASSERT_EQ(store.value(store.find("c")), "2"); +} + +TEST (StringKeyStoreTest, Persistence) { + using value_type = boost::container::vector>; + using store_type = metall::container::string_key_store>; + const std::string file_path(test_utility::make_test_path()); + test_utility::create_test_dir(); + metall::mtlldetail::remove_file(file_path); + + // Create + { + bip::managed_mapped_file mfile(bip::create_only, file_path.c_str(), 1 << 20); + auto* store = mfile.construct(bip::unique_instance)(true, 123, mfile.get_allocator()); + store->insert("a"); + store->value(store->find("a")).push_back(10); + } + + // Append + { + bip::managed_mapped_file mfile(bip::open_only, file_path.c_str()); + auto* store = mfile.find(bip::unique_instance).first; + + ASSERT_EQ(store->size(), 1); + ASSERT_EQ(store->value(store->find("a"))[0], 10); + + value_type vec(mfile.get_allocator()); + vec.push_back(20); + vec.push_back(30); + store->insert("b", std::move(vec)); + } + + // Read only + { + bip::managed_mapped_file mfile(bip::open_read_only, file_path.c_str()); + auto* store = mfile.find(bip::unique_instance).first; + ASSERT_EQ(store->size(), 2); + ASSERT_EQ(store->value(store->find("a"))[0], 10); + ASSERT_EQ(store->value(store->find("b"))[0], 20); + ASSERT_EQ(store->value(store->find("b"))[1], 30); + } +} +} \ No newline at end of file From 331df4414fffd311555b14209e5a2e20e0d01ad0 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Mon, 25 Apr 2022 14:14:42 -0700 Subject: [PATCH 9/9] Release v0.20 --- CMakeLists.txt | 2 +- docs/Doxyfile.in | 2 +- include/metall/version.hpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f458fb1..fe2e6df8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ cmake_policy(SET CMP0077 NEW) # Metall general configuration # -------------------------------------------------------------------------------- # project(Metall - VERSION 0.19 + VERSION 0.20 DESCRIPTION "A persistent memory allocator for data-centric analytics" HOMEPAGE_URL "https://github.com/LLNL/metall") diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 46b68f25..66f4302d 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -38,7 +38,7 @@ PROJECT_NAME = "Metall" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = v0.19 +PROJECT_NUMBER = v0.20 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/include/metall/version.hpp b/include/metall/version.hpp index 958fbbcf..d401da67 100644 --- a/include/metall/version.hpp +++ b/include/metall/version.hpp @@ -14,7 +14,7 @@ /// METALL_VERSION / 100 % 1000 // the minor version. /// METALL_VERSION % 100 // the patch level. /// \endcode -#define METALL_VERSION 1900 +#define METALL_VERSION 2000 namespace metall { /// \brief Variable type to handle a version data.