Merge pull request #39 from ndsev/release/0.3.0

Release/0.3.0
ndsev · Oct 31, 2023 · bf4bdb8 · bf4bdb8
2 parents 72f050d + 3919b78
commit bf4bdb8
Show file tree

Hide file tree

Showing 21 changed files with 793 additions and 136 deletions.
diff --git a/.github/workflows/cmake.yaml b/.github/workflows/cmake.yaml
@@ -43,7 +43,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest, windows-latest]
+        os: [macos-latest, windows-2019]
         python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v2
@@ -73,8 +73,10 @@ jobs:
                 "$(pwd)"/bin/wheel-auditme/mapget*.whl \
                 "$(pwd)"/bin/wheel mapget
       - name: Build (Windows)
-        if: matrix.os == 'windows-latest'
+        if: matrix.os == 'windows-latest' || matrix.os == 'windows-2019'
         working-directory: build
+        env:
+          CMAKE_GENERATOR: "Visual Studio 16 2019"
         run: |
           choco install --no-progress -y openssl --version=1.1.1.2100
           echo "cmake -DPython3_ROOT_DIR=$env:pythonLocation"

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -16,7 +16,7 @@ option(MAPGET_WITH_HTTPLIB "Enable mapget-http-datasource and mapget-http-servic
 project(mapget)
 include(FetchContent)
 
-set(MAPGET_VERSION 0.1.1)
+set(MAPGET_VERSION 0.3.0)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -168,6 +168,23 @@ add_subdirectory(libs/logging)
 add_subdirectory(libs/model)
 
 if (MAPGET_WITH_SERVICE OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
+  if (NOT TARGET rocksdb)
+    # RocksDB adds ASM as language to the project, upon which CodeCoverage.cmake
+    # tries to find a matching compiler and fails. So RocksDB must be included
+    # after CodeCoverage.
+    set(WITH_GFLAGS NO CACHE BOOL "rocksdb without gflags")
+    set(WITH_TESTS NO CACHE BOOL "rocksdb without tests")
+    set(WITH_BENCHMARK_TOOLS NO CACHE BOOL "rocksdb without benchmarking")
+    set(BENCHMARK_ENABLE_GTEST_TESTS NO CACHE BOOL "rocksdb without gtest")
+    set(DISABLE_WARNING_AS_ERROR 1 CACHE BOOL "rocksdb warnings are ok")
+
+    FetchContent_Declare(rocksdb
+      GIT_REPOSITORY "https://github.com/facebook/rocksdb.git"
+      GIT_TAG        dc87847e65449ef1cb6f787c5d753cbe8562bff1 # Use version greater than v8.6.7 once released.
+      GIT_SHALLOW    OFF)  # Turn ON when using released tag.
+    FetchContent_MakeAvailable(rocksdb)
+  endif()
+
   add_subdirectory(libs/service)
 endif()
 

diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 **Main Capabilities:**
 
 * Coordinating requests for map data to various map data source processes.
-* Integrated map data cache based on sqlite or RocksDB (*TODO: Evaluate best cache DB solution*), or a simple memory cache.
+* Integrated map data cache based on RocksDB, or a simple in-memory cache.
 * Simple data-source API with bindings for C++, Python and JS.
 * Compact GeoJSON feature storage model - [25 to 50% smaller than BSON/msgpack](docs/size-comparison/table.md).
 * Integrated deep [feature filter language](https://github.com/klebert-engineering/simfil) based on (a subset of) *JSONata*
@@ -40,6 +40,18 @@ Sample configuration files can be found under `examples/config`:
 - [sample-first-datasource.toml](examples/config/sample-first-datasource.toml) and [sample-second-datasource.toml](examples/config/sample-second-datasource.toml) will configure mapget to run a simple datasource with sample data. Note: the two formats in config files for subcommand parameters can be used interchangeably.
 - [sample-service.toml](examples/config/sample-service.toml) to execute the `mapget serve` command. The instance will fetch and serve data from sources started with `sample-*-datasource.toml` configs above.
 
+### Cache
+
+`mapget` supports persistent tile caching using a RocksDB-backed cache, and non-persistent
+in-memory caching. The CLI options to configure caching behavior are:
+
+| Option                   | Description                                                                                          | Default Value   |
+|--------------------------|------------------------------------------------------------------------------------------------------|-----------------|
+| `-c,--cache-type`        | Choose between "memory" or "rocksdb" (Technology Preview).                                           | memory          |
+| `--cache-dir`            | Path to store RocksDB cache.                                                                         | mapget-cache    |
+| `--cache-max-tiles`      | Number of tiles to store. Tiles are purged from cache in FIFO order. Set to 0 for unlimited storage. | 1024            |
+| `--clear-cache`          | Clear existing cache entries at startup.                                                             | false           |
+
 ## Map Data Sources
 
 At the heart of *mapget* are data sources, which provide map feature data for

diff --git a/docs/components.png b/docs/components.png
diff --git a/examples/config/sample-service.toml b/examples/config/sample-service.toml
@@ -9,6 +9,9 @@ serve.port=61852
 # Datasources for the server in format <host:port>. Can be specified multiple times.
 serve.datasource-host=["127.0.0.1:61853", "127.0.0.1:61854"]
 
+# Using a persistent cache.
+serve.cache-type = "rocksdb"
+
 # Datasource executable paths, including arguments, for the server. Can be specified multiple times.
 # serve.datasource-exe=
 

diff --git a/libs/http-service/src/cli.cpp b/libs/http-service/src/cli.cpp
@@ -1,14 +1,14 @@
 #include "cli.h"
-#include "http-service.h"
 #include "http-client.h"
+#include "http-service.h"
 #include "mapget/log.h"
 
 #include "mapget/http-datasource/datasource-client.h"
+#include "mapget/service/rocksdbcache.h"
 
 #include <CLI/CLI.hpp>
-#include <vector>
 #include <string>
-#include <iostream>
+#include <vector>
 
 namespace mapget
 {
@@ -18,56 +18,101 @@ struct ServeCommand
     int port_ = 0;
     std::vector<std::string> datasourceHosts_;
     std::vector<std::string> datasourceExecutables_;
+    std::string cacheType_;
+    std::string cachePath_;
+    int64_t cacheMaxTiles_ = 1024;
+    bool clearCache_ = false;
     std::string webapp_;
 
-    explicit ServeCommand(CLI::App& app) {
+    explicit ServeCommand(CLI::App& app)
+    {
         auto serveCmd = app.add_subcommand("serve", "Starts the server.");
-        serveCmd->add_option("-p,--port", port_, "Port to start the server on. Default is 0.")->default_val("0");
-        serveCmd->add_option("-d,--datasource-host", datasourceHosts_, "Datasources for the server in format <host:port>. Can be specified multiple times.");
-        serveCmd->add_option("-e,--datasource-exe", datasourceExecutables_, "Datasource executable paths, including arguments, for the server. Can be specified multiple times.");
-        serveCmd->add_option("-w,--webapp", webapp_, "Serve a static web application, in the format [<url-scope>:]<filesystem-path>.");
+        serveCmd->add_option(
+            "-p,--port",
+            port_,
+            "Port to start the server on. Default is 0.")
+            ->default_val("0");
+        serveCmd->add_option(
+            "-d,--datasource-host",
+            datasourceHosts_,
+            "Data sources in format <host:port>. Can be specified multiple times.");
+        serveCmd->add_option(
+            "-e,--datasource-exe",
+            datasourceExecutables_,
+            "Data source executable paths, including arguments. "
+            "Can be specified multiple times.");
+        serveCmd->add_option(
+            "-c,--cache-type", cacheType_, "From [memory|rocksdb], default memory, rocksdb (Technology Preview).")
+            ->default_val("memory");
+        serveCmd->add_option(
+            "--cache-dir", cachePath_, "Path to store RocksDB cache.")
+            ->default_val("mapget-cache");
+        serveCmd->add_option(
+            "--cache-max-tiles", cacheMaxTiles_, "0 for unlimited, default 1024.")
+            ->default_val(1024);
+        serveCmd->add_option(
+            "--clear-cache", clearCache_, "Clear existing cache at startup.")
+            ->default_val(false);
+        serveCmd->add_option(
+            "-w,--webapp",
+            webapp_,
+            "Serve a static web application, in the format [<url-scope>:]<filesystem-path>.");
         serveCmd->callback([this]() { serve(); });
     }
 
     void serve()
     {
         log().info("Starting server on port {}.", port_);
-        HttpService srv;
 
-        if (!datasourceHosts_.empty()){
-            for (auto& ds : datasourceHosts_){
+        std::shared_ptr<Cache> cache;
+        if (cacheType_ == "rocksdb") {
+            cache = std::make_shared<RocksDBCache>(cacheMaxTiles_, cachePath_, clearCache_);
+        }
+        else if (cacheType_ == "memory") {
+            log().info("Initializing in-memory cache.");
+            cache = std::make_shared<MemCache>(cacheMaxTiles_);
+        }
+        else {
+            logRuntimeError(stx::format("Cache type {} not supported!", cacheType_));
+        }
+
+        HttpService srv(cache);
+
+        if (!datasourceHosts_.empty()) {
+            for (auto& ds : datasourceHosts_) {
                 auto delimiterPos = ds.find(':');
                 std::string dsHost = ds.substr(0, delimiterPos);
-                int dsPort = std::stoi(ds.substr(delimiterPos+1, ds.size()));
+                int dsPort = std::stoi(ds.substr(delimiterPos + 1, ds.size()));
                 log().info("Connecting to datasource at {}:{}.", dsHost, dsPort);
                 try {
                     srv.add(std::make_shared<RemoteDataSource>(dsHost, dsPort));
                 }
-                catch(std::exception const& e) {
+                catch (std::exception const& e) {
                     log().error("  ...failed: {}", e.what());
                 }
             }
         }
 
-        if (!datasourceExecutables_.empty()){
-            for (auto& ds : datasourceExecutables_){
+        if (!datasourceExecutables_.empty()) {
+            for (auto& ds : datasourceExecutables_) {
                 log().info("Launching datasource exe: {}", ds);
                 try {
                     srv.add(std::make_shared<RemoteDataSourceProcess>(ds));
                 }
-                catch(std::exception const& e) {
+                catch (std::exception const& e) {
                     log().error("  ...failed: {}", e.what());
                 }
             }
         }
 
-        if (!webapp_.empty()){
+        if (!webapp_.empty()) {
             log().info("Webapp: {}", webapp_);
-            if (!srv.mountFileSystem(webapp_)){
+            if (!srv.mountFileSystem(webapp_)) {
                 log().error("  ...failed to mount!");
                 exit(1);
             }
         }
+
         srv.go("0.0.0.0", port_);
         srv.waitForSignal();
     }
@@ -78,16 +123,24 @@ struct FetchCommand
     std::string server_, map_, layer_;
     std::vector<uint64_t> tiles_;
 
-    explicit FetchCommand(CLI::App& app) {
+    explicit FetchCommand(CLI::App& app)
+    {
         auto fetchCmd = app.add_subcommand("fetch", "Connects to the server to fetch tiles.");
-        fetchCmd->add_option("-s,--server", server_, "Server to connect to in format <host:port>.")->required();
+        fetchCmd->add_option("-s,--server", server_, "Server to connect to in format <host:port>.")
+            ->required();
         fetchCmd->add_option("-m,--map", map_, "Map to retrieve.")->required();
         fetchCmd->add_option("-l,--layer", layer_, "Layer of the map to retrieve.")->required();
-        fetchCmd->add_option("-t,--tile", tiles_, "Tile of the map to retrieve. Can be specified multiple times.")->required();
+        fetchCmd
+            ->add_option(
+                "-t,--tile",
+                tiles_,
+                "Tile of the map to retrieve. Can be specified multiple times.")
+            ->required();
         fetchCmd->callback([this]() { fetch(); });
     }
 
-    void fetch() {
+    void fetch()
+    {
         if (log().level() <= spdlog::level::debug) {
             // Skips building the tile list string if it will not be logged.
             std::string tileList = "";
@@ -104,12 +157,15 @@ struct FetchCommand
 
         auto delimiterPos = server_.find(':');
         std::string host = server_.substr(0, delimiterPos);
-        int port = std::stoi(server_.substr(delimiterPos+1, server_.size()));
+        int port = std::stoi(server_.substr(delimiterPos + 1, server_.size()));
 
         mapget::HttpClient cli(host, port);
         auto request = std::make_shared<LayerTilesRequest>(
-            map_, layer_, std::vector<TileId>{tiles_.begin(), tiles_.end()},
-            [](auto&& tile){
+            map_,
+            layer_,
+            std::vector<TileId>{tiles_.begin(), tiles_.end()},
+            [](auto&& tile)
+            {
                 if (log().level() == spdlog::level::trace) {
                     log().trace(tile->toGeoJson().dump());
                 }
@@ -123,11 +179,14 @@ int runFromCommandLine(std::vector<std::string> args)
     CLI::App app{"A client/server application for map data retrieval."};
     std::string log_level_;
     app.add_option(
-        "--log-level",
-        log_level_,
-        "From: trace, debug, info, warn, error, critical. Overrides MAPGET_LOG_LEVEL."
-    )->default_val("");
-    app.set_config("--config", "", "Optional path to a file with configuration arguments for mapget.");
+           "--log-level",
+           log_level_,
+           "From [trace|debug|info|warn|error|critical], overrides MAPGET_LOG_LEVEL.")
+        ->default_val("");
+    app.set_config(
+        "--config",
+        "",
+        "Optional path to a file with configuration arguments for mapget.");
 
     app.require_subcommand(1);
 
@@ -141,10 +200,11 @@ int runFromCommandLine(std::vector<std::string> args)
     try {
         std::reverse(args.begin(), args.end());
         app.parse(std::move(args));
-    } catch(const CLI::ParseError &e) {
+    }
+    catch (const CLI::ParseError& e) {
         return app.exit(e);
     }
     return 0;
 }
 
-}
+}  // namespace mapget
diff --git a/libs/model/include/mapget/model/featurelayer.h b/libs/model/include/mapget/model/featurelayer.h
@@ -49,8 +49,7 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool
      * @param layerInfo Information about the map layer this feature is associated with.
      *  Each feature in this layer must have a feature type which is also present in
      *  the layer. Therefore, feature ids from this layer can be verified to conform
-     *  to one of the allowed feature id compositions for the allowed type.
-     * TODO "for the allowed type" -> "for the feature type" ?
+     *  to one of the allowed feature id compositions for the feature type.
      * @param fields Shared field name dictionary, which allows compressed storage
      *  of object field name strings. It is auto-filled, and one instance may be used
      *  by multiple TileFeatureLayer instances.
@@ -218,7 +217,7 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool
     model_ptr<FeatureId> resolveFeatureId(simfil::ModelNode const& n) const;
 
     /**
-     * Generic node resolution overload
+     * Generic node resolution overload.
      */
     void resolve(const simfil::ModelNode &n, const ResolveFn &cb) const override;
 

diff --git a/libs/model/include/mapget/model/fields.h b/libs/model/include/mapget/model/fields.h
@@ -6,17 +6,15 @@ namespace mapget
 {
 
 /**
- * The Fields class is a case-insensitive dictionary for field names.
- * It acts as a container that automatically populates itself when fields are added to an object.
- * Multiple TileFeatureLayers can share the same Fields dictionary, which helps reduce the size
- * of serialized FeatureLayers.
+ * The Fields class is a case-insensitive dictionary of uint16_t to field name strings.
+ * It populates itself when fields are added to an object. Multiple TileFeatureLayers
+ * can share the same Fields dictionary, reducing the size of serialized FeatureLayers.
  *
- * The inherited mapget::Fields provides additional static field IDs that can be used for various purposes.
+ * The inherited mapget::Fields contains static field IDs for various purposes.
  *
- * Field IDs managed by the Fields dictionary are stored as uint16 values, which are smaller and faster
- * to work with compared to strings. When querying fields, the Fields dictionary is used to look up the
- * field ID for a given name. Once the ID is obtained, subsequent searches are performed using this 16-bit
- * integer, resulting in improved efficiency.
+ * Field IDs are uint16 values, which are smaller and faster to work with than strings.
+ * When querying fields by name, the Fields dictionary is used to look up a field ID.
+ * Subsequent searches are performed using this 16-bit integer for improved efficiency.
  *
  * Note: A fields dictionary is always unique per datasource node. Therefore,
  * the Fields object must be constructed with a datasource node id.
@@ -33,7 +31,7 @@ struct Fields : public simfil::Fields
         PropertiesStr,
     };
 
-    explicit Fields(std::string nodeId);
+    explicit Fields(const std::string_view& nodeId);
 
     /**
      * Write is overloaded, because it prepends the stream with