From a75309919e77e82b71daed1bab4ccedab84428d9 Mon Sep 17 00:00:00 2001 From: Denis Angell Date: Fri, 20 Sep 2024 23:26:21 +0200 Subject: [PATCH] feat(SQLite): allow configurable database pragma values (#5135) Make page_size and journal_size_limit configurable values in rippled.cfg --- cfg/rippled-example.cfg | 14 +++ src/test/nodestore/Database_test.cpp | 109 ++++++++++++++++++++++ src/xrpld/app/main/DBInit.h | 22 ----- src/xrpld/app/rdb/backend/detail/Node.cpp | 4 +- src/xrpld/app/rdb/detail/Vacuum.cpp | 4 +- src/xrpld/app/rdb/detail/Wallet.cpp | 4 +- src/xrpld/core/DatabaseCon.h | 12 ++- src/xrpld/core/detail/DatabaseCon.cpp | 32 ++++++- 8 files changed, 167 insertions(+), 34 deletions(-) diff --git a/cfg/rippled-example.cfg b/cfg/rippled-example.cfg index 673ab3213e8..0ce3406f6b9 100644 --- a/cfg/rippled-example.cfg +++ b/cfg/rippled-example.cfg @@ -1106,6 +1106,20 @@ # This setting may not be combined with the # "safety_level" setting. # +# page_size Valid values: integer (MUST be power of 2 between 512 and 65536) +# The default is 4096 bytes. This setting determines +# the size of a page in the transaction.db file. +# See https://www.sqlite.org/pragma.html#pragma_page_size +# for more details about the available options. +# +# journal_size_limit Valid values: integer +# The default is 1582080. This setting limits +# the size of the journal for transaction.db file. When the limit is +# reached, older entries will be deleted. +# See https://www.sqlite.org/pragma.html#pragma_journal_size_limit +# for more details about the available options. +# +# #------------------------------------------------------------------------------- # # 7. Diagnostics diff --git a/src/test/nodestore/Database_test.cpp b/src/test/nodestore/Database_test.cpp index d866247da89..59557f3c79f 100644 --- a/src/test/nodestore/Database_test.cpp +++ b/src/test/nodestore/Database_test.cpp @@ -439,6 +439,115 @@ class Database_test : public TestBase BEAST_EXPECT(found); } } + { + // N/A: Default values + Env env(*this); + auto const s = setup_DatabaseCon(env.app().config()); + if (BEAST_EXPECT(s.txPragma.size() == 4)) + { + BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=4096;"); + BEAST_EXPECT( + s.txPragma.at(1) == "PRAGMA journal_size_limit=1582080;"); + BEAST_EXPECT( + s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;"); + BEAST_EXPECT( + s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;"); + } + } + { + // Success: Valid values + Env env = [&]() { + auto p = test::jtx::envconfig(); + { + auto& section = p->section("sqlite"); + section.set("page_size", "512"); + section.set("journal_size_limit", "2582080"); + } + return Env(*this, std::move(p)); + }(); + auto const s = setup_DatabaseCon(env.app().config()); + if (BEAST_EXPECT(s.txPragma.size() == 4)) + { + BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=512;"); + BEAST_EXPECT( + s.txPragma.at(1) == "PRAGMA journal_size_limit=2582080;"); + BEAST_EXPECT( + s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;"); + BEAST_EXPECT( + s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;"); + } + } + { + // Error: Invalid values + auto const expected = + "Invalid page_size. Must be between 512 and 65536."; + bool found = false; + auto p = test::jtx::envconfig(); + { + auto& section = p->section("sqlite"); + section.set("page_size", "256"); + } + try + { + Env env( + *this, + std::move(p), + std::make_unique(expected, &found), + beast::severities::kWarning); + fail(); + } + catch (...) + { + BEAST_EXPECT(found); + } + } + { + // Error: Invalid values + auto const expected = + "Invalid page_size. Must be between 512 and 65536."; + bool found = false; + auto p = test::jtx::envconfig(); + { + auto& section = p->section("sqlite"); + section.set("page_size", "131072"); + } + try + { + Env env( + *this, + std::move(p), + std::make_unique(expected, &found), + beast::severities::kWarning); + fail(); + } + catch (...) + { + BEAST_EXPECT(found); + } + } + { + // Error: Invalid values + auto const expected = "Invalid page_size. Must be a power of 2."; + bool found = false; + auto p = test::jtx::envconfig(); + { + auto& section = p->section("sqlite"); + section.set("page_size", "513"); + } + try + { + Env env( + *this, + std::move(p), + std::make_unique(expected, &found), + beast::severities::kWarning); + fail(); + } + catch (...) + { + BEAST_EXPECT(found); + } + } } //-------------------------------------------------------------------------- diff --git a/src/xrpld/app/main/DBInit.h b/src/xrpld/app/main/DBInit.h index 6eaf19671ac..192b1bedae3 100644 --- a/src/xrpld/app/main/DBInit.h +++ b/src/xrpld/app/main/DBInit.h @@ -42,9 +42,6 @@ inline constexpr std::uint32_t SQLITE_TUNING_CUTOFF = 10'000'000; // Ledger database holds ledgers and ledger confirmations inline constexpr auto LgrDBName{"ledger.db"}; -inline constexpr std::array LgrDBPragma{ - {"PRAGMA journal_size_limit=1582080;"}}; - inline constexpr std::array LgrDBInit{ {"BEGIN TRANSACTION;", @@ -72,25 +69,6 @@ inline constexpr std::array LgrDBInit{ // Transaction database holds transactions and public keys inline constexpr auto TxDBName{"transaction.db"}; -// In C++17 omitting the explicit template parameters caused -// a crash -inline constexpr std::array TxDBPragma -{ - "PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;", - "PRAGMA max_page_count=4294967294;", - -#if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP) - "PRAGMA mmap_size=17179869184;" -#else - - // Provide an explicit `no-op` SQL statement - // in order to keep the size of the array - // constant regardless of the preprocessor - // condition evaluation - "PRAGMA sqlite_noop_statement;" -#endif -}; - inline constexpr std::array TxDBInit{ {"BEGIN TRANSACTION;", diff --git a/src/xrpld/app/rdb/backend/detail/Node.cpp b/src/xrpld/app/rdb/backend/detail/Node.cpp index 36ff70d8dc0..2ea6bd12c62 100644 --- a/src/xrpld/app/rdb/backend/detail/Node.cpp +++ b/src/xrpld/app/rdb/backend/detail/Node.cpp @@ -72,7 +72,7 @@ makeLedgerDBs( { // ledger database auto lgr{std::make_unique( - setup, LgrDBName, LgrDBPragma, LgrDBInit, checkpointerSetup, j)}; + setup, LgrDBName, setup.lgrPragma, LgrDBInit, checkpointerSetup, j)}; lgr->getSession() << boost::str( boost::format("PRAGMA cache_size=-%d;") % kilobytes(config.getValueFor(SizedItem::lgrDBCache))); @@ -81,7 +81,7 @@ makeLedgerDBs( { // transaction database auto tx{std::make_unique( - setup, TxDBName, TxDBPragma, TxDBInit, checkpointerSetup, j)}; + setup, TxDBName, setup.txPragma, TxDBInit, checkpointerSetup, j)}; tx->getSession() << boost::str( boost::format("PRAGMA cache_size=-%d;") % kilobytes(config.getValueFor(SizedItem::txnDBCache))); diff --git a/src/xrpld/app/rdb/detail/Vacuum.cpp b/src/xrpld/app/rdb/detail/Vacuum.cpp index 92e20c86831..cc7f01a8409 100644 --- a/src/xrpld/app/rdb/detail/Vacuum.cpp +++ b/src/xrpld/app/rdb/detail/Vacuum.cpp @@ -40,8 +40,8 @@ doVacuumDB(DatabaseCon::Setup const& setup, beast::Journal j) return false; } - auto txnDB = - std::make_unique(setup, TxDBName, TxDBPragma, TxDBInit, j); + auto txnDB = std::make_unique( + setup, TxDBName, setup.txPragma, TxDBInit, j); auto& session = txnDB->getSession(); std::uint32_t pageSize; diff --git a/src/xrpld/app/rdb/detail/Wallet.cpp b/src/xrpld/app/rdb/detail/Wallet.cpp index d0631e6e633..ffb28596918 100644 --- a/src/xrpld/app/rdb/detail/Wallet.cpp +++ b/src/xrpld/app/rdb/detail/Wallet.cpp @@ -27,7 +27,7 @@ makeWalletDB(DatabaseCon::Setup const& setup, beast::Journal j) { // wallet database return std::make_unique( - setup, WalletDBName, std::array(), WalletDBInit, j); + setup, WalletDBName, std::array(), WalletDBInit, j); } std::unique_ptr @@ -38,7 +38,7 @@ makeTestWalletDB( { // wallet database return std::make_unique( - setup, dbname.data(), std::array(), WalletDBInit, j); + setup, dbname.data(), std::array(), WalletDBInit, j); } void diff --git a/src/xrpld/core/DatabaseCon.h b/src/xrpld/core/DatabaseCon.h index f656d83f9e1..935147815ca 100644 --- a/src/xrpld/core/DatabaseCon.h +++ b/src/xrpld/core/DatabaseCon.h @@ -102,6 +102,8 @@ class DatabaseCon } static std::unique_ptr const> globalPragma; + std::array txPragma; + std::array lgrPragma; }; struct CheckpointerSetup @@ -114,7 +116,7 @@ class DatabaseCon DatabaseCon( Setup const& setup, std::string const& dbName, - std::array const& pragma, + std::array const& pragma, std::array const& initSQL, beast::Journal journal) // Use temporary files or regular DB files? @@ -136,7 +138,7 @@ class DatabaseCon DatabaseCon( Setup const& setup, std::string const& dbName, - std::array const& pragma, + std::array const& pragma, std::array const& initSQL, CheckpointerSetup const& checkpointerSetup, beast::Journal journal) @@ -149,7 +151,7 @@ class DatabaseCon DatabaseCon( boost::filesystem::path const& dataDir, std::string const& dbName, - std::array const& pragma, + std::array const& pragma, std::array const& initSQL, beast::Journal journal) : DatabaseCon(dataDir / dbName, nullptr, pragma, initSQL, journal) @@ -161,7 +163,7 @@ class DatabaseCon DatabaseCon( boost::filesystem::path const& dataDir, std::string const& dbName, - std::array const& pragma, + std::array const& pragma, std::array const& initSQL, CheckpointerSetup const& checkpointerSetup, beast::Journal journal) @@ -199,7 +201,7 @@ class DatabaseCon DatabaseCon( boost::filesystem::path const& pPath, std::vector const* commonPragma, - std::array const& pragma, + std::array const& pragma, std::array const& initSQL, beast::Journal journal) : session_(std::make_shared()), j_(journal) diff --git a/src/xrpld/core/detail/DatabaseCon.cpp b/src/xrpld/core/detail/DatabaseCon.cpp index 10b34efd41e..7000899b1f6 100644 --- a/src/xrpld/core/detail/DatabaseCon.cpp +++ b/src/xrpld/core/detail/DatabaseCon.cpp @@ -175,7 +175,7 @@ setup_DatabaseCon(Config const& c, std::optional j) } { - //#synchronous Valid values : off, normal, full, extra + // #synchronous Valid values : off, normal, full, extra if (set(synchronous, "synchronous", sqlite) && !safety_level.empty()) { @@ -237,6 +237,36 @@ setup_DatabaseCon(Config const& c, std::optional j) } setup.useGlobalPragma = true; + auto setPragma = + [](std::string& pragma, std::string const& key, int64_t value) { + pragma = "PRAGMA " + key + "=" + std::to_string(value) + ";"; + }; + + // Lgr Pragma + setPragma(setup.lgrPragma[0], "journal_size_limit", 1582080); + + // TX Pragma + int64_t page_size = 4096; + int64_t journal_size_limit = 1582080; + if (c.exists("sqlite")) + { + auto& s = c.section("sqlite"); + set(journal_size_limit, "journal_size_limit", s); + set(page_size, "page_size", s); + if (page_size < 512 || page_size > 65536) + Throw( + "Invalid page_size. Must be between 512 and 65536."); + + if (page_size & (page_size - 1)) + Throw( + "Invalid page_size. Must be a power of 2."); + } + + setPragma(setup.txPragma[0], "page_size", page_size); + setPragma(setup.txPragma[1], "journal_size_limit", journal_size_limit); + setPragma(setup.txPragma[2], "max_page_count", 4294967294); + setPragma(setup.txPragma[3], "mmap_size", 17179869184); + return setup; }