Skip to content

Commit

Permalink
feat(SQLite): allow configurable database pragma values (#5135)
Browse files Browse the repository at this point in the history 
Make page_size and journal_size_limit configurable values in rippled.cfg
  • Loading branch information
@dangell7
dangell7 authored Sep 20, 2024
1 parent 0ece395 commit a753099
Show file tree
Hide file tree
Showing 8 changed files with 167 additions and 34 deletions.
14 changes: 14 additions & 0 deletions cfg/rippled-example.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1106,6 +1106,20 @@
# This setting may not be combined with the
# "safety_level" setting.
#
# page_size Valid values: integer (MUST be power of 2 between 512 and 65536)
# The default is 4096 bytes. This setting determines
# the size of a page in the transaction.db file.
# See https://www.sqlite.org/pragma.html#pragma_page_size
# for more details about the available options.
#
# journal_size_limit Valid values: integer
# The default is 1582080. This setting limits
# the size of the journal for transaction.db file. When the limit is
# reached, older entries will be deleted.
# See https://www.sqlite.org/pragma.html#pragma_journal_size_limit
# for more details about the available options.
#
#
#-------------------------------------------------------------------------------
#
# 7. Diagnostics
Expand Down
109 changes: 109 additions & 0 deletions src/test/nodestore/Database_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,115 @@ class Database_test : public TestBase
BEAST_EXPECT(found);
}
}
{
// N/A: Default values
Env env(*this);
auto const s = setup_DatabaseCon(env.app().config());
if (BEAST_EXPECT(s.txPragma.size() == 4))
{
BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=4096;");
BEAST_EXPECT(
s.txPragma.at(1) == "PRAGMA journal_size_limit=1582080;");
BEAST_EXPECT(
s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
BEAST_EXPECT(
s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
}
}
{
// Success: Valid values
Env env = [&]() {
auto p = test::jtx::envconfig();
{
auto& section = p->section("sqlite");
section.set("page_size", "512");
section.set("journal_size_limit", "2582080");
}
return Env(*this, std::move(p));
}();
auto const s = setup_DatabaseCon(env.app().config());
if (BEAST_EXPECT(s.txPragma.size() == 4))
{
BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=512;");
BEAST_EXPECT(
s.txPragma.at(1) == "PRAGMA journal_size_limit=2582080;");
BEAST_EXPECT(
s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
BEAST_EXPECT(
s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
}
}
{
// Error: Invalid values
auto const expected =
"Invalid page_size. Must be between 512 and 65536.";
bool found = false;
auto p = test::jtx::envconfig();
{
auto& section = p->section("sqlite");
section.set("page_size", "256");
}
try
{
Env env(
*this,
std::move(p),
std::make_unique<CheckMessageLogs>(expected, &found),
beast::severities::kWarning);
fail();
}
catch (...)
{
BEAST_EXPECT(found);
}
}
{
// Error: Invalid values
auto const expected =
"Invalid page_size. Must be between 512 and 65536.";
bool found = false;
auto p = test::jtx::envconfig();
{
auto& section = p->section("sqlite");
section.set("page_size", "131072");
}
try
{
Env env(
*this,
std::move(p),
std::make_unique<CheckMessageLogs>(expected, &found),
beast::severities::kWarning);
fail();
}
catch (...)
{
BEAST_EXPECT(found);
}
}
{
// Error: Invalid values
auto const expected = "Invalid page_size. Must be a power of 2.";
bool found = false;
auto p = test::jtx::envconfig();
{
auto& section = p->section("sqlite");
section.set("page_size", "513");
}
try
{
Env env(
*this,
std::move(p),
std::make_unique<CheckMessageLogs>(expected, &found),
beast::severities::kWarning);
fail();
}
catch (...)
{
BEAST_EXPECT(found);
}
}
}

//--------------------------------------------------------------------------
Expand Down
22 changes: 0 additions & 22 deletions src/xrpld/app/main/DBInit.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,6 @@ inline constexpr std::uint32_t SQLITE_TUNING_CUTOFF = 10'000'000;
// Ledger database holds ledgers and ledger confirmations
inline constexpr auto LgrDBName{"ledger.db"};

inline constexpr std::array<char const*, 1> LgrDBPragma{
{"PRAGMA journal_size_limit=1582080;"}};

inline constexpr std::array<char const*, 5> LgrDBInit{
{"BEGIN TRANSACTION;",

Expand Down Expand Up @@ -72,25 +69,6 @@ inline constexpr std::array<char const*, 5> LgrDBInit{
// Transaction database holds transactions and public keys
inline constexpr auto TxDBName{"transaction.db"};

// In C++17 omitting the explicit template parameters caused
// a crash
inline constexpr std::array<char const*, 4> TxDBPragma
{
"PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;",
"PRAGMA max_page_count=4294967294;",

#if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP)
"PRAGMA mmap_size=17179869184;"
#else

// Provide an explicit `no-op` SQL statement
// in order to keep the size of the array
// constant regardless of the preprocessor
// condition evaluation
"PRAGMA sqlite_noop_statement;"
#endif
};

inline constexpr std::array<char const*, 8> TxDBInit{
{"BEGIN TRANSACTION;",

Expand Down
4 changes: 2 additions & 2 deletions src/xrpld/app/rdb/backend/detail/Node.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ makeLedgerDBs(
{
// ledger database
auto lgr{std::make_unique<DatabaseCon>(
setup, LgrDBName, LgrDBPragma, LgrDBInit, checkpointerSetup, j)};
setup, LgrDBName, setup.lgrPragma, LgrDBInit, checkpointerSetup, j)};
lgr->getSession() << boost::str(
boost::format("PRAGMA cache_size=-%d;") %
kilobytes(config.getValueFor(SizedItem::lgrDBCache)));
Expand All @@ -81,7 +81,7 @@ makeLedgerDBs(
{
// transaction database
auto tx{std::make_unique<DatabaseCon>(
setup, TxDBName, TxDBPragma, TxDBInit, checkpointerSetup, j)};
setup, TxDBName, setup.txPragma, TxDBInit, checkpointerSetup, j)};
tx->getSession() << boost::str(
boost::format("PRAGMA cache_size=-%d;") %
kilobytes(config.getValueFor(SizedItem::txnDBCache)));
Expand Down
4 changes: 2 additions & 2 deletions src/xrpld/app/rdb/detail/Vacuum.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ doVacuumDB(DatabaseCon::Setup const& setup, beast::Journal j)
return false;
}

auto txnDB =
std::make_unique<DatabaseCon>(setup, TxDBName, TxDBPragma, TxDBInit, j);
auto txnDB = std::make_unique<DatabaseCon>(
setup, TxDBName, setup.txPragma, TxDBInit, j);
auto& session = txnDB->getSession();
std::uint32_t pageSize;

Expand Down
4 changes: 2 additions & 2 deletions src/xrpld/app/rdb/detail/Wallet.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ makeWalletDB(DatabaseCon::Setup const& setup, beast::Journal j)
{
// wallet database
return std::make_unique<DatabaseCon>(
setup, WalletDBName, std::array<char const*, 0>(), WalletDBInit, j);
setup, WalletDBName, std::array<std::string, 0>(), WalletDBInit, j);
}

std::unique_ptr<DatabaseCon>
Expand All @@ -38,7 +38,7 @@ makeTestWalletDB(
{
// wallet database
return std::make_unique<DatabaseCon>(
setup, dbname.data(), std::array<char const*, 0>(), WalletDBInit, j);
setup, dbname.data(), std::array<std::string, 0>(), WalletDBInit, j);
}

void
Expand Down
12 changes: 7 additions & 5 deletions src/xrpld/core/DatabaseCon.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,8 @@ class DatabaseCon
}

static std::unique_ptr<std::vector<std::string> const> globalPragma;
std::array<std::string, 4> txPragma;
std::array<std::string, 1> lgrPragma;
};

struct CheckpointerSetup
Expand All @@ -114,7 +116,7 @@ class DatabaseCon
DatabaseCon(
Setup const& setup,
std::string const& dbName,
std::array<char const*, N> const& pragma,
std::array<std::string, N> const& pragma,
std::array<char const*, M> const& initSQL,
beast::Journal journal)
// Use temporary files or regular DB files?
Expand All @@ -136,7 +138,7 @@ class DatabaseCon
DatabaseCon(
Setup const& setup,
std::string const& dbName,
std::array<char const*, N> const& pragma,
std::array<std::string, N> const& pragma,
std::array<char const*, M> const& initSQL,
CheckpointerSetup const& checkpointerSetup,
beast::Journal journal)
Expand All @@ -149,7 +151,7 @@ class DatabaseCon
DatabaseCon(
boost::filesystem::path const& dataDir,
std::string const& dbName,
std::array<char const*, N> const& pragma,
std::array<std::string, N> const& pragma,
std::array<char const*, M> const& initSQL,
beast::Journal journal)
: DatabaseCon(dataDir / dbName, nullptr, pragma, initSQL, journal)
Expand All @@ -161,7 +163,7 @@ class DatabaseCon
DatabaseCon(
boost::filesystem::path const& dataDir,
std::string const& dbName,
std::array<char const*, N> const& pragma,
std::array<std::string, N> const& pragma,
std::array<char const*, M> const& initSQL,
CheckpointerSetup const& checkpointerSetup,
beast::Journal journal)
Expand Down Expand Up @@ -199,7 +201,7 @@ class DatabaseCon
DatabaseCon(
boost::filesystem::path const& pPath,
std::vector<std::string> const* commonPragma,
std::array<char const*, N> const& pragma,
std::array<std::string, N> const& pragma,
std::array<char const*, M> const& initSQL,
beast::Journal journal)
: session_(std::make_shared<soci::session>()), j_(journal)
Expand Down
32 changes: 31 additions & 1 deletion src/xrpld/core/detail/DatabaseCon.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ setup_DatabaseCon(Config const& c, std::optional<beast::Journal> j)
}

{
//#synchronous Valid values : off, normal, full, extra
// #synchronous Valid values : off, normal, full, extra
if (set(synchronous, "synchronous", sqlite) &&
!safety_level.empty())
{
Expand Down Expand Up @@ -237,6 +237,36 @@ setup_DatabaseCon(Config const& c, std::optional<beast::Journal> j)
}
setup.useGlobalPragma = true;

auto setPragma =
[](std::string& pragma, std::string const& key, int64_t value) {
pragma = "PRAGMA " + key + "=" + std::to_string(value) + ";";
};

// Lgr Pragma
setPragma(setup.lgrPragma[0], "journal_size_limit", 1582080);

// TX Pragma
int64_t page_size = 4096;
int64_t journal_size_limit = 1582080;
if (c.exists("sqlite"))
{
auto& s = c.section("sqlite");
set(journal_size_limit, "journal_size_limit", s);
set(page_size, "page_size", s);
if (page_size < 512 || page_size > 65536)
Throw<std::runtime_error>(
"Invalid page_size. Must be between 512 and 65536.");

if (page_size & (page_size - 1))
Throw<std::runtime_error>(
"Invalid page_size. Must be a power of 2.");
}

setPragma(setup.txPragma[0], "page_size", page_size);
setPragma(setup.txPragma[1], "journal_size_limit", journal_size_limit);
setPragma(setup.txPragma[2], "max_page_count", 4294967294);
setPragma(setup.txPragma[3], "mmap_size", 17179869184);

return setup;
}

Expand Down

0 comments on commit a753099

Please sign in to comment.