Skip to content

Commit

Permalink
issue-95: implemented average compaction thresholds (#743)
Browse files Browse the repository at this point in the history 
* issue-95: implemented average compaction thresholds

* issue-95: average compaction thresholds - fixed unused mem usage in ut

* issue-95: average compaction thresholds - fixed uts after rebase
  • Loading branch information
@qkrorlqr
qkrorlqr authored Mar 14, 2024
1 parent f271bcc commit 76ecb5a
Show file tree
Hide file tree
Showing 6 changed files with 243 additions and 14 deletions.
10 changes: 10 additions & 0 deletions cloud/filestore/config/storage.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,4 +216,14 @@ message TStorageConfig
optional uint32 EntryTimeout = 335;
optional uint32 NegativeEntryTimeout = 336;
optional uint32 AttrTimeout = 337;

// Threshold for the number of garbage blocks in a compaction range that
// triggers automatic compaction.
optional uint32 GarbageCompactionThreshold = 338;
// Threshold for average CompactionScore for the whole FS.
optional uint32 CompactionThresholdAverage = 339;
// Threshold for average GarbageCompactionScore for the whole FS.
optional uint32 GarbageCompactionThresholdAverage = 340;
// Enables 3 aforementioned thresholds.
optional bool NewCompactionEnabled = 341;
}
26 changes: 15 additions & 11 deletions cloud/filestore/libs/storage/core/config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,21 @@ namespace {
xxx(WriteBatchTimeout, TDuration, TDuration::MilliSeconds(0) )\
xxx(WriteBlobThreshold, ui32, 128_KB )\
\
xxx(MaxBlobSize, ui32, 4_MB )\
xxx(FlushThreshold, ui32, 4_MB )\
xxx(CleanupThreshold, ui32, 512 )\
xxx(CompactionThreshold, ui32, 20 )\
xxx(CollectGarbageThreshold, ui32, 4_MB )\
xxx(FlushBytesThreshold, ui32, 4_MB )\
xxx(MaxDeleteGarbageBlobsPerTx, ui32, 16384 )\
xxx(LoadedCompactionRangesPerTx, ui32, 10 * 1024 * 1024 )\
xxx(MaxBlocksPerTruncateTx, ui32, 0 /*TODO: 8388608 32gb/4kb*/)\
xxx(MaxTruncateTxInflight, ui32, 10 )\
xxx(CompactionRetryTimeout, TDuration, TDuration::Seconds(1) )\
xxx(MaxBlobSize, ui32, 4_MB )\
xxx(FlushThreshold, ui32, 4_MB )\
xxx(CleanupThreshold, ui32, 512 )\
xxx(CompactionThreshold, ui32, 20 )\
xxx(GarbageCompactionThreshold, ui32, 100 )\
xxx(CompactionThresholdAverage, ui32, 4 )\
xxx(GarbageCompactionThresholdAverage, ui32, 20 )\
xxx(NewCompactionEnabled, bool, false )\
xxx(CollectGarbageThreshold, ui32, 4_MB )\
xxx(FlushBytesThreshold, ui32, 4_MB )\
xxx(MaxDeleteGarbageBlobsPerTx, ui32, 16384 )\
xxx(LoadedCompactionRangesPerTx, ui32, 10 * 1024 * 1024 )\
xxx(MaxBlocksPerTruncateTx, ui32, 0 /*TODO: 32GiB/4KiB*/ )\
xxx(MaxTruncateTxInflight, ui32, 10 )\
xxx(CompactionRetryTimeout, TDuration, TDuration::Seconds(1) )\
\
xxx(FlushThresholdForBackpressure, ui32, 128_MB )\
xxx(CleanupThresholdForBackpressure, ui32, 32768 )\
Expand Down
4 changes: 4 additions & 0 deletions cloud/filestore/libs/storage/core/config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ class TStorageConfig
ui32 GetFlushThreshold() const;
ui32 GetCleanupThreshold() const;
ui32 GetCompactionThreshold() const;
ui32 GetGarbageCompactionThreshold() const;
ui32 GetCompactionThresholdAverage() const;
ui32 GetGarbageCompactionThresholdAverage() const;
bool GetNewCompactionEnabled() const;
ui32 GetCollectGarbageThreshold() const;
ui32 GetFlushBytesThreshold() const;
ui32 GetMaxDeleteGarbageBlobsPerTx() const;
Expand Down
9 changes: 7 additions & 2 deletions cloud/filestore/libs/storage/tablet/model/compaction_map.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -449,10 +449,15 @@ TCompactionMapStats TCompactionMap::GetStats(ui32 topSize) const
TCompactionMapStats stats = {
.UsedRangesCount = Impl->UsedRangesCount,
.AllocatedRangesCount = Impl->Groups.Size() * TCompactionMap::GroupSize,
.TopRangesByCleanupScore = GetTopRangesByCleanupScore(topSize),
.TopRangesByCompactionScore = GetTopRangesByCompactionScore(topSize),
};

if (!topSize) {
return stats;
}

stats.TopRangesByCleanupScore = GetTopRangesByCleanupScore(topSize);
stats.TopRangesByCompactionScore = GetTopRangesByCompactionScore(topSize);

return stats;
}

Expand Down
25 changes: 25 additions & 0 deletions cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,31 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx)
if (BlobIndexOps.Empty()) {
if (compactionScore >= Config->GetCompactionThreshold()) {
BlobIndexOps.Push(EBlobIndexOp::Compaction);
} else if (Config->GetNewCompactionEnabled()) {
const auto& stats = GetFileSystemStats();
const auto compactionStats = GetCompactionMapStats(0);
const auto used = stats.GetUsedBlocksCount();
auto alive = stats.GetMixedBlocksCount();
if (alive > stats.GetGarbageBlocksCount()) {
alive -= stats.GetGarbageBlocksCount();
} else {
alive = 0;
}
const auto avgGarbagePercentage = used && alive > used
? 100 * static_cast<double>(alive - used) / used
: 0;
const auto rangeCount = compactionStats.UsedRangesCount;
const auto avgCompactionScore = rangeCount
? static_cast<double>(stats.GetMixedBlobsCount()) / rangeCount
: 0;
// TODO: use GarbageCompactionThreshold

const bool shouldCompact =
avgGarbagePercentage >= Config->GetGarbageCompactionThresholdAverage()
|| avgCompactionScore >= Config->GetCompactionThresholdAverage();
if (compactionScore > 1 && shouldCompact) {
BlobIndexOps.Push(EBlobIndexOp::Compaction);
}
}

if (cleanupScore >= Config->GetCleanupThreshold()) {
Expand Down
183 changes: 182 additions & 1 deletion cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,28 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data)

using namespace NCloud::NStorage;

#define TABLET_TEST_IMPL(name, largeBS) \
#define TABLET_TEST_HEAD(name) \
void TestImpl##name(TFileSystemConfig tabletConfig); \
Y_UNIT_TEST(name) \
{ \
TestImpl##name(TFileSystemConfig{.BlockSize = 4_KB}); \
} \
// TABLET_TEST_HEAD

#define TABLET_TEST_IMPL(name, largeBS) \
TABLET_TEST_HEAD(name) \
Y_UNIT_TEST(name##largeBS) \
{ \
TestImpl##name(TFileSystemConfig{.BlockSize = largeBS}); \
} \
void TestImpl##name(TFileSystemConfig tabletConfig) \
// TABLET_TEST_IMPL

#define TABLET_TEST_4K_ONLY(name) \
TABLET_TEST_HEAD(name) \
void TestImpl##name(TFileSystemConfig tabletConfig) \
// TABLET_TEST_4K_ONLY

#define TABLET_TEST(name) \
TABLET_TEST_IMPL(name, 128_KB) \
// TABLET_TEST
Expand Down Expand Up @@ -1440,6 +1449,178 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data)
tablet.DestroyHandle(handle);
}

TABLET_TEST_4K_ONLY(ShouldAutomaticallyRunCompactionDueToCompactionThresholdAverage)
{
const auto block = tabletConfig.BlockSize;

NProto::TStorageConfig storageConfig;
storageConfig.SetNewCompactionEnabled(true);
storageConfig.SetCompactionThresholdAverage(2);
storageConfig.SetGarbageCompactionThresholdAverage(999'999);
storageConfig.SetCompactionThreshold(999'999);
storageConfig.SetCleanupThreshold(999'999);
storageConfig.SetWriteBlobThreshold(block);

TTestEnv env({}, std::move(storageConfig));
env.CreateSubDomain("nfs");

ui32 nodeIdx = env.CreateNode("nfs");
ui64 tabletId = env.BootIndexTablet(nodeIdx);

TIndexTabletClient tablet(
env.GetRuntime(),
nodeIdx,
tabletId,
tabletConfig);
tablet.InitSession("client", "session");

auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test"));
auto handle = CreateHandle(tablet, id);

// allocating 4 compaction ranges
TSetNodeAttrArgs args(id);
args.SetFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE);
args.SetSize(1_MB);
tablet.SetNodeAttr(args);

// 3 blobs in 3 different ranges
tablet.WriteData(handle, 0, block, 'a');
tablet.WriteData(handle, 256_KB, block, 'a');
tablet.WriteData(handle, 512_KB, block, 'a');

{
auto response = tablet.GetStorageStats();
const auto& stats = response->Record.GetStats();
UNIT_ASSERT_VALUES_EQUAL(3, stats.GetMixedBlobsCount());
UNIT_ASSERT_VALUES_EQUAL(3, stats.GetMixedBlocksCount());
UNIT_ASSERT_VALUES_EQUAL(1_MB / block, stats.GetUsedBlocksCount());
UNIT_ASSERT_VALUES_EQUAL(3, stats.GetUsedCompactionRanges());
// GroupSize (64) * ranges needed for our file (1_MB / 256_KB)
UNIT_ASSERT_VALUES_EQUAL(256, stats.GetAllocatedCompactionRanges());
}

// 3 more blobs in 3 different ranges
tablet.WriteData(handle, block, block, 'b');
tablet.WriteData(handle, 256_KB + block, block, 'b');
tablet.WriteData(handle, 512_KB + block, block, 'b');

// average CompactionScore per used range became 2 => Compaction
// should've been triggered for one of the ranges and should've
// decreased its blob count from 2 to 1 => we should have 5 blobs
// now
{
auto response = tablet.GetStorageStats();
const auto& stats = response->Record.GetStats();
UNIT_ASSERT_VALUES_EQUAL(5, stats.GetMixedBlobsCount());
UNIT_ASSERT_VALUES_EQUAL(6, stats.GetMixedBlocksCount());
}

TString expected(1_MB, 0);
memset(expected.begin(), 'a', block);
memset(expected.begin() + block, 'b', block);
memset(expected.begin() + 256_KB, 'a', block);
memset(expected.begin() + 256_KB + block, 'b', block);
memset(expected.begin() + 512_KB, 'a', block);
memset(expected.begin() + 512_KB + block, 'b', block);

{
auto response = tablet.ReadData(handle, 0, 1_MB);
const auto& buffer = response->Record.GetBuffer();
UNIT_ASSERT_EQUAL(expected, buffer);
}

tablet.DestroyHandle(handle);
}

TABLET_TEST_4K_ONLY(ShouldAutomaticallyRunCompactionDueToGarbageCompactionThresholdAverage)
{
const auto block = tabletConfig.BlockSize;

NProto::TStorageConfig storageConfig;
storageConfig.SetNewCompactionEnabled(true);
storageConfig.SetCompactionThresholdAverage(999'999);
storageConfig.SetGarbageCompactionThresholdAverage(20);
storageConfig.SetCompactionThreshold(999'999);
storageConfig.SetCleanupThreshold(999'999);
storageConfig.SetWriteBlobThreshold(block);

TTestEnv env({}, std::move(storageConfig));
env.CreateSubDomain("nfs");

ui32 nodeIdx = env.CreateNode("nfs");
ui64 tabletId = env.BootIndexTablet(nodeIdx);

TIndexTabletClient tablet(
env.GetRuntime(),
nodeIdx,
tabletId,
tabletConfig);
tablet.InitSession("client", "session");

auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test"));
auto handle = CreateHandle(tablet, id);

// allocating 4 compaction ranges
TSetNodeAttrArgs args(id);
args.SetFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE);
args.SetSize(1_MB);
tablet.SetNodeAttr(args);

// 768_KB in 3 different ranges
tablet.WriteData(handle, 0, 256_KB, 'a');
tablet.WriteData(handle, 256_KB, 256_KB, 'a');
tablet.WriteData(handle, 512_KB, 256_KB, 'a');

{
auto response = tablet.GetStorageStats();
const auto& stats = response->Record.GetStats();
UNIT_ASSERT_VALUES_EQUAL(3, stats.GetMixedBlobsCount());
UNIT_ASSERT_VALUES_EQUAL(768_KB / block, stats.GetMixedBlocksCount());
UNIT_ASSERT_VALUES_EQUAL(1_MB / block, stats.GetUsedBlocksCount());
UNIT_ASSERT_VALUES_EQUAL(0, stats.GetGarbageBlocksCount());
UNIT_ASSERT_VALUES_EQUAL(3, stats.GetUsedCompactionRanges());
// GroupSize (64) * ranges needed for our file (1_MB / 256_KB)
UNIT_ASSERT_VALUES_EQUAL(256, stats.GetAllocatedCompactionRanges());
}

// 512_KB more
tablet.WriteData(handle, 0, 256_KB, 'b');
tablet.WriteData(handle, 256_KB, 256_KB, 'b');

// garbage fraction became 1280_KB / 1024_KB > 1.2 => Compaction
// should've been triggered for one of the ranges and should've
// decreased its blob count from 2 to 1 => we should have 5 blobs
// byte count in that range should've been decreased from 512_KB to
// 256_KB
{
auto response = tablet.GetStorageStats();
const auto& stats = response->Record.GetStats();
UNIT_ASSERT_VALUES_EQUAL(4, stats.GetMixedBlobsCount());
// now we have 1_MB of data
UNIT_ASSERT_VALUES_EQUAL(
1_MB / block,
stats.GetMixedBlocksCount());
// 0 garbage blocks
UNIT_ASSERT_VALUES_EQUAL(0, stats.GetGarbageBlocksCount());
// 1280_KB new (written by user)
// 256_KB new (rewritten by Compaction)
// 512_KB garbage (marked after Compaction)
UNIT_ASSERT_VALUES_EQUAL(2_MB, stats.GetGarbageQueueSize());
}

TString expected(1_MB, 0);
memset(expected.begin(), 'b', 512_KB);
memset(expected.begin() + 512_KB, 'a', 256_KB);

{
auto response = tablet.ReadData(handle, 0, 1_MB);
const auto& buffer = response->Record.GetBuffer();
UNIT_ASSERT_EQUAL(expected, buffer);
}

tablet.DestroyHandle(handle);
}

TABLET_TEST(ShouldAutomaticallyRunCleanup)
{
const auto block = tabletConfig.BlockSize;
Expand Down

0 comments on commit 76ecb5a

Please sign in to comment.