NEBIUSNBS-70: implemented ability to get compaction map range stats v…

…ia private api, added some uts regarding compaction map initialization (#137) * NEBIUSNBS-70: implemented ability to get compaction map range stats via private api, added some uts regarding compaction map initialization * NEBIUSNBS-70: implemented ability to get compaction map range stats via private api, added some uts regarding compaction map initialization - removed unneeded include
ydb-platform · Jan 12, 2024 · 36f0325 · 36f0325
1 parent cd360c1
commit 36f0325
Show file tree

Hide file tree

Showing 5 changed files with 150 additions and 12 deletions.
diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp
@@ -328,6 +328,30 @@ void TIndexTabletActor::HandleGetStorageStats(
     stats->SetUsedCompactionRanges(cmStats.UsedRangesCount);
     stats->SetAllocatedCompactionRanges(cmStats.AllocatedRangesCount);
 
+    const auto& req = ev->Get()->Record;
+
+    if (req.GetCompactionRangeCountByCompactionScore()) {
+        const auto topRanges = GetTopRangesByCompactionScore(
+            req.GetCompactionRangeCountByCompactionScore());
+        for (const auto& r: topRanges) {
+            auto* out = stats->AddCompactionRangeStats();
+            out->SetRangeId(r.RangeId);
+            out->SetBlobCount(r.Stats.BlobsCount);
+            out->SetDeletionCount(r.Stats.DeletionsCount);
+        }
+    }
+
+    if (req.GetCompactionRangeCountByCleanupScore()) {
+        const auto topRanges = GetTopRangesByCleanupScore(
+            req.GetCompactionRangeCountByCleanupScore());
+        for (const auto& r: topRanges) {
+            auto* out = stats->AddCompactionRangeStats();
+            out->SetRangeId(r.RangeId);
+            out->SetBlobCount(r.Stats.BlobsCount);
+            out->SetDeletionCount(r.Stats.DeletionsCount);
+        }
+    }
+
     NCloud::Reply(ctx, *ev, std::move(response));
 }
 

diff --git a/cloud/filestore/libs/storage/tablet/tablet_database_ut.cpp b/cloud/filestore/libs/storage/tablet/tablet_database_ut.cpp
@@ -344,6 +344,74 @@ Y_UNIT_TEST_SUITE(TIndexTabletDatabaseTest)
             UNIT_ASSERT(!serviceConfig->HasSSDSystemChannelPoolKind());
         });
     }
+
+    Y_UNIT_TEST(ShouldStoreCompactionMap)
+    {
+        TTestExecutor executor;
+        executor.WriteTx([&] (TIndexTabletDatabase db) {
+            db.InitSchema();
+        });
+
+        using TEntries = TVector<TCompactionRangeInfo>;
+        TEntries entries = {
+            {1, {50, 100}},
+            {4, {42, 10}},
+            {32, {50, 200}},
+            {65, {1, 400}},
+            {110, {2, 333}},
+            {113, {7, 444}},
+            {4233, {150, 555}},
+            {5632, {1000, 3}},
+            {6000, {30, 11}},
+            {6001, {12, 15}},
+            {6002, {2, 20}},
+            {6005, {99, 7}},
+            {7000, {1000, 5000}},
+        };
+
+        executor.WriteTx([&] (TIndexTabletDatabase db) {
+            for (const auto& entry: entries) {
+                db.WriteCompactionMap(
+                    entry.RangeId,
+                    entry.Stats.BlobsCount,
+                    entry.Stats.DeletionsCount);
+            }
+        });
+
+        auto toString = [] (const TEntries& v, ui32 i = 0, ui32 c = Max<ui32>()) {
+            TStringBuilder sb;
+            ui32 processed = 0;
+            while (i < v.size() && processed < c) {
+                if (processed) {
+                    sb << " ";
+                }
+                sb << v[i].RangeId
+                    << "," << v[i].Stats.BlobsCount
+                    << "," << v[i].Stats.DeletionsCount;
+                ++processed;
+                ++i;
+            }
+            return sb;
+        };
+
+        executor.ReadTx([&] (TIndexTabletDatabase db) {
+            TVector<TCompactionRangeInfo> chunk;
+            UNIT_ASSERT(db.ReadCompactionMap(chunk, 0, 5));
+            UNIT_ASSERT_VALUES_EQUAL(toString(entries, 0, 5), toString(chunk));
+
+            chunk.clear();
+            UNIT_ASSERT(db.ReadCompactionMap(chunk, 111, 5));
+            UNIT_ASSERT_VALUES_EQUAL(toString(entries, 5, 5), toString(chunk));
+
+            chunk.clear();
+            UNIT_ASSERT(db.ReadCompactionMap(chunk, 6002, 5));
+            UNIT_ASSERT_VALUES_EQUAL(toString(entries, 10, 5), toString(chunk));
+
+            chunk.clear();
+            UNIT_ASSERT(db.ReadCompactionMap(chunk, 7001, 5));
+            UNIT_ASSERT_VALUES_EQUAL("", toString(chunk));
+        });
+    }
 }
 
 }   // namespace NCloud::NFileStore::NStorage
diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp
@@ -2819,12 +2819,46 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data)
             tablet.WriteData(handle1, 0, 2 * BlockGroupSize * block, 'x');
         }
 
-        {
-            auto response = tablet.GetStorageStats();
+        auto checkCompactionMap = [&] () {
+            auto response = tablet.GetStorageStats(10);
             const auto& stats = response->Record.GetStats();
             UNIT_ASSERT_VALUES_EQUAL(8, stats.GetUsedCompactionRanges());
             UNIT_ASSERT_VALUES_EQUAL(1024, stats.GetAllocatedCompactionRanges());
-        }
+            UNIT_ASSERT_VALUES_EQUAL(8, stats.CompactionRangeStatsSize());
+            auto rangeToString = [] (const NProtoPrivate::TCompactionRangeStats& rs) {
+                return Sprintf(
+                    "r=%u b=%u d=%u",
+                    rs.GetRangeId(),
+                    rs.GetBlobCount(),
+                    rs.GetDeletionCount());
+            };
+            UNIT_ASSERT_VALUES_EQUAL(
+                "r=1656356864 b=16 d=1024",
+                rangeToString(stats.GetCompactionRangeStats(0)));
+            UNIT_ASSERT_VALUES_EQUAL(
+                "r=1656356865 b=16 d=1024",
+                rangeToString(stats.GetCompactionRangeStats(1)));
+            UNIT_ASSERT_VALUES_EQUAL(
+                "r=4283236352 b=16 d=1024",
+                rangeToString(stats.GetCompactionRangeStats(2)));
+            UNIT_ASSERT_VALUES_EQUAL(
+                "r=4283236353 b=16 d=1024",
+                rangeToString(stats.GetCompactionRangeStats(3)));
+            UNIT_ASSERT_VALUES_EQUAL(
+                "r=1177944064 b=14 d=833",
+                rangeToString(stats.GetCompactionRangeStats(4)));
+            UNIT_ASSERT_VALUES_EQUAL(
+                "r=1177944065 b=13 d=832",
+                rangeToString(stats.GetCompactionRangeStats(5)));
+            UNIT_ASSERT_VALUES_EQUAL(
+                "r=737148928 b=3 d=192",
+                rangeToString(stats.GetCompactionRangeStats(6)));
+            UNIT_ASSERT_VALUES_EQUAL(
+                "r=737148929 b=3 d=192",
+                rangeToString(stats.GetCompactionRangeStats(7)));
+        };
+
+        checkCompactionMap();
 
         tablet.RebootTablet();
         tablet.RecoverSession();
@@ -2843,12 +2877,7 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data)
 
         UNIT_ASSERT_VALUES_EQUAL(7, loadChunkCount);
 
-        {
-            auto response = tablet.GetStorageStats();
-            const auto& stats = response->Record.GetStats();
-            UNIT_ASSERT_VALUES_EQUAL(8, stats.GetUsedCompactionRanges());
-            UNIT_ASSERT_VALUES_EQUAL(1024, stats.GetAllocatedCompactionRanges());
-        }
+        checkCompactionMap();
 
         tablet.DestroyHandle(handle);
     }

diff --git a/cloud/filestore/libs/storage/testlib/tablet_client.h b/cloud/filestore/libs/storage/testlib/tablet_client.h
@@ -208,9 +208,11 @@ class TIndexTabletClient
         return std::make_unique<TEvIndexTablet::TEvWaitReadyRequest>();
     }
 
-    auto CreateGetStorageStatsRequest()
+    auto CreateGetStorageStatsRequest(ui32 compactionRangeCount = 0)
     {
-        return std::make_unique<TEvIndexTablet::TEvGetStorageStatsRequest>();
+        auto request = std::make_unique<TEvIndexTablet::TEvGetStorageStatsRequest>();
+        request->Record.SetCompactionRangeCountByCompactionScore(compactionRangeCount);
+        return request;
     }
 
     auto CreateGetFileSystemConfigRequest()

diff --git a/cloud/filestore/private/api/protos/tablet.proto b/cloud/filestore/private/api/protos/tablet.proto
@@ -86,6 +86,13 @@ message TDestroySessionResponse
 ////////////////////////////////////////////////////////////////////////////////
 // GetStorageStats request/response.
 
+message TCompactionRangeStats
+{
+    uint32 RangeId = 1;
+    uint32 BlobCount = 2;
+    uint32 DeletionCount = 3;
+}
+
 message TStorageStats
 {
     // index stats
@@ -115,6 +122,9 @@ message TStorageStats
 
     // tx stats
     uint64 TxDeleteGarbageRwCompleted = 2000;
+
+    // compaction map range stats
+    repeated TCompactionRangeStats CompactionRangeStats = 3000;
 }
 
 message TGetStorageStatsRequest
@@ -124,6 +134,11 @@ message TGetStorageStatsRequest
 
     // FileSystem identifier.
     string FileSystemId = 2;
+
+    // If nonzero, requests the stats for the top N ranges by compaction score.
+    uint32 CompactionRangeCountByCompactionScore = 3;
+    // The same but for cleanup score.
+    uint32 CompactionRangeCountByCleanupScore = 4;
 }
 
 message TGetStorageStatsResponse
@@ -203,7 +218,7 @@ message TGetStorageConfigFieldsRequest
 
     // FileSystem identifier.
     string FileSystemId = 2;
-    
+
     // Storage config fields to get values for.
     repeated string StorageConfigFields = 3;
 }