[Filestore] Implement throttling for DescribeData and GenerateBlobIds…

… requests (#1838) * [Filestore] Implement throttling for DescribeData and GenerateBlobIds requests * No second throttle * Fixes * fix * Fix * Fix * By option * clean * Copy-paste test * Fix * clean * Clean * clean * Fix merge
ydb-platform · Sep 13, 2024 · ab31ef0 · ab31ef0
1 parent 29ceb0c
commit ab31ef0
Show file tree

Hide file tree

Showing 10 changed files with 158 additions and 5 deletions.
diff --git a/cloud/filestore/config/storage.proto b/cloud/filestore/config/storage.proto
@@ -387,4 +387,7 @@ message TStorageConfig
     // If the number of blocks marked for deletion via large deletion markers
     // exceeds this threshold, Cleanup will be triggered.
     optional uint64 LargeDeletionMarkersCleanupThreshold = 388;
+
+    // Throttle DescribeData and GenerateBlobIds requests
+    optional bool MultipleStageRequestThrottlingEnabled = 389;
 }
diff --git a/cloud/filestore/libs/storage/core/config.cpp b/cloud/filestore/libs/storage/core/config.cpp
@@ -203,6 +203,8 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
     xxx(NodeRegistrationMaxAttempts,         ui32,      10                    )\
     xxx(NodeRegistrationTimeout,             TDuration, TDuration::Seconds(5) )\
     xxx(NodeRegistrationErrorTimeout,        TDuration, TDuration::Seconds(1) )\
+                                                                               \
+    xxx(MultipleStageRequestThrottlingEnabled,          bool,      false      )\
 // FILESTORE_STORAGE_CONFIG
 
 #define FILESTORE_STORAGE_CONFIG_REF(xxx)                                      \

diff --git a/cloud/filestore/libs/storage/core/config.h b/cloud/filestore/libs/storage/core/config.h
@@ -264,6 +264,8 @@ class TStorageConfig
     ui64 GetLargeDeletionMarkerBlocks() const;
     ui64 GetLargeDeletionMarkersThreshold() const;
     ui64 GetLargeDeletionMarkersCleanupThreshold() const;
+
+    bool GetMultipleStageRequestThrottlingEnabled() const;
 };
 
 }   // namespace NCloud::NFileStore::NStorage
diff --git a/cloud/filestore/libs/storage/service/service_actor_readdata.cpp b/cloud/filestore/libs/storage/service/service_actor_readdata.cpp
@@ -503,6 +503,7 @@ void TReadDataActor::ReadData(
 
     auto request = std::make_unique<TEvService::TEvReadDataRequest>();
     request->Record = std::move(ReadRequest);
+    request->Record.MutableHeaders()->SetThrottlingDisabled(true);
 
     // forward request through tablet proxy
     ctx.Send(MakeIndexTabletProxyServiceId(), request.release());

diff --git a/cloud/filestore/libs/storage/service/service_actor_writedata.cpp b/cloud/filestore/libs/storage/service/service_actor_writedata.cpp
@@ -379,6 +379,7 @@ class TWriteDataActor final: public TActorBootstrapped<TWriteDataActor>
 
         auto request = std::make_unique<TEvService::TEvWriteDataRequest>();
         request->Record = std::move(WriteRequest);
+        request->Record.MutableHeaders()->SetThrottlingDisabled(true);
 
         // forward request through tablet proxy
         ctx.Send(MakeIndexTabletProxyServiceId(), request.release());

diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp
@@ -253,6 +253,12 @@ void TIndexTabletActor::HandleGenerateBlobIds(
         return;
     }
 
+    if (Config->GetMultipleStageRequestThrottlingEnabled() &&
+        ThrottleIfNeeded<TEvIndexTablet::TGenerateBlobIdsMethod>(ev, ctx))
+    {
+        return;
+    }
+
     // We schedule this event for the case if the client does not call AddData.
     // Thus we ensure that the collect barrier will be released eventually.
     ctx.Schedule(

diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp
@@ -617,6 +617,12 @@ void TIndexTabletActor::HandleDescribeData(
         return;
     }
 
+    if (Config->GetMultipleStageRequestThrottlingEnabled() &&
+        ThrottleIfNeeded<TEvIndexTablet::TDescribeDataMethod>(ev, ctx))
+    {
+        return;
+    }
+
     auto* msg = ev->Get();
     const TByteRange byteRange(
         msg->Record.GetOffset(),

diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_throttling.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_throttling.cpp
@@ -61,6 +61,30 @@ TThrottlingRequestInfo BuildRequestInfo(
     };
 }
 
+template <>
+TThrottlingRequestInfo BuildRequestInfo(
+    const TEvIndexTablet::TEvDescribeDataRequest& request,
+    ui32 policyVersion)
+{
+    return {
+        static_cast<ui32>(CalculateByteCount(request.Record)),
+        static_cast<ui32>(TThrottlingPolicy::EOpType::Read),
+        policyVersion
+    };
+}
+
+template <>
+TThrottlingRequestInfo BuildRequestInfo(
+    const TEvIndexTablet::TEvGenerateBlobIdsRequest& request,
+    ui32 policyVersion)
+{
+    return {
+        static_cast<ui32>(CalculateByteCount(request.Record)),
+        static_cast<ui32>(TThrottlingPolicy::EOpType::Write),
+        policyVersion
+    };
+}
+
 }   // namespace
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -146,7 +170,8 @@ bool TIndexTabletActor::ThrottleIfNeeded(
     const NActors::TActorContext& ctx)
 {
     if (!Config->GetThrottlingEnabled() ||
-        !GetPerformanceProfile().GetThrottlingEnabled())
+        !GetPerformanceProfile().GetThrottlingEnabled() ||
+        ev->Get()->Record.GetHeaders().GetThrottlingDisabled())
     {
         return false;
     }
@@ -185,6 +210,8 @@ template bool TIndexTabletActor::ThrottleIfNeeded<ns::T##name##Method>(        \
 
 GENERATE_IMPL(ReadData,  TEvService)
 GENERATE_IMPL(WriteData, TEvService)
+GENERATE_IMPL(DescribeData, TEvIndexTablet)
+GENERATE_IMPL(GenerateBlobIds, TEvIndexTablet)
 
 #undef GENERATE_IMPL
 

diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_throttling.cpp
@@ -22,6 +22,7 @@ struct TTablet
     std::unique_ptr<TTestEnv> Env = nullptr;
     std::unique_ptr<TIndexTabletClient> Tablet = nullptr;
     ui64 Handle = 0;
+    ui64 NodeId = 0;
 
 public:
     void SetUp(NUnitTest::TTestContext& /*context*/) override
@@ -45,6 +46,8 @@ struct TTablet
         NProto::TStorageConfig storageConfig;
         if (throttlingEnabled.Defined()) {
             storageConfig.SetThrottlingEnabled(throttlingEnabled.GetRef());
+            storageConfig.SetMultipleStageRequestThrottlingEnabled(
+                throttlingEnabled.GetRef());
         }
 
         Env = std::make_unique<TTestEnv>(envConfig, storageConfig);
@@ -58,9 +61,8 @@ struct TTablet
             tabletId);
         Tablet->InitSession("client", "session");
 
-        const auto nodeId =
-            CreateNode(*Tablet, TCreateNodeArgs::File(RootNodeId, "test"));
-        Handle = CreateHandle(*Tablet, nodeId);
+        NodeId = CreateNode(*Tablet, TCreateNodeArgs::File(RootNodeId, "test"));
+        Handle = CreateHandle(*Tablet, NodeId);
     }
 
     auto UpdateConfig(const TFileSystemConfig& config)
@@ -78,6 +80,16 @@ struct TTablet
         Tablet->SendReadDataRequest(Handle, offset, len);
     }
 
+    void DescribeData(ui64 offset, ui32 len)
+    {
+        Tablet->SendDescribeDataRequest(Handle, offset, len);
+    }
+
+    void GenerateBlobIds(ui64 offset, ui32 len)
+    {
+        Tablet->SendGenerateBlobIdsRequest(NodeId, Handle, offset, len);
+    }
+
     void Tick(TDuration duration)
     {
         Tablet->AdvanceTime(duration);
@@ -108,6 +120,8 @@ struct TTablet
 
     TEST_CLIENT_DECLARE_METHOD(ReadData);
     TEST_CLIENT_DECLARE_METHOD(WriteData);
+    TEST_CLIENT_DECLARE_METHOD(DescribeData);
+    TEST_CLIENT_DECLARE_METHOD(GenerateBlobIds);
 
 #undef TEST_CLIENT_DECLARE_METHOD
 
@@ -219,7 +233,7 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Throttling)
 
         // 2. Testing that we start rejecting requests after
         // our postponed limit saturates.
-        WriteData(0, 1, 'y');   // Event 1 byte request must be rejected.
+        WriteData(0, 1, 'y');   // Even 1 byte request must be rejected.
         AssertWriteDataQuickResponse(E_FS_THROTTLED);
         ReadData(0, 1_KB);
         AssertReadDataQuickResponse(E_FS_THROTTLED);
@@ -281,6 +295,94 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Throttling)
         // TODO: 7. Test max count (NBS-2278).
     }
 
+    Y_UNIT_TEST_F(ShouldThrottleMultipleStage, TTablet)
+    {
+        const auto config = MakeThrottlerConfig(
+            true,                                    // throttlingEnabled
+            2,                                       // maxReadIops
+            2,                                       // maxWriteIops
+            8_KB,                                    // maxReadBandwidth
+            8_KB,                                    // maxWriteBandwidth
+            TDuration::Minutes(30).MilliSeconds(),   // boostTime
+            TDuration::Hours(12).MilliSeconds(),     // boostRefillTime
+            10,                                      // boostPercentage
+            32_KB,                                   // maxPostponedWeight
+            10,                                      // maxWriteCostMultiplier
+            TDuration::Seconds(25).MilliSeconds(),   // maxPostponedTime
+            64,                                      // maxPostponedCount
+            100,                                     // burstPercentage
+            1_KB   // defaultPostponedRequestWeight
+        );
+        UpdateConfig(config);
+
+        // 0. Testing that at 1rps nothing is throttled.
+        for (size_t i = 0; i < 10; ++i) {
+            Tick(TDuration::Seconds(1));
+            DescribeData(4_KB * i, 4_KB);
+            AssertDescribeDataQuickResponse(S_OK);
+            Tick(TDuration::Seconds(1));
+            GenerateBlobIds(4_KB * (i + 1), 4_KB);
+            AssertGenerateBlobIdsQuickResponse(S_OK);
+        }
+
+        // 1. Testing that excess requests are postponed.
+        for (size_t i = 0; i < 20; ++i) {
+            DescribeData(4_KB * i, 4_KB);
+            AssertDescribeDataNoResponse();
+        }
+
+        // Now we have 20_KB in PostponeQueue.
+
+        for (size_t i = 0; i < 3; ++i) {
+            GenerateBlobIds(0, 4_KB);
+            AssertGenerateBlobIdsNoResponse();
+        }
+
+        // Now we have 32_KB in PostponedQueue. Equal to MaxPostponedWeight.
+
+        // 2. Testing that we start rejecting requests after
+        // our postponed limit saturates.
+
+        GenerateBlobIds(0, config.BlockSize);   // Request must be rejected.
+        AssertGenerateBlobIdsQuickResponse(E_FS_THROTTLED);
+        DescribeData(0, config.BlockSize);
+        AssertDescribeDataQuickResponse(E_FS_THROTTLED);
+
+        // 3. Testing that after some time passes our postponed requests
+        // are successfully processed.
+        // Test actor runtime will automatically advance the timer for us.
+        for (ui32 i = 0; i < 20; ++i) {
+            Tick(TDuration::Seconds(1));
+            AssertDescribeDataResponse(S_OK);
+        }
+
+        for (ui32 i = 0; i < 3; ++i) {
+            Tick(TDuration::Seconds(1));
+            AssertGenerateBlobIdsResponse(S_OK);
+        }
+
+        // Now PostponedQueue is empty.
+
+        // 4. Testing that bursts actually work.
+        Tick(TDuration::Seconds(2));
+        DescribeData(0, 12_KB);
+        {
+            // Postponed WriteDataRequest must write 4_KB of 'z'.
+            AssertDescribeDataQuickResponse(S_OK);
+        }
+        DescribeData(12_KB, 4_KB);
+        AssertDescribeDataNoResponse();
+        {
+            AssertDescribeDataResponse(S_OK);
+            Tick(TDuration::Seconds(1));
+        }
+
+        // 5. Requests of any size should work, but not immediately.
+        GenerateBlobIds(0, 32_KB);
+        Tick(TDuration::Seconds(5));
+        AssertGenerateBlobIdsResponse(S_OK);
+    }
+
     Y_UNIT_TEST_F(ShouldNotThrottleIfThrottlerIsDisabled, TTablet)
     {
         auto config = MakeThrottlerConfig(

diff --git a/cloud/filestore/public/api/protos/headers.proto b/cloud/filestore/public/api/protos/headers.proto
@@ -56,6 +56,9 @@ message THeaders
 
     // Explicitly disables multitablet forwarding.
     bool DisableMultiTabletForwarding = 11;
+
+    // Throttle already performed
+    bool ThrottlingDisabled = 12;
 }
 
 ////////////////////////////////////////////////////////////////////////////////