issue-1795: 1. making MaxFileBlocks configurable via StorageServiceCo…

…nfig; 2. added a bunch of parameters which will control the way LargeDeletionMarkers work; 3. loading LargeDeletionMarkers upon tablet startup
ydb-platform · Aug 30, 2024 · d8217d9 · d8217d9
1 parent 386ded3
commit d8217d9
Show file tree

Hide file tree

Showing 20 changed files with 96 additions and 31 deletions.
diff --git a/cloud/filestore/config/storage.proto b/cloud/filestore/config/storage.proto
@@ -360,4 +360,18 @@ message TStorageConfig
     // Used to send non-network metrics as network ones to HIVE,
     // while we use them for load balancing
     optional uint32 NonNetworkMetricsBalancingFactor = 378;
+
+    // Max block count per file.
+    optional uint64 MaxFileBlocks = 400;
+    // Enables the usage of large deletion markers (needed for efficient
+    // truncate ops on large files).
+    optional bool LargeDeletionMarkersEnabled = 401;
+    // Sets max block count per single large deletion marker.
+    optional uint64 LargeDeletionMarkerBlocks = 402;
+    // Truncate and allocate ops that exceed this threshold will lead to large
+    // deletion marker generation.
+    optional uint64 LargeDeletionMarkersThreshold = 403;
+    // If the number of blocks marked for deletion via large deletion markers
+    // exceeds this threshold, Cleanup will be triggered.
+    optional uint64 LargeDeletionMarkersCleanupThreshold = 404;
 }
diff --git a/cloud/filestore/libs/service/filestore.h b/cloud/filestore/libs/service/filestore.h
@@ -34,7 +34,6 @@ constexpr ui32 MaxLink = NProto::E_FS_LIMITS_LINK;
 constexpr ui32 MaxName = NProto::E_FS_LIMITS_NAME;
 constexpr ui32 MaxPath = NProto::E_FS_LIMITS_PATH;
 constexpr ui32 MaxSymlink = NProto::E_FS_LIMITS_SYMLINK;
-constexpr ui64 MaxFileBlocks = static_cast<ui32>(NProto::E_FS_LIMITS_FILEBLOCKS);
 constexpr ui64 MaxNodes = static_cast<ui32>(NProto::E_FS_LIMITS_INODES);
 constexpr ui64 MaxXAttrName = NProto::E_FS_LIMITS_XATTR_NAME;
 constexpr ui64 MaxXAttrValue = NProto::E_FS_LIMITS_XATTR_VALUE;

diff --git a/cloud/filestore/libs/storage/core/config.cpp b/cloud/filestore/libs/storage/core/config.cpp
@@ -48,6 +48,13 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
     xxx(LoadedCompactionRangesPerTx,        ui32,   10 * 1024 * 1024          )\
     xxx(MaxBlocksPerTruncateTx,             ui32,   0 /*TODO: 32GiB/4KiB*/    )\
     xxx(MaxTruncateTxInflight,              ui32,   10                        )\
+                                                                               \
+    xxx(MaxFileBlocks,                          ui64,   300_GB / 4_KB         )\
+    xxx(LargeDeletionMarkersEnabled,            bool,   false                 )\
+    xxx(LargeDeletionMarkerBlocks,              ui64,   1_GB / 4_KB           )\
+    xxx(LargeDeletionMarkersThreshold,          ui64,   128_GB / 4_KB         )\
+    xxx(LargeDeletionMarkersCleanupThreshold,   ui64,   10_TB / 4_KB          )\
+                                                                               \
     xxx(CompactionRetryTimeout,             TDuration, TDuration::Seconds(1)  )\
     xxx(BlobIndexOpsPriority,                                                  \
             NProto::EBlobIndexOpsPriority,                                     \

diff --git a/cloud/filestore/libs/storage/core/config.h b/cloud/filestore/libs/storage/core/config.h
@@ -255,6 +255,12 @@ class TStorageConfig
 
     ui32 GetChannelFreeSpaceThreshold() const;
     ui32 GetChannelMinFreeSpace() const;
+
+    ui64 GetMaxFileBlocks() const;
+    bool GetLargeDeletionMarkersEnabled() const;
+    ui64 GetLargeDeletionMarkerBlocks() const;
+    ui64 GetLargeDeletionMarkersThreshold() const;
+    ui64 GetLargeDeletionMarkersCleanupThreshold() const;
 };
 
 }   // namespace NCloud::NFileStore::NStorage
diff --git a/cloud/filestore/libs/storage/tablet/helpers.cpp b/cloud/filestore/libs/storage/tablet/helpers.cpp
@@ -150,13 +150,13 @@ NProto::TError ValidateXAttrValue(const TString& name, const TString& value)
     return {};
 }
 
-NProto::TError ValidateRange(TByteRange byteRange)
+NProto::TError ValidateRange(TByteRange byteRange, ui32 maxFileBlocks)
 {
     if (!byteRange.Length) {
         return ErrorInvalidArgument();
     }
 
-    if (byteRange.LastBlock() + 1 > MaxFileBlocks) {
+    if (byteRange.LastBlock() + 1 > maxFileBlocks) {
         return ErrorFileTooBig();
     }
 

diff --git a/cloud/filestore/libs/storage/tablet/helpers.h b/cloud/filestore/libs/storage/tablet/helpers.h
@@ -131,7 +131,7 @@ NProto::ELockType ConvertToImpl(ELockMode source, TTag<NProto::ELockType>);
 NProto::TError ValidateNodeName(const TString& name);
 NProto::TError ValidateXAttrName(const TString& name);
 NProto::TError ValidateXAttrValue(const TString& name, const TString& value);
-NProto::TError ValidateRange(TByteRange byteRange);
+NProto::TError ValidateRange(TByteRange byteRange, ui32 maxFileBlocks);
 
 ////////////////////////////////////////////////////////////////////////////////
 

diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp
@@ -329,7 +329,8 @@ NProto::TError TIndexTabletActor::ValidateWriteRequest(
     const TRequest& request,
     const TByteRange& range)
 {
-    if (auto error = ValidateRange(range); HasError(error)) {
+    auto error = ValidateRange(range, Config->GetMaxFileBlocks());
+    if (HasError(error)) {
         return error;
     }
 

diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_acquirelock.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_acquirelock.cpp
@@ -15,14 +15,17 @@ namespace {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-NProto::TError ValidateRequest(const NProto::TAcquireLockRequest& request, ui32 blockSize)
+NProto::TError ValidateRequest(
+    const NProto::TAcquireLockRequest& request,
+    ui32 blockSize,
+    ui32 maxFileBlocks)
 {
     if (request.GetHandle() == InvalidHandle) {
         return ErrorInvalidHandle();
     }
 
     TByteRange range(request.GetOffset(), request.GetLength(), blockSize);
-    if (range.BlockCount() > MaxFileBlocks) {
+    if (range.BlockCount() > maxFileBlocks) {
         return ErrorFileTooBig();
     }
 
@@ -38,7 +41,10 @@ void TIndexTabletActor::HandleAcquireLock(
     const TActorContext& ctx)
 {
     auto validator = [&] (const NProto::TAcquireLockRequest& request) {
-        return ValidateRequest(request, GetBlockSize());
+        return ValidateRequest(
+            request,
+            GetBlockSize(),
+            Config->GetMaxFileBlocks());
     };
 
     if (!AcceptRequest<TEvService::TAcquireLockMethod>(ev, ctx, validator)) {

diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_allocatedata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_allocatedata.cpp
@@ -13,7 +13,8 @@ namespace {
 
 NProto::TError ValidateRequest(
     const NProto::TAllocateDataRequest& request,
-    ui32 blockSize)
+    ui32 blockSize,
+    ui32 maxFileBlocks)
 {
     if (!request.GetHandle()) {
         return ErrorInvalidHandle();
@@ -28,7 +29,7 @@ NProto::TError ValidateRequest(
     }
 
     TByteRange range(request.GetOffset(), request.GetLength(), blockSize);
-    if (range.BlockCount() > MaxFileBlocks) {
+    if (range.BlockCount() > maxFileBlocks) {
         return ErrorFileTooBig();
     }
 
@@ -71,7 +72,10 @@ void TIndexTabletActor::HandleAllocateData(
     }
 
     auto validator = [&] (const NProto::TAllocateDataRequest& request) {
-        return ValidateRequest(request, GetBlockSize());
+        return ValidateRequest(
+            request,
+            GetBlockSize(),
+            Config->GetMaxFileBlocks());
     };
 
     if (!AcceptRequest<TEvService::TAllocateDataMethod>(ev, ctx, validator)) {

diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_loadstate.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_loadstate.cpp
@@ -93,7 +93,6 @@ bool TIndexTabletActor::PrepareTx_LoadState(
 
     TIndexTabletDatabase db(tx.DB);
 
-    // TODO: load LargeBlocks
     std::initializer_list<bool> results = {
         db.ReadFileSystem(args.FileSystem),
         db.ReadFileSystemStats(args.FileSystemStats),
@@ -111,7 +110,8 @@ bool TIndexTabletActor::PrepareTx_LoadState(
         db.ReadTruncateQueue(args.TruncateQueue),
         db.ReadStorageConfig(args.StorageConfig),
         db.ReadSessionHistoryEntries(args.SessionHistory),
-        db.ReadOpLog(args.OpLog)
+        db.ReadOpLog(args.OpLog),
+        db.ReadLargeDeletionMarkers(args.LargeDeletionMarkers),
     };
 
     bool ready = std::accumulate(
@@ -231,13 +231,17 @@ void TIndexTabletActor::CompleteTx_LoadState(
 
     LOG_INFO_S(ctx, TFileStoreComponents::TABLET,
         LogTag << " Initializing tablet state");
+    LOG_INFO_S(ctx, TFileStoreComponents::TABLET,
+        LogTag << " Read " << args.LargeDeletionMarkers.size()
+        << " large deletion markers");
 
     LoadState(
         Executor()->Generation(),
         *Config,
         args.FileSystem,
         args.FileSystemStats,
         args.TabletStorageInfo,
+        args.LargeDeletionMarkers,
         config);
     UpdateLogTag();
 

diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp
@@ -502,15 +502,18 @@ STFUNC(TReadDataActor::StateWork)
 ////////////////////////////////////////////////////////////////////////////////
 
 template <typename TReadRequest>
-NProto::TError ValidateRequest(const TReadRequest& request, ui32 blockSize)
+NProto::TError ValidateRequest(
+    const TReadRequest& request,
+    ui32 blockSize,
+    ui32 maxFileBlocks)
 {
     const TByteRange range(
         request.GetOffset(),
         request.GetLength(),
         blockSize
     );
 
-    if (auto error = ValidateRange(range); HasError(error)) {
+    if (auto error = ValidateRange(range, maxFileBlocks); HasError(error)) {
         return error;
     }
 
@@ -526,7 +529,10 @@ void TIndexTabletActor::HandleReadData(
     const TActorContext& ctx)
 {
     auto validator = [&] (const NProto::TReadDataRequest& request) {
-        return ValidateRequest(request, GetBlockSize());
+        return ValidateRequest(
+            request,
+            GetBlockSize(),
+            Config->GetMaxFileBlocks());
     };
 
     if (!AcceptRequest<TEvService::TReadDataMethod>(ev, ctx, validator)) {
@@ -586,7 +592,10 @@ void TIndexTabletActor::HandleDescribeData(
     const TActorContext& ctx)
 {
     auto validator = [&] (const NProtoPrivate::TDescribeDataRequest& request) {
-        return ValidateRequest(request, GetBlockSize());
+        return ValidateRequest(
+            request,
+            GetBlockSize(),
+            Config->GetMaxFileBlocks());
     };
 
     if (!AcceptRequest<TEvIndexTablet::TDescribeDataMethod>(ev, ctx, validator)) {

diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_setnodeattr.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_setnodeattr.cpp
@@ -15,17 +15,19 @@ namespace {
 
 NProto::TError ValidateRequest(
     const NProto::TSetNodeAttrRequest& request,
-    ui32 blockSize)
+    ui32 blockSize,
+    ui32 maxFileBlocks)
 {
     if (request.GetNodeId() == InvalidNodeId || request.GetFlags() == 0) {
         return ErrorInvalidArgument();
     }
 
-    if (HasFlag(request.GetFlags(), NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE) &&
+    const auto setSize = NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE;
+    if (HasFlag(request.GetFlags(), setSize) &&
         request.GetUpdate().GetSize() > 0)
     {
         TByteRange range(0, request.GetUpdate().GetSize(), blockSize);
-        if (range.BlockCount() > MaxFileBlocks) {
+        if (range.BlockCount() > maxFileBlocks) {
             return ErrorFileTooBig();
         }
     }
@@ -57,7 +59,10 @@ void TIndexTabletActor::HandleSetNodeAttr(
     }
 
     auto validator = [&] (const NProto::TSetNodeAttrRequest& request) {
-        return ValidateRequest(request, GetBlockSize());
+        return ValidateRequest(
+            request,
+            GetBlockSize(),
+            Config->GetMaxFileBlocks());
     };
 
     if (!AcceptRequest<TEvService::TSetNodeAttrMethod>(ev, ctx, validator)) {

diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.cpp b/cloud/filestore/libs/storage/tablet/tablet_state.cpp
@@ -82,6 +82,7 @@ void TIndexTabletState::LoadState(
     const NProto::TFileSystem& fileSystem,
     const NProto::TFileSystemStats& fileSystemStats,
     const NCloud::NProto::TTabletStorageInfo& tabletStorageInfo,
+    const TVector<TDeletionMarker>& largeDeletionMarkers,
     const TThrottlerConfig& throttlerConfig)
 {
     Generation = generation;
@@ -122,6 +123,9 @@ void TIndexTabletState::LoadState(
         config.GetInMemoryIndexCacheNodeAttrsVerCapacity(),
         config.GetInMemoryIndexCacheNodeRefsCapacity(),
         config.GetInMemoryIndexCacheNodeRefsVerCapacity());
+    for (const auto& deletionMarker: largeDeletionMarkers) {
+        Impl->LargeBlocks.AddDeletionMarker(deletionMarker);
+    }
 }
 
 void TIndexTabletState::UpdateConfig(

diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.h b/cloud/filestore/libs/storage/tablet/tablet_state.h
@@ -163,6 +163,7 @@ class TIndexTabletState
         const NProto::TFileSystem& fileSystem,
         const NProto::TFileSystemStats& fileSystemStats,
         const NCloud::NProto::TTabletStorageInfo& tabletStorageInfo,
+        const TVector<TDeletionMarker>& largeDeletionMarkers,
         const TThrottlerConfig& throttlerConfig);
 
     bool IsStateLoaded() const

diff --git a/cloud/filestore/libs/storage/tablet/tablet_tx.h b/cloud/filestore/libs/storage/tablet/tablet_tx.h
@@ -307,6 +307,7 @@ struct TTxIndexTablet
         TMaybe<NProto::TStorageConfig> StorageConfig;
         TVector<NProto::TSessionHistoryEntry> SessionHistory;
         TVector<NProto::TOpLogEntry> OpLog;
+        TVector<TDeletionMarker> LargeDeletionMarkers;
 
         NProto::TError Error;
 
@@ -329,6 +330,7 @@ struct TTxIndexTablet
             StorageConfig.Clear();
             SessionHistory.clear();
             OpLog.clear();
+            LargeDeletionMarkers.clear();
         }
     };
 

diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp
@@ -1217,7 +1217,7 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data)
         ui32 nodeIdx = env.CreateNode("nfs");
         ui64 tabletId = env.BootIndexTablet(nodeIdx);
 
-        tabletConfig.BlockCount = MaxFileBlocks * 2;
+        tabletConfig.BlockCount = GetDefaultMaxFileBlocks() * 2;
         TIndexTabletClient tablet(
             env.GetRuntime(),
             nodeIdx,
@@ -1245,21 +1245,21 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data)
 
         tablet.WriteData(
             handle,
-            (MaxFileBlocks - 1) * block,
+            (GetDefaultMaxFileBlocks() - 1) * block,
             block,
             '5');
         UNIT_ASSERT_VALUES_EQUAL(
             GetNodeAttrs(tablet, id).GetSize(),
-            MaxFileBlocks * block);
+            GetDefaultMaxFileBlocks() * block);
 
         tablet.AssertWriteDataFailed(
             handle,
-            MaxFileBlocks * block,
+            GetDefaultMaxFileBlocks() * block,
             1_KB,
             '6');
         UNIT_ASSERT_VALUES_EQUAL(
             GetNodeAttrs(tablet, id).GetSize(),
-            MaxFileBlocks * block);
+            GetDefaultMaxFileBlocks() * block);
     }
 
     TABLET_TEST(ShouldTrackUsedBlocks)

diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_data_stress.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_data_stress.cpp
@@ -256,7 +256,7 @@ struct TEnvironment
     {
         static auto hasher = CreateRangeIdHasher(RangeIdHasherType);
 
-        const TByteRange maxFileBlocks(0, MaxFileBlocks, BlockSize);
+        const TByteRange maxFileBlocks(0, GetDefaultMaxFileBlocks(), BlockSize);
 
         TVector<ui32> rangeIds;
         SplitRange(

diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_data_truncate.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_data_truncate.cpp
@@ -113,7 +113,7 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data_Truncate)
         UNIT_ASSERT_VALUES_EQUAL(GetNodeAttrs(tablet, id).GetSize(), 0);
         UNIT_ASSERT_VALUES_EQUAL(ReadData(tablet, handle, 8_KB), TString());
 
-        args.SetSize(MaxFileBlocks * DefaultBlockSize + 1);
+        args.SetSize(GetDefaultMaxFileBlocks() * DefaultBlockSize + 1);
         tablet.AssertSetNodeAttrFailed(args);
 
         args.SetSize(18_KB);

diff --git a/cloud/filestore/libs/storage/testlib/test_env.h b/cloud/filestore/libs/storage/testlib/test_env.h
@@ -35,6 +35,12 @@ constexpr ui32 DefaultChannelCount = 7;
 
 void CheckForkJoin(const NLWTrace::TShuttleTrace& trace, bool forkRequired);
 
+inline auto GetDefaultMaxFileBlocks()
+{
+    static const auto VALUE = TStorageConfig().GetMaxFileBlocks();
+    return VALUE;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 struct TTestEnvConfig

diff --git a/cloud/filestore/public/api/protos/const.proto b/cloud/filestore/public/api/protos/const.proto
@@ -76,9 +76,6 @@ enum EFilestoreLimits
     // Maximum length of extended attribute value.
     E_FS_LIMITS_XATTR_VALUE = 65536;
 
-    // Until NBS-2979 it's limited to 300GB
-    E_FS_LIMITS_FILEBLOCKS = 78643200;
-
     // Maximum number of inodes.
     E_FS_LIMITS_INODES = -2; // 0xFFFFFFFE
 };