[Filestore, merge to stable] add force operation to delete all zero c…

…ompaction stat ranges; add action to write specified data to compaction map table, nfs-client.txt, support aliases for filestore names (#1817) * add force operation to delete all zero compaction stat ranges; add aсtion to write specified data to compaction map table (#1690) * add force operation to delete all zero compaction stat ranges; add action to write specified data to compaction map table * fix-issues * fix issues * fix * [Filestore] add `nfs-client.txt` config for filestore analogous to the `nbs-client.txt` one (#1687) add nfs-client.txt config for filestore analogous to the nbs-client one * issue-1794: support aliases for filestore names (#1774) * issue-1794: support aliases for filestore names * fix cmakelists * issue-1794: fixed use-after-free in fs search by alias, not making a new THashMap upon each alias search, removed incorrect TABLET_VERIFY in TIndexTabletActor::CompleteTx_CreateSession (#1816) * issue-1794: fixed use-after-free in fs search by alias, not making a new THashMap upon each alias search, removed incorrect TABLET_VERIFY in TIndexTabletActor::CompleteTx_CreateSession * issue-1794: fixed use-after-free in fs search by alias, not making a new THashMap upon each alias search, removed incorrect TABLET_VERIFY in TIndexTabletActor::CompleteTx_CreateSession - fixed comment * fix build: exclude TCertificate from config.cpp * fix build * fix build * fix build --------- Co-authored-by: Darya Frolova <[email protected]> Co-authored-by: Darya Frolova <[email protected]> Co-authored-by: Andrei Strelkovskii <[email protected]>
ydb-platform · Aug 19, 2024 · 1fbce29 · 1fbce29
1 parent 8202eae
commit 1fbce29
Show file tree

Hide file tree

Showing 41 changed files with 932 additions and 23 deletions.
diff --git a/cloud/blockstore/apps/client/lib/command.cpp b/cloud/blockstore/apps/client/lib/command.cpp
@@ -122,6 +122,7 @@ TCommand::TCommand(IBlockStorePtr client)
             << "config file name. Default is "
             << DefaultConfigFile)
         .RequiredArgument("STR")
+        .DefaultValue(DefaultConfigFile)
         .StoreResult(&ConfigFile);
 
     Opts.AddLongOption("iam-config")
@@ -610,10 +611,8 @@ TString TCommand::GetIamTokenFromClient()
 void TCommand::InitClientConfig()
 {
     NProto::TClientAppConfig appConfig;
-    if (ConfigFile) {
+    if (NFs::Exists(ConfigFile)) {
         ParseFromTextFormat(ConfigFile, appConfig);
-    } else if (NFs::Exists(DefaultConfigFile)) {
-        ParseFromTextFormat(DefaultConfigFile, appConfig);
     }
 
     auto& clientConfig = *appConfig.MutableClientConfig();

diff --git a/cloud/filestore/apps/client/lib/command.cpp b/cloud/filestore/apps/client/lib/command.cpp
@@ -8,6 +8,7 @@
 #include <cloud/storage/core/libs/common/timer.h>
 
 #include <library/cpp/lwtrace/mon/mon_lwtrace.h>
+#include <library/cpp/protobuf/util/pb_io.h>
 
 #include <util/datetime/base.h>
 #include <util/folder/dirut.h>
@@ -32,6 +33,7 @@ namespace {
 
 constexpr TDuration WaitTimeout = TDuration::Seconds(1);
 
+const TString DefaultConfigFile = "/Berkanavt/nfs-server/cfg/nfs-client.txt";
 const TString DefaultIamTokenFile = "~/.nfs-client/iam-token";
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -103,6 +105,14 @@ TCommand::TCommand()
         .RequiredArgument("STR")
         .StoreResult(&IamTokenFile);
 
+    Opts.AddLongOption("config")
+        .Help(TStringBuilder()
+            << "config file name. Default is "
+            << DefaultConfigFile)
+        .RequiredArgument("STR")
+        .DefaultValue(DefaultConfigFile)
+        .StoreResult(&ConfigFile);
+
     Opts.AddLongOption("json")
         .StoreTrue(&JsonOutput);
 }
@@ -174,6 +184,10 @@ void TCommand::Init()
     Scheduler = CreateScheduler();
 
     NProto::TClientConfig config;
+    if (NFs::Exists(ConfigFile)) {
+        ParseFromTextFormat(ConfigFile, config);
+    }
+
     if (ServerAddress) {
         config.SetHost(ServerAddress);
     }

diff --git a/cloud/filestore/apps/client/lib/command.h b/cloud/filestore/apps/client/lib/command.h
@@ -53,6 +53,7 @@ class TCommand
     ui32 SecurePort = 0;
     bool SkipCertVerification = false;
     TString IamTokenFile;
+    TString ConfigFile;
 
     TClientConfigPtr ClientConfig;
 

diff --git a/cloud/filestore/config/storage.proto b/cloud/filestore/config/storage.proto
@@ -312,4 +312,19 @@ message TStorageConfig
 
     // Enables ThreeStageWrite for unaligned requests.
     optional bool UnalignedThreeStageWriteEnabled = 366;
+
+    // Number of ranges with zero compaction stats to delete per tx.
+    optional uint32 MaxZeroCompactionRangesToDeletePerTx = 367;
+
+    // Mapping allowing for multiple fs ids to point to the same fs
+    message TFilestoreAliasEntry
+    {
+        optional string Alias = 1;
+        optional string FsId = 2;
+    }
+    message TFilestoreAliases
+    {
+        repeated TFilestoreAliasEntry Entries = 1;
+    }
+    optional TFilestoreAliases FilestoreAliases = 368;
 }
diff --git a/cloud/filestore/libs/storage/api/tablet.h b/cloud/filestore/libs/storage/api/tablet.h
@@ -31,6 +31,7 @@ namespace NCloud::NFileStore::NStorage {
     xxx(ConfigureAsFollower,        __VA_ARGS__)                               \
     xxx(GetStorageConfig,           __VA_ARGS__)                               \
     xxx(GetNodeAttrBatch,           __VA_ARGS__)                               \
+    xxx(WriteCompactionMap,         __VA_ARGS__)                               \
 // FILESTORE_TABLET_REQUESTS
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -93,6 +94,9 @@ struct TEvIndexTablet
         EvGetNodeAttrBatchRequest = EvBegin + 31,
         EvGetNodeAttrBatchResponse,
 
+        EvWriteCompactionMapRequest = EvBegin + 33,
+        EvWriteCompactionMapResponse,
+
         EvEnd
     };
 

diff --git a/cloud/filestore/libs/storage/core/config.cpp b/cloud/filestore/libs/storage/core/config.cpp
@@ -2,6 +2,7 @@
 
 #include <library/cpp/monlib/service/pages/templates.h>
 
+#include <util/generic/hash.h>
 #include <util/generic/size_literals.h>
 
 #include <google/protobuf/text_format.h>
@@ -12,6 +13,8 @@ namespace {
 
 ////////////////////////////////////////////////////////////////////////////////
 
+using TAliases = NProto::TStorageConfig::TFilestoreAliases;
+
 #define FILESTORE_STORAGE_CONFIG(xxx)                                          \
     xxx(SchemeShardDir,                TString,   "/Root"                     )\
                                                                                \
@@ -168,8 +171,14 @@ namespace {
     xxx(AllowFileStoreForceDestroy,                     bool,      false      )\
     xxx(BlobCompressionRate,             ui32,                  0             )\
     xxx(BlobCompressionCodec,            TString,               "lz4"         )\
+                                                                               \
+    xxx(MaxZeroCompactionRangesToDeletePerTx,           ui32,      10000      )\
 // FILESTORE_STORAGE_CONFIG
 
+#define FILESTORE_STORAGE_CONFIG_REF(xxx)                                      \
+    xxx(FilestoreAliases,                TAliases,              {}            )\
+// FILESTORE_STORAGE_CONFIG_REF
+
 #define FILESTORE_DECLARE_CONFIG(name, type, value)                            \
     Y_DECLARE_UNUSED static const type Default##name = value;                  \
 // FILESTORE_DECLARE_CONFIG
@@ -186,6 +195,12 @@ bool IsEmpty(const T& t)
     return !t;
 }
 
+template <>
+bool IsEmpty(const TAliases& value)
+{
+    return value.GetEntries().empty();
+}
+
 template <typename TTarget, typename TSource>
 TTarget ConvertValue(const TSource& value)
 {
@@ -218,6 +233,16 @@ void DumpImpl(const T& t, IOutputStream& os)
     os << t;
 }
 
+template <>
+void DumpImpl(const TAliases& value, IOutputStream& os)
+{
+    os << "{ ";
+    for (const auto& x: value.GetEntries()) {
+        os << x.GetAlias() << ": " << x.GetFsId() << ", ";
+    }
+    os << " }";
+}
+
 }   // namespace
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -234,6 +259,17 @@ FILESTORE_STORAGE_CONFIG(FILESTORE_CONFIG_GETTER)
 
 #undef FILESTORE_CONFIG_GETTER
 
+#define FILESTORE_CONFIG_GETTER_REF(name, type, ...)                           \
+const type& TStorageConfig::Get##name() const                                  \
+{                                                                              \
+    return ProtoConfig.Get##name();                                            \
+}                                                                              \
+// FILESTORE_CONFIG_GETTER_REF
+
+FILESTORE_STORAGE_CONFIG_REF(FILESTORE_CONFIG_GETTER_REF)
+
+#undef FILESTORE_CONFIG_GETTER_REF
+
 void TStorageConfig::Dump(IOutputStream& out) const
 {
 #define FILESTORE_DUMP_CONFIG(name, ...)                                       \
@@ -243,6 +279,7 @@ void TStorageConfig::Dump(IOutputStream& out) const
 // FILESTORE_DUMP_CONFIG
 
     FILESTORE_STORAGE_CONFIG(FILESTORE_DUMP_CONFIG);
+    FILESTORE_STORAGE_CONFIG_REF(FILESTORE_DUMP_CONFIG);
 
 #undef FILESTORE_DUMP_CONFIG
 }
@@ -260,6 +297,7 @@ void TStorageConfig::DumpHtml(IOutputStream& out) const
         TABLE_CLASS("table table-condensed") {
             TABLEBODY() {
                 FILESTORE_STORAGE_CONFIG(FILESTORE_DUMP_CONFIG);
+                FILESTORE_STORAGE_CONFIG_REF(FILESTORE_DUMP_CONFIG);
             }
         }
     }
@@ -284,6 +322,7 @@ void TStorageConfig::DumpOverridesHtml(IOutputStream& out) const
         TABLE_CLASS("table table-condensed") {
             TABLEBODY() {
                 FILESTORE_STORAGE_CONFIG(FILESTORE_DUMP_CONFIG);
+                FILESTORE_STORAGE_CONFIG_REF(FILESTORE_DUMP_CONFIG);
             }
         }
     }
@@ -329,4 +368,16 @@ const NProto::TStorageConfig& TStorageConfig::GetStorageConfigProto() const
     return ProtoConfig;
 }
 
+const TString* TStorageConfig::FindFileSystemIdByAlias(
+    const TString& alias) const
+{
+    const auto& entries = GetFilestoreAliases().GetEntries();
+    for (const auto& entry: entries) {
+        if (entry.GetAlias() == alias) {
+            return &entry.GetFsId();
+        }
+    }
+    return nullptr;
+}
+
 }   // namespace NCloud::NFileStore::NStorage
diff --git a/cloud/filestore/libs/storage/core/config.h b/cloud/filestore/libs/storage/core/config.h
@@ -214,6 +214,8 @@ class TStorageConfig
 
     bool GetAllowFileStoreForceDestroy() const;
 
+    ui32 GetMaxZeroCompactionRangesToDeletePerTx() const;
+
     void Dump(IOutputStream& out) const;
     void DumpHtml(IOutputStream& out) const;
     void DumpOverridesHtml(IOutputStream& out) const;
@@ -222,6 +224,9 @@ class TStorageConfig
     TString GetBlobCompressionCodec() const;
 
     const NProto::TStorageConfig& GetStorageConfigProto() const;
+
+    const NProto::TStorageConfig::TFilestoreAliases& GetFilestoreAliases() const;
+    const TString* FindFileSystemIdByAlias(const TString& alias) const;
 };
 
 }   // namespace NCloud::NFileStore::NStorage
diff --git a/cloud/filestore/libs/storage/service/CMakeLists.darwin-x86_64.txt b/cloud/filestore/libs/storage/service/CMakeLists.darwin-x86_64.txt
@@ -40,6 +40,7 @@ target_sources(filestore-libs-storage-service PRIVATE
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_reassign_tablet.cpp
+  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_write_compaction_map.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_alterfs.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_complete.cpp

diff --git a/cloud/filestore/libs/storage/service/CMakeLists.linux-aarch64.txt b/cloud/filestore/libs/storage/service/CMakeLists.linux-aarch64.txt
@@ -41,6 +41,7 @@ target_sources(filestore-libs-storage-service PRIVATE
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_reassign_tablet.cpp
+  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_write_compaction_map.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_alterfs.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_complete.cpp

diff --git a/cloud/filestore/libs/storage/service/CMakeLists.linux-x86_64.txt b/cloud/filestore/libs/storage/service/CMakeLists.linux-x86_64.txt
@@ -41,6 +41,7 @@ target_sources(filestore-libs-storage-service PRIVATE
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_reassign_tablet.cpp
+  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_write_compaction_map.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_alterfs.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_complete.cpp

diff --git a/cloud/filestore/libs/storage/service/CMakeLists.windows-x86_64.txt b/cloud/filestore/libs/storage/service/CMakeLists.windows-x86_64.txt
@@ -40,6 +40,7 @@ target_sources(filestore-libs-storage-service PRIVATE
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_reassign_tablet.cpp
+  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_write_compaction_map.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_alterfs.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_complete.cpp

diff --git a/cloud/filestore/libs/storage/service/service_actor.h b/cloud/filestore/libs/storage/service/service_actor.h
@@ -185,6 +185,10 @@ class TStorageServiceActor final
         TRequestInfoPtr requestInfo,
         TString input);
 
+    NActors::IActorPtr CreateWriteCompactionMapActionActor(
+        TRequestInfoPtr requestInfo,
+        TString input);
+
 private:
     void RenderSessions(IOutputStream& out);
     void RenderLocalFileStores(IOutputStream& out);

diff --git a/cloud/filestore/libs/storage/service/service_actor_actions.cpp b/cloud/filestore/libs/storage/service/service_actor_actions.cpp
@@ -68,6 +68,10 @@ void TStorageServiceActor::HandleExecuteAction(
             "getstorageconfig",
             &TStorageServiceActor::CreateGetStorageConfigActionActor
         },
+        {
+            "writecompactionmap",
+            &TStorageServiceActor::CreateWriteCompactionMapActionActor
+        },
     };
 
     auto it = actions.find(action);