NBSNEBIUS-371: support reconnect for local file system

ydb-platform · Oct 8, 2024 · e3e1c7b · e3e1c7b
1 parent 0f60027
commit e3e1c7b
Show file tree

Hide file tree

Showing 13 changed files with 724 additions and 62 deletions.
diff --git a/cloud/filestore/config/server.proto b/cloud/filestore/config/server.proto
@@ -72,6 +72,15 @@ message TLocalServiceConfig
     optional uint32 DefaultPermissions = 3;
     optional uint32 IdleSessionTimeout = 4;
     optional uint32 NumThreads = 5;
+
+    // directory where state for session recovery is saved
+    optional string StatePath = 6;
+
+    // Maximum number of inodes which can be created by all sessions
+    optional uint32 MaxInodeCount = 7;
+
+    // Maximum number of file handles which can be opened by single session
+    optional uint32 MaxHandlePerSessionCount = 8;
 }
 
 ////////////////////////////////////////////////////////////////////////////////

diff --git a/cloud/filestore/libs/service_local/config.cpp b/cloud/filestore/libs/service_local/config.cpp
@@ -14,6 +14,9 @@ namespace {
     xxx(DefaultPermissions,          ui32,          0775                      )\
     xxx(IdleSessionTimeout,          TDuration,     TDuration::Seconds(30)    )\
     xxx(NumThreads,                  ui32,          4                         )\
+    xxx(StatePath,                   TString,       "./"                      )\
+    xxx(MaxInodeCount,               ui32,          1000000                   )\
+    xxx(MaxHandlePerSessionCount,    ui32,          10000                     )\
 // FILESTORE_SERVICE_CONFIG
 
 #define FILESTORE_SERVICE_DECLARE_CONFIG(name, type, value)                    \

diff --git a/cloud/filestore/libs/service_local/config.h b/cloud/filestore/libs/service_local/config.h
@@ -26,6 +26,9 @@ class TLocalFileStoreConfig
     ui32 GetDefaultPermissions() const;
     TDuration GetIdleSessionTimeout() const;
     ui32 GetNumThreads() const;
+    TString GetStatePath() const;
+    ui32 GetMaxInodeCount() const;
+    ui32 GetMaxHandlePerSessionCount() const;
 
     void Dump(IOutputStream& out) const;
     void DumpHtml(IOutputStream& out) const;

diff --git a/cloud/filestore/libs/service_local/fs.cpp b/cloud/filestore/libs/service_local/fs.cpp
@@ -8,29 +8,29 @@ TLocalFileSystem::TLocalFileSystem(
         TLocalFileStoreConfigPtr config,
         NProto::TFileStore store,
         TFsPath root,
+        TFsPath statePath,
         ITimerPtr timer,
         ISchedulerPtr scheduler,
         ILoggingServicePtr logging,
         IFileIOServicePtr fileIOService)
     : Config(std::move(config))
     , Root(std::move(root))
+    , StatePath(std::move(statePath))
     , Timer(std::move(timer))
     , Scheduler(std::move(scheduler))
+    , Logging(std::move(logging))
     , FileIOService(std::move(fileIOService))
     , Store(std::move(store))
 {
-    Log = logging->CreateLog(Store.GetFileSystemId());
+    Log = Logging->CreateLog(Store.GetFileSystemId());
 
-    InitIndex();
-    ScheduleCleanupSessions();
-}
+    Index = std::make_shared<TLocalIndex>(
+        Root,
+        StatePath,
+        Config->GetMaxInodeCount(),
+        Log);
 
-void TLocalFileSystem::InitIndex()
-{
-    TLocalIndex::TNodeMap nodes;
-    nodes.insert(TIndexNode::CreateRoot(Root));
-
-    Index = std::make_shared<TLocalIndex>(std::move(nodes));
+    ScheduleCleanupSessions();
 }
 
 ////////////////////////////////////////////////////////////////////////////////

diff --git a/cloud/filestore/libs/service_local/fs.h b/cloud/filestore/libs/service_local/fs.h
@@ -89,8 +89,10 @@ class TLocalFileSystem final
 private:
     const TLocalFileStoreConfigPtr Config;
     const TFsPath Root;
+    const TFsPath StatePath;
     const ITimerPtr Timer;
     const ISchedulerPtr Scheduler;
+    const ILoggingServicePtr Logging;
     const IFileIOServicePtr FileIOService;
 
     NProto::TFileStore Store;
@@ -111,6 +113,7 @@ class TLocalFileSystem final
         TLocalFileStoreConfigPtr config,
         NProto::TFileStore store,
         TFsPath root,
+        TFsPath statePath,
         ITimerPtr timer,
         ISchedulerPtr scheduler,
         ILoggingServicePtr logging,
@@ -144,7 +147,6 @@ class TLocalFileSystem final
     }
 
 private:
-    void InitIndex();
 
     void ScheduleCleanupSessions();
     void CleanupSessions();

diff --git a/cloud/filestore/libs/service_local/fs_attrs.cpp b/cloud/filestore/libs/service_local/fs_attrs.cpp
@@ -87,7 +87,13 @@ NProto::TGetNodeAttrResponse TLocalFileSystem::GetNodeAttr(
             // TODO: better? race between statting one child and creating another one
             // but maybe too costly...
             stat = child->Stat();
-            session->TryInsertNode(std::move(child));
+            if (!session->TryInsertNode(
+                    std::move(child),
+                    node->GetNodeId(),
+                    name))
+            {
+                return TErrorResponse(ErrorNoSpaceLeft());
+            }
         }
     } else {
         stat = node->Stat();

diff --git a/cloud/filestore/libs/service_local/fs_data.cpp b/cloud/filestore/libs/service_local/fs_data.cpp
@@ -28,29 +28,44 @@ NProto::TCreateHandleResponse TLocalFileSystem::CreateHandle(
         return TErrorResponse(ErrorInvalidParent(request.GetNodeId()));
     }
 
-    const int flags = HandleFlagsToSystem(request.GetFlags());
+    int flags = HandleFlagsToSystem(request.GetFlags());
     const int mode = request.GetMode()
         ? request.GetMode() : Config->GetDefaultPermissions();
 
     TFileHandle handle;
     TFileStat stat;
+    ui64 nodeId;
     if (const auto& pathname = request.GetName()) {
         handle = node->OpenHandle(pathname, flags, mode);
 
         auto newnode = TIndexNode::Create(*node, pathname);
         stat = newnode->Stat();
-
-        session->TryInsertNode(std::move(newnode));
+        nodeId = newnode->GetNodeId();
+
+        if (!session->TryInsertNode(
+                std::move(newnode),
+                node->GetNodeId(),
+                pathname))
+        {
+            return TErrorResponse(ErrorNoSpaceLeft());
+        }
     } else {
         handle = node->OpenHandle(flags);
         stat = node->Stat();
+        nodeId = node->GetNodeId();
     }
 
-    const FHANDLE fd = handle;
-    session->InsertHandle(std::move(handle));
+    // Don't persist flags that only make sense on initial open
+    flags = flags & ~(O_CREAT | O_EXCL | O_TRUNC);
+
+    auto handleId =
+        session->InsertHandle(std::move(handle), nodeId, flags);
+    if (!handleId) {
+        return TErrorResponse(ErrorNoSpaceLeft());
+    }
 
     NProto::TCreateHandleResponse response;
-    response.SetHandle(fd);
+    response.SetHandle(*handleId);
     ConvertStats(stat, *response.MutableNodeAttr());
 
     return response;

diff --git a/cloud/filestore/libs/service_local/fs_node.cpp b/cloud/filestore/libs/service_local/fs_node.cpp
@@ -80,7 +80,13 @@ NProto::TCreateNodeResponse TLocalFileSystem::CreateNode(
     }
 
     auto stat = target->Stat();
-    session->TryInsertNode(std::move(target));
+    if (!session->TryInsertNode(
+            std::move(target),
+            parent->GetNodeId(),
+            request.GetName()))
+    {
+        return TErrorResponse(ErrorNoSpaceLeft());
+    }
 
     NProto::TCreateNodeResponse response;
     ConvertStats(stat, *response.MutableNode());
@@ -192,7 +198,13 @@ NProto::TListNodesResponse TLocalFileSystem::ListNodes(
         if (!session->LookupNode(entry.second.INode)) {
             auto node = TryCreateChildNode(*parent, entry.first);
             if (node && node->GetNodeId() == entry.second.INode) {
-                session->TryInsertNode(std::move(node));
+                if (!session->TryInsertNode(
+                        std::move(node),
+                        parent->GetNodeId(),
+                        entry.first))
+                {
+                    return TErrorResponse(ErrorNoSpaceLeft());
+                }
             }
         }
 

diff --git a/cloud/filestore/libs/service_local/fs_session.cpp b/cloud/filestore/libs/service_local/fs_session.cpp
@@ -1,6 +1,5 @@
 #include "fs.h"
 
-#include <util/generic/guid.h>
 #include <util/string/builder.h>
 
 namespace NCloud::NFileStore {
@@ -46,22 +45,27 @@ NProto::TCreateSessionResponse TLocalFileSystem::CreateSession(
         (*it->second)->GetInfo(*response.MutableSession(), sessionSeqNo);
         return response;
     }
-    sessionId = CreateGuidAsString();
+
+    auto clientSessionStatePath = StatePath / ("client_" + clientId);
+    clientSessionStatePath.MkDir();
 
     session = std::make_shared<TSession>(
-        Root.GetPath(),
+        Store.GetFileSystemId(),
+        Root,
+        clientSessionStatePath,
         clientId,
-        sessionId,
-        Index);
+        Index,
+        Logging);
 
+    session->Init(request.GetRestoreClientSession(), Config->GetMaxHandlePerSessionCount());
     session->AddSubSession(sessionSeqNo, readOnly);
 
     SessionsList.push_front(session);
 
     auto [_, inserted1] = SessionsByClient.emplace(clientId, SessionsList.begin());
     Y_ABORT_UNLESS(inserted1);
 
-    auto [dummyIt, inserted2] = SessionsById.emplace(sessionId, SessionsList.begin());
+    auto [dummyIt, inserted2] = SessionsById.emplace(session->SessionId, SessionsList.begin());
     Y_ABORT_UNLESS(inserted2);
 
     NProto::TCreateSessionResponse response;
@@ -160,6 +164,8 @@ void TLocalFileSystem::RemoveSession(
         SessionsList.erase(it->second);
         SessionsById.erase(it);
     }
+
+    session->StatePath.ForceDelete();
 }
 
 void TLocalFileSystem::ScheduleCleanupSessions()