[Filestore] issue-2190: support reconnect for local file system

cloud/filestore/config/server.proto

@@ -72,6 +72,17 @@ message TLocalServiceConfig
     optional uint32 DefaultPermissions = 3;
     optional uint32 IdleSessionTimeout = 4;
     optional uint32 NumThreads = 5;
+
+    // directory where state for session recovery is saved
+    optional string StatePath = 6;
+
+    // Maximum number of nodes which can be opened by all sessions
+    // for single local file system
+    optional uint32 MaxNodeCount = 7;
+
+    // Maximum number of file handles which can be opened by single session
+    // for single local file system
+    optional uint32 MaxHandlePerSessionCount = 8;
 }
 
 ////////////////////////////////////////////////////////////////////////////////

cloud/filestore/libs/diagnostics/critical_events.h

@@ -26,6 +26,9 @@ namespace NCloud::NFileStore{
     xxx(InvalidDupCacheEntry)                                                  \
     xxx(GeneratedOrphanNode)                                                   \
     xxx(ReceivedNodeOpErrorFromShard)                                          \
+    xxx(LocalFsMaxSessionNodesInUse)                                           \
+    xxx(LocalFsMaxSessionFileHandlesInUse)                                     \
+    xxx(LocalFsMissingHandleNode)                                              \
 // FILESTORE_CRITICAL_EVENTS
 
 #define FILESTORE_IMPOSSIBLE_EVENTS(xxx)                                       \
@@ -41,6 +44,7 @@ namespace NCloud::NFileStore{
     xxx(NewChildNodeIsNull)                                                    \
     xxx(IndexOutOfBounds)                                                      \
     xxx(CheckFreshBytesFailed)                                                 \
+    xxx(LocalFsDuplicateFileHandle)                                            \
 // FILESTORE_IMPOSSIBLE_EVENTS
 
 ////////////////////////////////////////////////////////////////////////////////

cloud/filestore/libs/service_local/config.cpp

@@ -14,6 +14,9 @@ namespace {
     xxx(DefaultPermissions,          ui32,          0775                      )\
     xxx(IdleSessionTimeout,          TDuration,     TDuration::Seconds(30)    )\
     xxx(NumThreads,                  ui32,          4                         )\
+    xxx(StatePath,                   TString,       "./"                      )\
+    xxx(MaxNodeCount,                ui32,          1000000                   )\
+    xxx(MaxHandlePerSessionCount,    ui32,          10000                     )\
 // FILESTORE_SERVICE_CONFIG
 
 #define FILESTORE_SERVICE_DECLARE_CONFIG(name, type, value)                    \

cloud/filestore/libs/service_local/config.h

@@ -26,6 +26,9 @@ class TLocalFileStoreConfig
     ui32 GetDefaultPermissions() const;
     TDuration GetIdleSessionTimeout() const;
     ui32 GetNumThreads() const;
+    TString GetStatePath() const;
+    ui32 GetMaxNodeCount() const;
+    ui32 GetMaxHandlePerSessionCount() const;
 
     void Dump(IOutputStream& out) const;
     void DumpHtml(IOutputStream& out) const;

cloud/filestore/libs/service_local/fs.cpp

@@ -8,29 +8,29 @@ TLocalFileSystem::TLocalFileSystem(
         TLocalFileStoreConfigPtr config,
         NProto::TFileStore store,
         TFsPath root,
+        TFsPath statePath,
         ITimerPtr timer,
         ISchedulerPtr scheduler,
         ILoggingServicePtr logging,
         IFileIOServicePtr fileIOService)
     : Config(std::move(config))
     , Root(std::move(root))
+    , StatePath(std::move(statePath))
     , Timer(std::move(timer))
     , Scheduler(std::move(scheduler))
+    , Logging(std::move(logging))
     , FileIOService(std::move(fileIOService))
     , Store(std::move(store))
 {
-    Log = logging->CreateLog(Store.GetFileSystemId());
+    Log = Logging->CreateLog(Store.GetFileSystemId());
 
-    InitIndex();
-    ScheduleCleanupSessions();
-}
+    Index = std::make_shared<TLocalIndex>(
+        Root,
+        StatePath,
+        Config->GetMaxNodeCount(),
+        Log);
 
-void TLocalFileSystem::InitIndex()
-{
-    TLocalIndex::TNodeMap nodes;
-    nodes.insert(TIndexNode::CreateRoot(Root));
-
-    Index = std::make_shared<TLocalIndex>(std::move(nodes));
+    ScheduleCleanupSessions();
 }
 
 ////////////////////////////////////////////////////////////////////////////////

cloud/filestore/libs/service_local/fs.h

@@ -89,8 +89,10 @@ class TLocalFileSystem final
 private:
     const TLocalFileStoreConfigPtr Config;
     const TFsPath Root;
+    const TFsPath StatePath;
     const ITimerPtr Timer;
     const ISchedulerPtr Scheduler;
+    const ILoggingServicePtr Logging;
     const IFileIOServicePtr FileIOService;
 
     NProto::TFileStore Store;
@@ -111,6 +113,7 @@ class TLocalFileSystem final
         TLocalFileStoreConfigPtr config,
         NProto::TFileStore store,
         TFsPath root,
+        TFsPath statePath,
         ITimerPtr timer,
         ISchedulerPtr scheduler,
         ILoggingServicePtr logging,
@@ -144,7 +147,6 @@ class TLocalFileSystem final
     }
 
 private:
-    void InitIndex();
 
     void ScheduleCleanupSessions();
     void CleanupSessions();

cloud/filestore/libs/service_local/fs_attrs.cpp

@@ -1,5 +1,7 @@
 #include "fs.h"
 
+#include <cloud/filestore/libs/diagnostics/critical_events.h>
+
 #include <optional>
 
 namespace NCloud::NFileStore {
@@ -87,7 +89,14 @@ NProto::TGetNodeAttrResponse TLocalFileSystem::GetNodeAttr(
             // TODO: better? race between statting one child and creating another one
             // but maybe too costly...
             stat = child->Stat();
-            session->TryInsertNode(std::move(child));
+            if (!session->TryInsertNode(
+                    std::move(child),
+                    node->GetNodeId(),
+                    name))
+            {
+                ReportLocalFsMaxSessionNodesInUse();
+                return TErrorResponse(ErrorNoSpaceLeft());
+            }
         }
     } else {
         stat = node->Stat();

cloud/filestore/libs/service_local/fs_data.cpp

@@ -2,6 +2,7 @@
 
 #include "lowlevel.h"
 
+#include <cloud/filestore/libs/diagnostics/critical_events.h>
 #include <cloud/storage/core/libs/common/file_io_service.h>
 
 #include <util/string/builder.h>
@@ -28,29 +29,46 @@ NProto::TCreateHandleResponse TLocalFileSystem::CreateHandle(
         return TErrorResponse(ErrorInvalidParent(request.GetNodeId()));
     }
 
-    const int flags = HandleFlagsToSystem(request.GetFlags());
+    int flags = HandleFlagsToSystem(request.GetFlags());
     const int mode = request.GetMode()
         ? request.GetMode() : Config->GetDefaultPermissions();
 
     TFileHandle handle;
     TFileStat stat;
+    ui64 nodeId;
     if (const auto& pathname = request.GetName()) {
         handle = node->OpenHandle(pathname, flags, mode);
 
         auto newnode = TIndexNode::Create(*node, pathname);
         stat = newnode->Stat();
-
-        session->TryInsertNode(std::move(newnode));
+        nodeId = newnode->GetNodeId();
+
+        if (!session->TryInsertNode(
+                std::move(newnode),
+                node->GetNodeId(),
+                pathname))
+        {
+            ReportLocalFsMaxSessionNodesInUse();
+            return TErrorResponse(ErrorNoSpaceLeft());
+        }
     } else {
         handle = node->OpenHandle(flags);
         stat = node->Stat();
+        nodeId = node->GetNodeId();
     }
 
-    const FHANDLE fd = handle;
-    session->InsertHandle(std::move(handle));
+    // Don't persist flags that only make sense on initial open
+    flags = flags & ~(O_CREAT | O_EXCL | O_TRUNC);
+
+    auto [handleId, error] =
+        session->InsertHandle(std::move(handle), nodeId, flags);
+    if (HasError(error)) {
+        ReportLocalFsMaxSessionFileHandlesInUse();
+        return TErrorResponse(error);
+    }
 
     NProto::TCreateHandleResponse response;
-    response.SetHandle(fd);
+    response.SetHandle(handleId);
     ConvertStats(stat, *response.MutableNodeAttr());
 
     return response;

cloud/filestore/libs/service_local/fs_node.cpp

@@ -1,5 +1,7 @@
 #include "fs.h"
 
+#include <cloud/filestore/libs/diagnostics/critical_events.h>
+
 #include <util/generic/guid.h>
 
 namespace NCloud::NFileStore {
@@ -80,7 +82,14 @@ NProto::TCreateNodeResponse TLocalFileSystem::CreateNode(
     }
 
     auto stat = target->Stat();
-    session->TryInsertNode(std::move(target));
+    if (!session->TryInsertNode(
+            std::move(target),
+            parent->GetNodeId(),
+            request.GetName()))
+    {
+        ReportLocalFsMaxSessionNodesInUse();
+        return TErrorResponse(ErrorNoSpaceLeft());
+    }
 
     NProto::TCreateNodeResponse response;
     ConvertStats(stat, *response.MutableNode());
@@ -192,7 +201,14 @@ NProto::TListNodesResponse TLocalFileSystem::ListNodes(
         if (!session->LookupNode(entry.second.INode)) {
             auto node = TryCreateChildNode(*parent, entry.first);
             if (node && node->GetNodeId() == entry.second.INode) {
-                session->TryInsertNode(std::move(node));
+                if (!session->TryInsertNode(
+                        std::move(node),
+                        parent->GetNodeId(),
+                        entry.first))
+                {
+                    ReportLocalFsMaxSessionNodesInUse();
+                    return TErrorResponse(ErrorNoSpaceLeft());
+                }
             }
         }
 

cloud/filestore/libs/service_local/fs_session.cpp

@@ -1,6 +1,5 @@
 #include "fs.h"
 
-#include <util/generic/guid.h>
 #include <util/string/builder.h>
 
 namespace NCloud::NFileStore {
@@ -13,7 +12,7 @@ NProto::TCreateSessionResponse TLocalFileSystem::CreateSession(
     STORAGE_INFO("CreateSession " << DumpMessage(request));
 
     const auto& clientId = GetClientId(request);
-    auto sessionId = GetSessionId(request);
+    const auto& sessionId = GetSessionId(request);
     const auto sessionSeqNo = request.GetMountSeqNumber();
     const auto readOnly = request.GetReadOnly();
 
@@ -46,22 +45,31 @@ NProto::TCreateSessionResponse TLocalFileSystem::CreateSession(
         (*it->second)->GetInfo(*response.MutableSession(), sessionSeqNo);
         return response;
     }
-    sessionId = CreateGuidAsString();
+
+    auto clientSessionStatePath = StatePath / ("client_" + clientId);
+    clientSessionStatePath.MkDir();
 
     session = std::make_shared<TSession>(
-        Root.GetPath(),
+        Store.GetFileSystemId(),
+        Root,
+        clientSessionStatePath,
         clientId,
-        sessionId,
-        Index);
+        Index,
+        Logging);
 
+    session->Init(
+        request.GetRestoreClientSession(),
+        Config->GetMaxHandlePerSessionCount());
     session->AddSubSession(sessionSeqNo, readOnly);
 
     SessionsList.push_front(session);
 
-    auto [_, inserted1] = SessionsByClient.emplace(clientId, SessionsList.begin());
+    auto [_, inserted1] =
+        SessionsByClient.emplace(clientId, SessionsList.begin());
     Y_ABORT_UNLESS(inserted1);
 
-    auto [dummyIt, inserted2] = SessionsById.emplace(sessionId, SessionsList.begin());
+    auto [dummyIt, inserted2] =
+        SessionsById.emplace(session->SessionId, SessionsList.begin());
     Y_ABORT_UNLESS(inserted2);
 
     NProto::TCreateSessionResponse response;
@@ -160,6 +168,8 @@ void TLocalFileSystem::RemoveSession(
         SessionsList.erase(it->second);
         SessionsById.erase(it);
     }
+
+    session->StatePath.ForceDelete();
 }
 
 void TLocalFileSystem::ScheduleCleanupSessions()