From aea322d4df0c845fb10f6ca2d177e99bdeb715c4 Mon Sep 17 00:00:00 2001 From: Evgeny Budilovsky Date: Tue, 22 Oct 2024 13:33:21 +0300 Subject: [PATCH] [Filestore] issue-2190: support reconnect for local file system (#2103) --- cloud/filestore/config/server.proto | 11 + .../libs/diagnostics/critical_events.h | 4 + cloud/filestore/libs/service_local/config.cpp | 3 + cloud/filestore/libs/service_local/config.h | 3 + cloud/filestore/libs/service_local/fs.cpp | 20 +- cloud/filestore/libs/service_local/fs.h | 4 +- .../filestore/libs/service_local/fs_attrs.cpp | 11 +- .../filestore/libs/service_local/fs_data.cpp | 30 +- .../filestore/libs/service_local/fs_node.cpp | 20 +- .../libs/service_local/fs_session.cpp | 26 +- cloud/filestore/libs/service_local/index.h | 173 ++++++++- .../filestore/libs/service_local/index_ut.cpp | 274 ++++++++++++++ .../filestore/libs/service_local/service.cpp | 25 +- .../libs/service_local/service_ut.cpp | 335 +++++++++++++++++- cloud/filestore/libs/service_local/session.h | 218 ++++++++++-- cloud/filestore/libs/service_local/ut/ya.make | 1 + cloud/filestore/libs/service_local/ya.make | 1 + .../fio/qemu-local-noserver-test/ya.make | 1 + 18 files changed, 1096 insertions(+), 64 deletions(-) create mode 100644 cloud/filestore/libs/service_local/index_ut.cpp diff --git a/cloud/filestore/config/server.proto b/cloud/filestore/config/server.proto index 291f9e3d82e..59f9108be05 100644 --- a/cloud/filestore/config/server.proto +++ b/cloud/filestore/config/server.proto @@ -72,6 +72,17 @@ message TLocalServiceConfig optional uint32 DefaultPermissions = 3; optional uint32 IdleSessionTimeout = 4; optional uint32 NumThreads = 5; + + // directory where state for session recovery is saved + optional string StatePath = 6; + + // Maximum number of nodes which can be opened by all sessions + // for single local file system + optional uint32 MaxNodeCount = 7; + + // Maximum number of file handles which can be opened by single session + // for single local file system + optional uint32 MaxHandlePerSessionCount = 8; } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/libs/diagnostics/critical_events.h b/cloud/filestore/libs/diagnostics/critical_events.h index f7553091f84..e1905a2e1b3 100644 --- a/cloud/filestore/libs/diagnostics/critical_events.h +++ b/cloud/filestore/libs/diagnostics/critical_events.h @@ -26,6 +26,9 @@ namespace NCloud::NFileStore{ xxx(InvalidDupCacheEntry) \ xxx(GeneratedOrphanNode) \ xxx(ReceivedNodeOpErrorFromShard) \ + xxx(LocalFsMaxSessionNodesInUse) \ + xxx(LocalFsMaxSessionFileHandlesInUse) \ + xxx(LocalFsMissingHandleNode) \ // FILESTORE_CRITICAL_EVENTS #define FILESTORE_IMPOSSIBLE_EVENTS(xxx) \ @@ -41,6 +44,7 @@ namespace NCloud::NFileStore{ xxx(NewChildNodeIsNull) \ xxx(IndexOutOfBounds) \ xxx(CheckFreshBytesFailed) \ + xxx(LocalFsDuplicateFileHandle) \ // FILESTORE_IMPOSSIBLE_EVENTS //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/libs/service_local/config.cpp b/cloud/filestore/libs/service_local/config.cpp index 7e61d1fdeeb..bd85861d68e 100644 --- a/cloud/filestore/libs/service_local/config.cpp +++ b/cloud/filestore/libs/service_local/config.cpp @@ -14,6 +14,9 @@ namespace { xxx(DefaultPermissions, ui32, 0775 )\ xxx(IdleSessionTimeout, TDuration, TDuration::Seconds(30) )\ xxx(NumThreads, ui32, 4 )\ + xxx(StatePath, TString, "./" )\ + xxx(MaxNodeCount, ui32, 1000000 )\ + xxx(MaxHandlePerSessionCount, ui32, 10000 )\ // FILESTORE_SERVICE_CONFIG #define FILESTORE_SERVICE_DECLARE_CONFIG(name, type, value) \ diff --git a/cloud/filestore/libs/service_local/config.h b/cloud/filestore/libs/service_local/config.h index 1332b19e8a3..a2af2fc7241 100644 --- a/cloud/filestore/libs/service_local/config.h +++ b/cloud/filestore/libs/service_local/config.h @@ -26,6 +26,9 @@ class TLocalFileStoreConfig ui32 GetDefaultPermissions() const; TDuration GetIdleSessionTimeout() const; ui32 GetNumThreads() const; + TString GetStatePath() const; + ui32 GetMaxNodeCount() const; + ui32 GetMaxHandlePerSessionCount() const; void Dump(IOutputStream& out) const; void DumpHtml(IOutputStream& out) const; diff --git a/cloud/filestore/libs/service_local/fs.cpp b/cloud/filestore/libs/service_local/fs.cpp index 7adf5ff83a8..c35a8ec71ca 100644 --- a/cloud/filestore/libs/service_local/fs.cpp +++ b/cloud/filestore/libs/service_local/fs.cpp @@ -8,29 +8,29 @@ TLocalFileSystem::TLocalFileSystem( TLocalFileStoreConfigPtr config, NProto::TFileStore store, TFsPath root, + TFsPath statePath, ITimerPtr timer, ISchedulerPtr scheduler, ILoggingServicePtr logging, IFileIOServicePtr fileIOService) : Config(std::move(config)) , Root(std::move(root)) + , StatePath(std::move(statePath)) , Timer(std::move(timer)) , Scheduler(std::move(scheduler)) + , Logging(std::move(logging)) , FileIOService(std::move(fileIOService)) , Store(std::move(store)) { - Log = logging->CreateLog(Store.GetFileSystemId()); + Log = Logging->CreateLog(Store.GetFileSystemId()); - InitIndex(); - ScheduleCleanupSessions(); -} + Index = std::make_shared( + Root, + StatePath, + Config->GetMaxNodeCount(), + Log); -void TLocalFileSystem::InitIndex() -{ - TLocalIndex::TNodeMap nodes; - nodes.insert(TIndexNode::CreateRoot(Root)); - - Index = std::make_shared(std::move(nodes)); + ScheduleCleanupSessions(); } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/libs/service_local/fs.h b/cloud/filestore/libs/service_local/fs.h index bc03886aed3..52381603d4a 100644 --- a/cloud/filestore/libs/service_local/fs.h +++ b/cloud/filestore/libs/service_local/fs.h @@ -89,8 +89,10 @@ class TLocalFileSystem final private: const TLocalFileStoreConfigPtr Config; const TFsPath Root; + const TFsPath StatePath; const ITimerPtr Timer; const ISchedulerPtr Scheduler; + const ILoggingServicePtr Logging; const IFileIOServicePtr FileIOService; NProto::TFileStore Store; @@ -111,6 +113,7 @@ class TLocalFileSystem final TLocalFileStoreConfigPtr config, NProto::TFileStore store, TFsPath root, + TFsPath statePath, ITimerPtr timer, ISchedulerPtr scheduler, ILoggingServicePtr logging, @@ -144,7 +147,6 @@ class TLocalFileSystem final } private: - void InitIndex(); void ScheduleCleanupSessions(); void CleanupSessions(); diff --git a/cloud/filestore/libs/service_local/fs_attrs.cpp b/cloud/filestore/libs/service_local/fs_attrs.cpp index 6fba77a52ca..c97d473f9e3 100644 --- a/cloud/filestore/libs/service_local/fs_attrs.cpp +++ b/cloud/filestore/libs/service_local/fs_attrs.cpp @@ -1,5 +1,7 @@ #include "fs.h" +#include + #include namespace NCloud::NFileStore { @@ -87,7 +89,14 @@ NProto::TGetNodeAttrResponse TLocalFileSystem::GetNodeAttr( // TODO: better? race between statting one child and creating another one // but maybe too costly... stat = child->Stat(); - session->TryInsertNode(std::move(child)); + if (!session->TryInsertNode( + std::move(child), + node->GetNodeId(), + name)) + { + ReportLocalFsMaxSessionNodesInUse(); + return TErrorResponse(ErrorNoSpaceLeft()); + } } } else { stat = node->Stat(); diff --git a/cloud/filestore/libs/service_local/fs_data.cpp b/cloud/filestore/libs/service_local/fs_data.cpp index 606a67cc977..d866a5a5db4 100644 --- a/cloud/filestore/libs/service_local/fs_data.cpp +++ b/cloud/filestore/libs/service_local/fs_data.cpp @@ -2,6 +2,7 @@ #include "lowlevel.h" +#include #include #include @@ -28,29 +29,46 @@ NProto::TCreateHandleResponse TLocalFileSystem::CreateHandle( return TErrorResponse(ErrorInvalidParent(request.GetNodeId())); } - const int flags = HandleFlagsToSystem(request.GetFlags()); + int flags = HandleFlagsToSystem(request.GetFlags()); const int mode = request.GetMode() ? request.GetMode() : Config->GetDefaultPermissions(); TFileHandle handle; TFileStat stat; + ui64 nodeId; if (const auto& pathname = request.GetName()) { handle = node->OpenHandle(pathname, flags, mode); auto newnode = TIndexNode::Create(*node, pathname); stat = newnode->Stat(); - - session->TryInsertNode(std::move(newnode)); + nodeId = newnode->GetNodeId(); + + if (!session->TryInsertNode( + std::move(newnode), + node->GetNodeId(), + pathname)) + { + ReportLocalFsMaxSessionNodesInUse(); + return TErrorResponse(ErrorNoSpaceLeft()); + } } else { handle = node->OpenHandle(flags); stat = node->Stat(); + nodeId = node->GetNodeId(); } - const FHANDLE fd = handle; - session->InsertHandle(std::move(handle)); + // Don't persist flags that only make sense on initial open + flags = flags & ~(O_CREAT | O_EXCL | O_TRUNC); + + auto [handleId, error] = + session->InsertHandle(std::move(handle), nodeId, flags); + if (HasError(error)) { + ReportLocalFsMaxSessionFileHandlesInUse(); + return TErrorResponse(error); + } NProto::TCreateHandleResponse response; - response.SetHandle(fd); + response.SetHandle(handleId); ConvertStats(stat, *response.MutableNodeAttr()); return response; diff --git a/cloud/filestore/libs/service_local/fs_node.cpp b/cloud/filestore/libs/service_local/fs_node.cpp index ebd7769188d..b5db31319ac 100644 --- a/cloud/filestore/libs/service_local/fs_node.cpp +++ b/cloud/filestore/libs/service_local/fs_node.cpp @@ -1,5 +1,7 @@ #include "fs.h" +#include + #include namespace NCloud::NFileStore { @@ -80,7 +82,14 @@ NProto::TCreateNodeResponse TLocalFileSystem::CreateNode( } auto stat = target->Stat(); - session->TryInsertNode(std::move(target)); + if (!session->TryInsertNode( + std::move(target), + parent->GetNodeId(), + request.GetName())) + { + ReportLocalFsMaxSessionNodesInUse(); + return TErrorResponse(ErrorNoSpaceLeft()); + } NProto::TCreateNodeResponse response; ConvertStats(stat, *response.MutableNode()); @@ -192,7 +201,14 @@ NProto::TListNodesResponse TLocalFileSystem::ListNodes( if (!session->LookupNode(entry.second.INode)) { auto node = TryCreateChildNode(*parent, entry.first); if (node && node->GetNodeId() == entry.second.INode) { - session->TryInsertNode(std::move(node)); + if (!session->TryInsertNode( + std::move(node), + parent->GetNodeId(), + entry.first)) + { + ReportLocalFsMaxSessionNodesInUse(); + return TErrorResponse(ErrorNoSpaceLeft()); + } } } diff --git a/cloud/filestore/libs/service_local/fs_session.cpp b/cloud/filestore/libs/service_local/fs_session.cpp index 56cc0b1ea25..ce6eb891424 100644 --- a/cloud/filestore/libs/service_local/fs_session.cpp +++ b/cloud/filestore/libs/service_local/fs_session.cpp @@ -1,6 +1,5 @@ #include "fs.h" -#include #include namespace NCloud::NFileStore { @@ -13,7 +12,7 @@ NProto::TCreateSessionResponse TLocalFileSystem::CreateSession( STORAGE_INFO("CreateSession " << DumpMessage(request)); const auto& clientId = GetClientId(request); - auto sessionId = GetSessionId(request); + const auto& sessionId = GetSessionId(request); const auto sessionSeqNo = request.GetMountSeqNumber(); const auto readOnly = request.GetReadOnly(); @@ -46,22 +45,31 @@ NProto::TCreateSessionResponse TLocalFileSystem::CreateSession( (*it->second)->GetInfo(*response.MutableSession(), sessionSeqNo); return response; } - sessionId = CreateGuidAsString(); + + auto clientSessionStatePath = StatePath / ("client_" + clientId); + clientSessionStatePath.MkDir(); session = std::make_shared( - Root.GetPath(), + Store.GetFileSystemId(), + Root, + clientSessionStatePath, clientId, - sessionId, - Index); + Index, + Logging); + session->Init( + request.GetRestoreClientSession(), + Config->GetMaxHandlePerSessionCount()); session->AddSubSession(sessionSeqNo, readOnly); SessionsList.push_front(session); - auto [_, inserted1] = SessionsByClient.emplace(clientId, SessionsList.begin()); + auto [_, inserted1] = + SessionsByClient.emplace(clientId, SessionsList.begin()); Y_ABORT_UNLESS(inserted1); - auto [dummyIt, inserted2] = SessionsById.emplace(sessionId, SessionsList.begin()); + auto [dummyIt, inserted2] = + SessionsById.emplace(session->SessionId, SessionsList.begin()); Y_ABORT_UNLESS(inserted2); NProto::TCreateSessionResponse response; @@ -160,6 +168,8 @@ void TLocalFileSystem::RemoveSession( SessionsList.erase(it->second); SessionsById.erase(it); } + + session->StatePath.ForceDelete(); } void TLocalFileSystem::ScheduleCleanupSessions() diff --git a/cloud/filestore/libs/service_local/index.h b/cloud/filestore/libs/service_local/index.h index 375b24cb4a9..10f734f91be 100644 --- a/cloud/filestore/libs/service_local/index.h +++ b/cloud/filestore/libs/service_local/index.h @@ -2,9 +2,14 @@ #include "public.h" +#include +#include + #include #include #include +#include +#include #include #include #include @@ -20,6 +25,7 @@ class TIndexNode private: const ui64 NodeId; const TFileHandle NodeFd; + ui64 RecordIndex = -1; public: TIndexNode(ui64 nodeId, TFileHandle node) @@ -27,6 +33,16 @@ class TIndexNode , NodeFd(std::move(node)) {} + ui64 GetRecordIndex() const + { + return RecordIndex; + } + + void SetRecordIndex(ui64 index) + { + RecordIndex = index; + } + static TIndexNodePtr CreateRoot(const TFsPath& path); static TIndexNodePtr Create(const TIndexNode& parent, const TString& name); @@ -101,17 +117,36 @@ class TLocalIndex } }; -public: - using TNodeMap = THashSet; + struct TNodeTableHeader + { + }; + + struct TNodeTableRecord + { + ui64 NodeId = 0; + ui64 ParentNodeId = 0; + char Name[NAME_MAX + 1] = {}; + }; private: + using TNodeMap = THashSet; + using TNodeTable = TPersistentTable; + TNodeMap Nodes; + std::unique_ptr NodeTable; TRWMutex NodesLock; + TLog Log; public: - TLocalIndex(TNodeMap nodes) - : Nodes(std::move(nodes)) - {} + TLocalIndex( + const TFsPath& root, + const TFsPath& statePath, + ui32 maxNodeCount, + TLog log) + : Log(std::move(log)) + { + Init(root, statePath, maxNodeCount); + } TIndexNodePtr LookupNode(ui64 nodeId) { @@ -125,30 +160,154 @@ class TLocalIndex return *it; } - bool TryInsertNode(TIndexNodePtr node) + [[nodiscard]] bool + TryInsertNode(TIndexNodePtr node, ui64 parentNodeId, const TString& name) { TWriteGuard guard(NodesLock); auto it = Nodes.find(node->GetNodeId()); if (it != Nodes.end()) { + // TODO: we can find existing node id for hard link since it has the + // same node id + return true; + } + + auto recordIndex = NodeTable->AllocRecord(); + if (recordIndex == TNodeTable::InvalidIndex) { return false; } + auto* record = NodeTable->RecordData(recordIndex); + + record->NodeId = node->GetNodeId(); + record->ParentNodeId = parentNodeId; + + std::strncpy(record->Name, name.c_str(), NAME_MAX); + record->Name[NAME_MAX] = 0; + + NodeTable->CommitRecord(recordIndex); + + node->SetRecordIndex(recordIndex); Nodes.emplace(std::move(node)); + return true; } - void ForgetNode(ui64 nodeId) + TIndexNodePtr ForgetNode(ui64 nodeId) { TWriteGuard guard(NodesLock); + TIndexNodePtr node = nullptr; auto it = Nodes.find(nodeId); if (it != Nodes.end()) { + node = *it; + NodeTable->DeleteRecord(node->GetRecordIndex()); Nodes.erase(it); } + + return node; } private: + void Init(const TFsPath& root, const TFsPath& statePath, ui32 maxNodeCount) + { + STORAGE_INFO( + "Init index, Root=" << root << + ", StatePath=" << statePath + << ", MaxNodeCount=" << maxNodeCount); + + Nodes.insert(TIndexNode::CreateRoot(root)); + + NodeTable = std::make_unique( + (statePath / "nodes").GetPath(), + maxNodeCount); + + RecoverNodesFromPersistentTable(); + } + + void RecoverNodesFromPersistentTable() + { + // enties are ordered by NodeId in TMap but this doesn't mean that + // a/b/c/d has order a, b, c, d usually inode number increased but + // directories can move so directory a which was created later can + // contain directory b which was created before so and NodeId(a) > + // NodeId(b) for a/b + TMap unresolvedRecords; + for (auto it = NodeTable->begin(); it != NodeTable->end(); it++) { + unresolvedRecords[it->NodeId] = it.GetIndex(); + STORAGE_TRACE( + "Unresolved record, NodeId=" << it->NodeId << ", ParentNodeId=" + << it->ParentNodeId + << ", Name=" << it->Name); + } + + while (!unresolvedRecords.empty()) { + TStack unresolvedPath; + unresolvedPath.push(unresolvedRecords.begin()->second); + + // For entry /a we can resolve immediately and create TIndexNode + // but for entry d in /a/b/c/d path we must resolve the whole path + // recursively + while (!unresolvedPath.empty()) { + auto pathElemIndex = unresolvedPath.top(); + auto* pathElemRecord = NodeTable->RecordData(pathElemIndex); + + STORAGE_TRACE( + "Resolve node start, NodeId=" << pathElemRecord->NodeId); + + auto parentNodeIt = Nodes.find(pathElemRecord->ParentNodeId); + if (parentNodeIt == Nodes.end()) { + // parent is not resolved + + STORAGE_TRACE( + "Need to resolve parent NodeId=" + << pathElemRecord->ParentNodeId); + auto parentRecordIt = + unresolvedRecords.find(pathElemRecord->ParentNodeId); + if (parentRecordIt == unresolvedRecords.end()) { + // parent was not saved in persistent table so we can't + // resolve it in case of d in path /a/b/c/d if we + // discover that b can't be resolved we need to discard + // b, c, d inodes + STORAGE_ERROR( + "Parent node is missing in table, NodeId=" + << pathElemRecord->ParentNodeId); + while (!unresolvedPath.empty()) { + auto discardedIndex = unresolvedPath.top(); + auto* discardedRecord = + NodeTable->RecordData(discardedIndex); + STORAGE_WARN( + "Discarding NodeId=" + << discardedRecord->NodeId); + unresolvedRecords.erase(discardedRecord->NodeId); + NodeTable->DeleteRecord(discardedIndex); + unresolvedPath.pop(); + } + continue; + } + + // add to unresolvedPath and solve recursively + unresolvedPath.push(parentRecordIt->second); + continue; + } + + // parent already resolved so we can create node and resolve + // this entry + auto node = + TIndexNode::Create(**parentNodeIt, pathElemRecord->Name); + node->SetRecordIndex(pathElemIndex); + + Nodes.insert(std::move(node)); + + unresolvedPath.pop(); + unresolvedRecords.erase(pathElemRecord->NodeId); + + STORAGE_TRACE( + "Resolve node end, NodeId=" << pathElemRecord->NodeId); + } + } + } + static auto GetNodeId(const TIndexNodePtr& node) { return node->GetNodeId(); diff --git a/cloud/filestore/libs/service_local/index_ut.cpp b/cloud/filestore/libs/service_local/index_ut.cpp new file mode 100644 index 00000000000..ef124431ca9 --- /dev/null +++ b/cloud/filestore/libs/service_local/index_ut.cpp @@ -0,0 +1,274 @@ +#include "index.h" + +#include + +#include + +#include + +namespace NCloud::NFileStore { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +struct TEnvironment + : public NUnitTest::TBaseFixture +{ + ILoggingServicePtr Logging = + CreateLoggingService("console", {TLOG_RESOURCES}); + TLog Log; + + const TTempDir TempDir; + const TFsPath RootPath = TempDir.Path() / "root"; + const TFsPath StatePath = TempDir.Path() / "state"; + + TEnvironment() : Log(Logging->CreateLog("INDEX_TEST")) + { + RootPath.MkDir(); + StatePath.MkDir(); + } + +protected: + bool CreateNestedDir( + ui32 pathLen, + TMap& nodeMap) + { + nodeMap.clear(); + + RootPath.ForceDelete(); + RootPath.MkDir(); + + StatePath.ForceDelete(); + StatePath.MkDir(); + + TLocalIndex index(RootPath, StatePath, pathLen, Log); + + auto node = index.LookupNode(RootNodeId); + UNIT_ASSERT_C(node, "Failed to lookup RootNode"); + + auto path = RootPath; + for (ui32 i = 0; i < pathLen; i++) { + TString name = ToString(i); + path = path / name; + path.MkDir(); + + auto childNode = TIndexNode::Create(*node, name); + UNIT_ASSERT_C(childNode, "Failed to create node: " << name); + + STORAGE_DEBUG( + "NodeId=" << childNode->GetNodeId() << " ,Name=" << name); + + auto inserted = + index.TryInsertNode(childNode, node->GetNodeId(), name); + UNIT_ASSERT_C(inserted, "Failed to insert node: " << name); + + nodeMap.emplace(name, childNode->GetNodeId()); + if (childNode->GetNodeId() <= node->GetNodeId()) { + // Rarely inodes reused and not increased when new dir created + // in this case we skip the test + STORAGE_WARN( + "node id=" << node->GetNodeId() << + " , child node id=" << childNode->GetNodeId()); + return false; + } + + UNIT_ASSERT_LT_C( + node->GetNodeId(), + childNode->GetNodeId(), + "node id=" << node->GetNodeId() << + " , child node id=" << childNode->GetNodeId()); + node = childNode; + } + return true; + } + + void SafeCreateNestedDir(ui32 pathLen, TMap& nodeMap) + { + for (int i = 0; i < 10; i++) { + if (CreateNestedDir(pathLen, nodeMap)) { + return; + } + STORAGE_WARN("Failed to create nested dir in iteration #" << i); + } + UNIT_ASSERT_C( + false, + "Failed to create nested dir with increasing inode numbers"); + } + + bool CreateReversedNodeIdNestedDir( + ui32 pathLen, + TMap& nodeMap) + { + nodeMap.clear(); + + RootPath.ForceDelete(); + RootPath.MkDir(); + + StatePath.ForceDelete(); + StatePath.MkDir(); + + TLocalIndex index(RootPath, StatePath, pathLen, Log); + + auto node = index.LookupNode(RootNodeId); + UNIT_ASSERT_C(node, "Failed to lookup RootNode"); + + auto path = RootPath; + for (ui32 i = 0; i < pathLen; i++) { + TString name = ToString(pathLen - 1 - i); + path = RootPath / name; + path.MkDir(); + + if (i > 0) { + TString prevName = ToString(pathLen - i); + auto prevPath = RootPath / prevName; + prevPath.RenameTo(RootPath / name / prevName); + } + } + + for (ui32 i = 0; i < pathLen; i++) { + TString name = ToString(i); + + auto childNode = TIndexNode::Create(*node, name); + UNIT_ASSERT_C(childNode, "Failed to create node: " << name); + + STORAGE_DEBUG( + "NodeId=" << childNode->GetNodeId() << " ,Name=" << name); + + auto inserted = + index.TryInsertNode(childNode, node->GetNodeId(), name); + UNIT_ASSERT_C(inserted, "Failed to insert node: " << name); + + nodeMap.emplace(name, childNode->GetNodeId()); + if (i > 0 && node->GetNodeId() <= childNode->GetNodeId()) { + // Rarely inodes reused and not increased when new dir created + // in this case we skip the test + STORAGE_WARN( + "node id=" << node->GetNodeId() << + " , child node id=" << childNode->GetNodeId()); + return false; + } + node = childNode; + } + + return true; + } + + void SafeCreateReversedNodeIdNestedDir(ui32 pathLen, TMap& nodeMap) + { + for (int i = 0; i < 10; i++) { + if (CreateReversedNodeIdNestedDir(pathLen, nodeMap)) { + return; + } + STORAGE_WARN("Failed to create nested dir in iteration #" << i); + } + UNIT_ASSERT_C( + false, + "Failed to create nested dir with decreaing inode numbers"); + } + + void CheckNestedDir(ui32 pathLen, const TMap& nodeMap) + { + TLocalIndex index(RootPath, StatePath, pathLen, Log); + auto node = index.LookupNode(RootNodeId); + UNIT_ASSERT_C(node, "Failed to lookup root node"); + + for (ui32 i = 0; i < pathLen; i++) { + TString name = ToString(i); + + auto nodes = node->List(); + UNIT_ASSERT_VALUES_EQUAL(nodes.size(), 1); + + auto& [nodeName, nodeStat] = nodes[0]; + UNIT_ASSERT_VALUES_EQUAL(name, nodeName); + + auto it = nodeMap.find(nodeName); + UNIT_ASSERT_C(it != nodeMap.end(), "node not found: " << nodeName); + + auto nodeId = it->second; + STORAGE_DEBUG("Found node: " << nodeName << ", NodeId=" << nodeId); + + node = index.LookupNode(nodeId); + UNIT_ASSERT_C(node, + "Failed to lookup node id: " << nodeId << + ", node: " << nodeName); + } + } + + void CheckMissingNodes(ui32 pathLen, const TVector& nodeIds) + { + TLocalIndex index(RootPath, StatePath, pathLen, Log); + auto node = index.LookupNode(RootNodeId); + UNIT_ASSERT_C(node, "Failed to lookup root node"); + + for (auto& nodeId: nodeIds) { + node = index.LookupNode(nodeId); + UNIT_ASSERT_C(!node, "Found node id: " << nodeId); + } + } +}; + +struct TNodeTableHeader +{ +}; + +struct TNodeTableRecord +{ + ui64 NodeId = 0; + ui64 ParentNodeId = 0; + char Name[NAME_MAX + 1]; +}; + +using TNodeTable = TPersistentTable; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TLocalIndex) +{ + Y_UNIT_TEST_F(ShouldRecoverNestedDir, TEnvironment) + { + ui32 pathLen = 10; + TMap nodeMap; + + SafeCreateNestedDir(pathLen, nodeMap); + CheckNestedDir(pathLen, nodeMap); + } + + Y_UNIT_TEST_F(ShouldRecoverNestedDirWithReversedInodeOrder, TEnvironment) + { + ui32 pathLen = 10; + TMap nodeMap; + + SafeCreateReversedNodeIdNestedDir(pathLen, nodeMap); + CheckNestedDir(pathLen, nodeMap); + } + + Y_UNIT_TEST_F(ShouldDiscardUnresolvedNodePath, TEnvironment) + { + ui32 pathLen = 10; + TMap nodeMap; + + SafeCreateNestedDir(pathLen, nodeMap); + + TNodeTable nodeTable((StatePath / "nodes").GetPath(), pathLen); + + nodeTable.DeleteRecord(pathLen / 2); + + CheckNestedDir(pathLen / 2, nodeMap); + + TVector missingNodes; + for (ui32 i = pathLen / 2; i < pathLen; i++) { + TString name = ToString(i); + auto it = nodeMap.find(name); + UNIT_ASSERT_C(it != nodeMap.end(), "Node " << name << " not found"); + + missingNodes.push_back(it->second); + } + + CheckMissingNodes(pathLen, missingNodes); + } +}; + +} // namespace NCloud::NFileStore diff --git a/cloud/filestore/libs/service_local/service.cpp b/cloud/filestore/libs/service_local/service.cpp index cd65c098dd9..d05c4df8cd4 100644 --- a/cloud/filestore/libs/service_local/service.cpp +++ b/cloud/filestore/libs/service_local/service.cpp @@ -334,9 +334,15 @@ FILESTORE_SERVICE_LOCAL_ASYNC(FILESTORE_IMPLEMENT_METHOD_ASYNC) TLocalFileSystemPtr InitFileSystem( const TString& id, const TFsPath& root, + const TFsPath& statePath, const NProto::TFileStore& store); TLocalFileSystemPtr FindFileSystem(const TString& id); + + TFsPath GetStatePath(const TString name) + { + return Concat(Config->GetStatePath(), ".state_" + name); + } }; //////////////////////////////////////////////////////////////////////////////// @@ -359,11 +365,14 @@ void TLocalFileStore::Start() continue; } + TFsPath statePath = GetStatePath(child.GetName()); + statePath.MkDir(Config->GetDefaultPermissions()); + STORAGE_INFO("restoring local store " << id.Quote()); NProto::TFileStore store; LoadFileStoreProto(path, store); - InitFileSystem(id, child, store); + InitFileSystem(id, child, statePath, store); STORAGE_INFO("restored local store " << id.Quote()); } @@ -411,7 +420,10 @@ NProto::TCreateFileStoreResponse TLocalFileStore::CreateFileStore( TFsPath root = Concat(Config->GetRootPath(), name); root.MkDir(Config->GetDefaultPermissions()); - InitFileSystem(id, root, store); + TFsPath statePath = GetStatePath(name); + statePath.MkDir(Config->GetDefaultPermissions()); + + InitFileSystem(id, root, statePath, store); NProto::TCreateFileStoreResponse response; response.MutableFileStore()->Swap(&store); @@ -437,9 +449,14 @@ NProto::TDestroyFileStoreResponse TLocalFileStore::DestroyFileStore( << "invalid file system: " << id.Quote()); } - TFsPath path = Concat(Config->GetRootPath(), Config->GetPathPrefix() + id); + const TString& name = Config->GetPathPrefix() + id; + + TFsPath path = Concat(Config->GetRootPath(), name); path.ForceDelete(); + TFsPath statePath = GetStatePath(name); + statePath.ForceDelete(); + FileSystems.erase(it); return {}; @@ -492,12 +509,14 @@ NProto::TListFileStoresResponse TLocalFileStore::ListFileStores( TLocalFileSystemPtr TLocalFileStore::InitFileSystem( const TString& id, const TFsPath& root, + const TFsPath& statePath, const NProto::TFileStore& store) { auto fs = std::make_shared( Config, store, root, + statePath, Timer, Scheduler, Logging, diff --git a/cloud/filestore/libs/service_local/service_ut.cpp b/cloud/filestore/libs/service_local/service_ut.cpp index ce6ee4594cf..5181814a81e 100644 --- a/cloud/filestore/libs/service_local/service_ut.cpp +++ b/cloud/filestore/libs/service_local/service_ut.cpp @@ -286,12 +286,15 @@ struct TTestBootstrap static constexpr pid_t DefaultPid = 123; - TTestBootstrap(const TTempDirectoryPtr& cwd = std::make_shared()) + TTestBootstrap( + const TTempDirectoryPtr& cwd = std::make_shared(), + ui32 maxNodeCount = 1000, + ui32 maxHandlePerSessionCount = 100) : Cwd(cwd) { AIOService->Start(); Store = CreateLocalFileStore( - CreateConfig(), + CreateConfig(maxNodeCount, maxHandlePerSessionCount), Timer, Scheduler, Logging, @@ -300,12 +303,17 @@ struct TTestBootstrap Store->Start(); } - TTestBootstrap(const TString& id, const TString& client = "client", const TString& session = {}) + TTestBootstrap( + const TString& id, + const TString& client = "client", + const TString& session = {}, + ui32 maxNodeCount = 1000, + ui32 maxHandlePerSessionCount = 100) : Cwd(std::make_shared()) { AIOService->Start(); Store = CreateLocalFileStore( - CreateConfig(), + CreateConfig(maxNodeCount, maxHandlePerSessionCount), Timer, Scheduler, Logging, @@ -324,10 +332,54 @@ struct TTestBootstrap AIOService->Stop(); } - TLocalFileStoreConfigPtr CreateConfig() + std::optional GetFsStateDir(const TString& id) + { + TVector dirs; + Cwd->List(dirs); + + for (auto& d: dirs) { + if (d.IsDirectory() && d.GetName().StartsWith(".state") && + d.GetName().EndsWith(id)) + { + return d; + } + } + + return {}; + } + + std::optional GetClientFsStateDir( + const TString& id, + const TString& client = "client") + { + auto fsStateDir = GetFsStateDir(id); + if (!fsStateDir) { + return {}; + } + + TVector dirs; + fsStateDir->List(dirs); + + for (auto& d: dirs) { + if (d.IsDirectory() && d.GetName().StartsWith("client_") && + d.GetName().EndsWith(client)) + { + return d; + } + } + + return {}; + } + + TLocalFileStoreConfigPtr CreateConfig( + ui32 maxNodeCount, + ui32 maxHandlePerSessionCount) { NProto::TLocalServiceConfig config; config.SetRootPath(Cwd->GetName()); + config.SetStatePath(Cwd->GetName()); + config.SetMaxNodeCount(maxNodeCount); + config.SetMaxHandlePerSessionCount(maxHandlePerSessionCount); return std::make_shared(config); } @@ -645,6 +697,7 @@ struct TTestBootstrap return request; } + auto CreateFsyncDirRequest(ui64 node, bool dataSync) { auto request = CreateRequest(); @@ -668,6 +721,15 @@ struct TTestBootstrap } \ \ template \ + NProto::T##name##Response NoAssert##name(Args&&... args) \ + { \ + auto request = Create##name##Request(std::forward(args)...); \ + auto dbg = request->ShortDebugString(); \ + auto response = Store->name(Ctx, std::move(request)).GetValueSync(); \ + return response; \ + } \ + \ + template \ NProto::T##name##Response Assert##name##Failed(Args&&... args) \ { \ auto request = Create##name##Request(std::forward(args)...); \ @@ -698,6 +760,55 @@ ui64 CreateDirectory(TTestBootstrap& bootstrap, ui64 parent, const TString& name return bootstrap.CreateNode(TCreateNodeArgs::Directory(parent, name, mode)).GetNode().GetId(); } +TVector CreateDirectories( + TTestBootstrap& bootstrap, + ui64 parent, + const TString& path, + int mode = 0) +{ + TVector nodes; + + TFsPath fsPath(path); + for (auto& pathPart: fsPath.PathSplit()) { + auto request = bootstrap.CreateRequest(); + auto rsp = bootstrap.NoAssertGetNodeAttr(parent, TString(pathPart)); + if (STATUS_FROM_CODE(rsp.GetError().GetCode()) == NProto::E_FS_NOENT) { + auto nodeId = + bootstrap.CreateNode(TCreateNodeArgs::Directory( + parent, + TString(pathPart), + mode)).GetNode().GetId(); + parent = nodeId; + nodes.push_back(parent); + continue; + } + + UNIT_ASSERT_C( + SUCCEEDED(rsp.GetError().GetCode()), + rsp.GetError().GetMessage() + "@" + request->ShortDebugString()); + + parent = rsp.GetNode().GetId(); + nodes.push_back(parent); + } + return nodes; +} + +void CheckDirectoryPath( + TTestBootstrap& bootstrap, + ui64 parent, + const TString& path, + TVector& expectedNodes) +{ + TFsPath fsPath(path); + auto expectedNodeIndex = 0UL; + for (auto& pathPart: fsPath.PathSplit()) { + UNIT_ASSERT(expectedNodeIndex < expectedNodes.size()); + auto stat = bootstrap.GetNodeAttr(parent, TString(pathPart)).GetNode(); + UNIT_ASSERT_VALUES_EQUAL(stat.GetId(), expectedNodes[expectedNodeIndex++]); + parent = stat.GetId(); + } +} + ui64 CreateHardLink(TTestBootstrap& bootstrap, ui64 parent, const TString& name, ui64 target) { return bootstrap.CreateNode(TCreateNodeArgs::Link(parent, name, target)).GetNode().GetId(); @@ -767,6 +878,7 @@ Y_UNIT_TEST_SUITE(LocalFileStore) { TTestBootstrap bootstrap; bootstrap.CreateFileStore("fs", "cloud", "folder", 100500, 500100); + UNIT_ASSERT(bootstrap.GetFsStateDir("fs")); auto response = bootstrap.CreateSession("fs", "client", ""); auto id = response.GetSession().GetSessionId(); @@ -789,6 +901,8 @@ Y_UNIT_TEST_SUITE(LocalFileStore) // restore bootstrap.CreateSession("fs", "client", id); + UNIT_ASSERT(bootstrap.GetClientFsStateDir("fs", "client")); + bootstrap.AssertCreateSessionFailed("fs", "client-xxx", id); bootstrap.SwitchToSession(session); @@ -802,6 +916,7 @@ Y_UNIT_TEST_SUITE(LocalFileStore) bootstrap.SwitchToSession(session); bootstrap.DestroySession(); + UNIT_ASSERT(!bootstrap.GetClientFsStateDir("fs", "client")); } Y_UNIT_TEST(ShouldNotCreateTheSameStore) @@ -813,7 +928,11 @@ Y_UNIT_TEST_SUITE(LocalFileStore) Y_UNIT_TEST(ShouldDestroyStore) { TTestBootstrap bootstrap("fs"); + UNIT_ASSERT(bootstrap.GetFsStateDir("fs")); + bootstrap.DestroyFileStore("fs"); + UNIT_ASSERT(!bootstrap.GetFsStateDir("fs")); + // intentionally bootstrap.DestroyFileStore("fs"); } @@ -833,6 +952,202 @@ Y_UNIT_TEST_SUITE(LocalFileStore) UNIT_ASSERT_VALUES_EQUAL(store.GetBlocksCount(), 500100); } + Y_UNIT_TEST(ShouldRecoverFsNodes) + { + TTestBootstrap bootstrap("fs", "client"); + auto ctx = MakeIntrusive(); + + TVector>> testPaths = { + {"a/b/c/d/e", {"file1", "file2", "file3"}}, + {"a/b/c/f/g", {"file4", "file5", "file6"}}, + }; + + TVector, TVector, TVector>> + testNodes; + + for (auto& [dir, files]: testPaths) { + auto dirNodes = CreateDirectories(bootstrap, RootNodeId, dir); + TVector fileNodes; + for (auto& file: files) { + auto fileNode = + CreateFile(bootstrap, dirNodes.back(), file, 0755); + fileNodes.push_back(fileNode); + } + + TVector deletedNodes; + deletedNodes.push_back( + CreateFile(bootstrap, dirNodes.back(), "deletedFile", 0755)); + deletedNodes.push_back(CreateDirectory( + bootstrap, + dirNodes.back(), + "deletedDir", + 0755)); + + testNodes.emplace_back( + std::move(dirNodes), + std::move(fileNodes), + std::move(deletedNodes)); + } + + // delete nodes after all files/directories were created so nodes won't + // be reused then we can safely check that delete nodes were not + // recovered + for (auto& nodes: testNodes) { + auto& dirNodes = std::get<0>(nodes); + bootstrap.UnlinkNode(dirNodes.back(), "deletedFile", false); + bootstrap.UnlinkNode(dirNodes.back(), "deletedDir", true); + } + + TTestBootstrap other(bootstrap.Cwd); + + auto id = other.CreateSession("fs", "client", "", false, 0, true) + .GetSession() + .GetSessionId(); + other.Headers.SessionId = id; + UNIT_ASSERT_VALUES_EQUAL( + bootstrap.Headers.SessionId, + other.Headers.SessionId); + + for (ui32 testIndex = 0; testIndex < testPaths.size(); testIndex++) { + auto& dir = testPaths[testIndex].first; + auto& dirNodes = std::get<0>(testNodes[testIndex]); + auto& files = testPaths[testIndex].second; + auto& fileNodes = std::get<1>(testNodes[testIndex]); + auto& deletedNodes = std::get<2>(testNodes[testIndex]); + + CheckDirectoryPath(other, RootNodeId, dir, dirNodes); + + auto listedFiles = ListNames(other, dirNodes.back()); + Sort(listedFiles); + UNIT_ASSERT_VALUES_EQUAL(listedFiles, files); + + for (ui32 fileIndex = 0; fileIndex < files.size(); fileIndex++) { + auto stat = other.GetNodeAttr(dirNodes.back(), files[fileIndex]) + .GetNode(); + UNIT_ASSERT_VALUES_EQUAL(fileNodes[fileIndex], stat.GetId()); + } + + for (auto& deletedNode: deletedNodes) { + auto response = other.AssertGetNodeAttrFailed(deletedNode); + UNIT_ASSERT_VALUES_EQUAL( + E_FS_NOENT, + response.GetError().GetCode()); + } + } + } + + Y_UNIT_TEST(ShouldRecoverSessionHandles) + { + TTestBootstrap bootstrap("fs", "client"); + auto ctx = MakeIntrusive(); + + TVector files = {"file1", "file2", "file3", "file4", "file5"}; + TVector handles; + TString expectedData = "aaaabbbbcccccdddddeeee"; + + for (auto& file: files) { + auto handle = + bootstrap + .CreateHandle(RootNodeId, file, TCreateHandleArgs::CREATE) + .GetHandle(); + + auto data = bootstrap.ReadData(handle, 0, 100).GetBuffer(); + UNIT_ASSERT_VALUES_EQUAL(data.size(), 0); + handles.push_back(handle); + + data = expectedData; + bootstrap.WriteData(handle, 0, data); + + auto buffer = bootstrap.ReadData(handle, 0, 100).GetBuffer(); + UNIT_ASSERT_VALUES_EQUAL(buffer, data); + } + + // close half of the handles + for (ui32 i = 0; i < handles.size(); i++) { + if (i % 2 == 0) { + bootstrap.DestroyHandle(handles[i]); + } + } + + TTestBootstrap other(bootstrap.Cwd); + + auto id = other.CreateSession("fs", "client", "", false, 0, true) + .GetSession() + .GetSessionId(); + other.Headers.SessionId = id; + UNIT_ASSERT_VALUES_EQUAL( + bootstrap.Headers.SessionId, + other.Headers.SessionId); + + for (ui32 i = 0; i < handles.size(); i++) { + if (i % 2 == 0) { + // check closed handles + auto response = other.AssertReadDataFailed(handles[i], 0, 100); + UNIT_ASSERT_VALUES_EQUAL( + E_FS_BADHANDLE, + response.GetError().GetCode()); + } else { + // check open handles + auto buffer = other.ReadData(handles[i], 0, 100).GetBuffer(); + UNIT_ASSERT_VALUES_EQUAL(buffer, expectedData); + } + } + } + + Y_UNIT_TEST(ShouldLimitNumberOfNodesAndHandles) + { + constexpr ui32 maxHandles = 4; + constexpr ui32 maxNodes = maxHandles + 2; + + TTestBootstrap bootstrap("fs", "client", {}, maxNodes, maxHandles); + + TVector nodes; + TVector handles; + for (ui32 i = 0; i < maxNodes+1; i++) { + auto fileName = ToString(i); + if (i >= maxNodes) { + auto response = bootstrap.AssertCreateNodeFailed( + TCreateNodeArgs::File(RootNodeId, fileName, 0755)); + UNIT_ASSERT_VALUES_EQUAL( + E_FS_NOSPC, + response.GetError().GetCode()); + continue; + } + + auto node = CreateFile(bootstrap, RootNodeId, fileName, 0755); + nodes.push_back(node); + + if (i < maxHandles) { + auto handle = + bootstrap.CreateHandle(node, "", TCreateHandleArgs::RDNLY) + .GetHandle(); + handles.push_back(handle); + } else { + auto response = bootstrap.AssertCreateHandleFailed( + node, + "", + TCreateHandleArgs::RDNLY); + UNIT_ASSERT_VALUES_EQUAL( + E_FS_NOSPC, + response.GetError().GetCode()); + } + } + + UNIT_ASSERT_VALUES_EQUAL(maxNodes, nodes.size()); + UNIT_ASSERT_VALUES_EQUAL(maxHandles, handles.size()); + + // make sure that after deleting node it's poissible to create node once + // again + bootstrap.UnlinkNode(RootNodeId, "0", false); + CreateFile(bootstrap, RootNodeId, "100", 0755); + + // make sure that after closing handle it's poissible to open handle + // once again + bootstrap.DestroyHandle(handles[0]); + bootstrap.CreateHandle(nodes[1], "", TCreateHandleArgs::RDNLY) + .GetHandle(); + } + Y_UNIT_TEST(ShouldCreateFileNode) { TTestBootstrap bootstrap("fs"); @@ -1292,6 +1607,16 @@ Y_UNIT_TEST_SUITE(LocalFileStore) auto response = bootstrap.CreateSession("fs", "client", "", true); UNIT_ASSERT_VALUES_EQUAL(response.GetSession().GetSessionState(), state); + + TTestBootstrap other(bootstrap.Cwd); + + auto otherResponse = + other.CreateSession("fs", "client", "", false, 0, true); + UNIT_ASSERT_VALUES_EQUAL(otherResponse.GetSession().GetSessionState(), state); + + UNIT_ASSERT_VALUES_EQUAL( + response.GetSession().GetSessionId(), + otherResponse.GetSession().GetSessionId()); } Y_UNIT_TEST(ShouldSupportMigration) diff --git a/cloud/filestore/libs/service_local/session.h b/cloud/filestore/libs/service_local/session.h index b8a20673849..7fc37ec3fd3 100644 --- a/cloud/filestore/libs/service_local/session.h +++ b/cloud/filestore/libs/service_local/session.h @@ -4,13 +4,21 @@ #include "index.h" +#include +#include #include #include +#include #include +#include #include +#include +#include +#include #include #include +#include namespace NCloud::NFileStore { @@ -20,56 +28,185 @@ class TSession { public: const TFsPath Root; + const TFsPath StatePath; const TString ClientId; - const TString SessionId; + TString SessionId; const TLocalIndexPtr Index; - TString State; - private: + struct THandle + { + TFileHandle FileHandle; + ui64 RecordIndex = -1; + }; + + struct THandleTableHeader + { + }; + + struct THandleTableRecord + { + ui64 HandleId = 0; + ui64 NodeId = 0; + int Flags = 0; + }; + + using THandleTable = + TPersistentTable; + THashMap Attrs; - THashMap Handles; + THashMap Handles; + std::unique_ptr HandleTable; + std::atomic NextHandleId = 0; + TString FuseState; TRWMutex Lock; + const ILoggingServicePtr Logging; + TLog Log; + THashMap> SubSessions; public: TSession( + const TString& fileSystemId, const TFsPath& root, + const TFsPath& statePath, TString clientId, - TString sessionId, - TLocalIndexPtr index) + TLocalIndexPtr index, + ILoggingServicePtr logging) : Root(root.RealPath()) + , StatePath(statePath.RealPath()) , ClientId(std::move(clientId)) - , SessionId(std::move(sessionId)) , Index(std::move(index)) - {} + , Logging(std::move(logging)) + { + Log = Logging->CreateLog(fileSystemId + "." + ClientId); + } + + void Init(bool restoreClientSession, ui32 maxHandlesPerSessionCount) + { + auto handlesPath = StatePath / "handles"; + + if (!restoreClientSession || !HasStateFile("session") || + !HasStateFile("fuse_state")) + { + DeleteStateFile("session"); + DeleteStateFile("fuse_state"); + handlesPath.DeleteIfExists(); + + SessionId = CreateGuidAsString(); + + STORAGE_INFO( + "Create session, StatePath=" << StatePath << + ", SessionId=" << SessionId << + ", MaxHandlesPerSessionCount=" << maxHandlesPerSessionCount); + + WriteStateFile("session", SessionId); + WriteStateFile("fuse_state", ""); + } else { + SessionId = ReadStateFile("session"); + FuseState = ReadStateFile("fuse_state"); + + STORAGE_INFO( + "Restore existing session, StatePath=" << StatePath << + ", SessionId=" << SessionId << + ", MaxHandlesPerSessionCount=" << maxHandlesPerSessionCount); + } + + HandleTable = std::make_unique( + handlesPath.GetPath(), + maxHandlesPerSessionCount); + + ui64 maxHandleId = 0; + for (auto it = HandleTable->begin(); it != HandleTable->end(); it++) { + maxHandleId = std::max(maxHandleId, it->HandleId); + + STORAGE_TRACE( + "Resolving, HandleId=" << it->HandleId << + ", NodeId=" << it->NodeId << + ", Flags=" << it->Flags); + auto node = LookupNode(it->NodeId); + if (!node) { + STORAGE_ERROR( + "Handle with missing node, HandleId=" << it->HandleId << + ", NodeId" << it->NodeId); + ReportLocalFsMissingHandleNode(); + HandleTable->DeleteRecord(it.GetIndex()); + continue; + } + + try { + auto handle = node->OpenHandle(it->Flags); + auto [_, inserted] = Handles.emplace( + it->HandleId, + THandle{std::move(handle), it.GetIndex()}); + Y_ABORT_UNLESS( + inserted, + "dup file handle for: %lu", + it->HandleId); + } catch (...) { + STORAGE_ERROR( + "Failed to open Handle, HandleId=" << it->HandleId << + ", NodeId" << it->NodeId << + ", Exception=" << CurrentExceptionMessage()); + HandleTable->DeleteRecord(it.GetIndex()); + continue; + } + } + + NextHandleId = maxHandleId + 1; + } - void InsertHandle(TFileHandle handle) + [[nodiscard]] TResultOrError + InsertHandle(TFileHandle handle, ui64 nodeId, int flags) { TWriteGuard guard(Lock); - const auto fhandle = static_cast(handle); - auto [_, inserted] = Handles.emplace(fhandle, std::move(handle)); - Y_ABORT_UNLESS(inserted, "dup file handle for: %d", fhandle); + const auto handleId = NextHandleId++; + + const auto recordIndex = HandleTable->AllocRecord(); + if (recordIndex == THandleTable::InvalidIndex) { + return ErrorNoSpaceLeft(); + } + + auto* state = HandleTable->RecordData(recordIndex); + state->HandleId = handleId; + state->NodeId = nodeId; + state->Flags = flags; + + if (Handles.find(handleId) != Handles.end()) { + ReportLocalFsDuplicateFileHandle(TStringBuilder() << + "HandleId=" << handleId << + ", HandlesCount=" << Handles.size()); + return ErrorInvalidHandle(handleId); + } + + Handles.emplace(handleId, THandle{std::move(handle), recordIndex}); + HandleTable->CommitRecord(recordIndex); + + return handleId; } - TFileHandle* LookupHandle(ui64 handle) + TFileHandle* LookupHandle(ui64 handleId) { TReadGuard guard(Lock); - auto it = Handles.find(handle); + auto it = Handles.find(handleId); if (it == Handles.end()) { return nullptr; } - return &it->second; + return &it->second.FileHandle; } - void DeleteHandle(ui64 handle) + void DeleteHandle(ui64 handleId) { TWriteGuard guard(Lock); - Handles.erase(handle); + auto it = Handles.find(handleId); + if (it != Handles.end()) { + HandleTable->DeleteRecord(it->second.RecordIndex); + Handles.erase(it); + } } TIndexNodePtr LookupNode(ui64 nodeId) @@ -77,9 +214,12 @@ class TSession return Index->LookupNode(nodeId); } - bool TryInsertNode(TIndexNodePtr node) + [[nodiscard]] bool TryInsertNode( + TIndexNodePtr node, + ui64 parentNodeId, + const TString& name) { - return Index->TryInsertNode(std::move(node)); + return Index->TryInsertNode(std::move(node), parentNodeId, name); } void ForgetNode(ui64 nodeId) @@ -100,7 +240,7 @@ class TSession void GetInfo(NProto::TSessionInfo& info, ui64 seqNo) const { info.SetSessionId(SessionId); - info.SetSessionState(State); + info.SetSessionState(FuseState); info.SetSessionSeqNo(seqNo); auto it = SubSessions.find(seqNo); if (it != SubSessions.end()) { @@ -110,7 +250,16 @@ class TSession void ResetState(TString state) { - State = std::move(state); + TWriteGuard guard(Lock); + + for (auto& [_, handle]: Handles) { + HandleTable->DeleteRecord(handle.RecordIndex); + } + + Handles.clear(); + + FuseState = std::move(state); + WriteStateFile("fuse_state", FuseState); } void AddSubSession(ui64 seqNo, bool readOnly) @@ -140,6 +289,33 @@ class TSession }); return SubSessions.empty(); } + +private: + TString ReadStateFile(const TString &fileName) + { + TFile file( + StatePath / fileName, + EOpenModeFlag::OpenExisting | EOpenModeFlag::RdOnly); + return TFileInput(file).ReadAll(); + } + + void WriteStateFile(const TString &fileName, const TString& value) + { + TFsPath tmpFilePath( + MakeTempName(StatePath.GetPath().c_str(), fileName.c_str())); + TFileOutput(tmpFilePath).Write(value); + tmpFilePath.ForceRenameTo(StatePath / fileName); + } + + bool HasStateFile(const TString &fileName) + { + return (StatePath / fileName).Exists(); + } + + void DeleteStateFile(const TString &fileName) + { + return (StatePath / fileName).DeleteIfExists(); + } }; } // namespace NCloud::NFileStore diff --git a/cloud/filestore/libs/service_local/ut/ya.make b/cloud/filestore/libs/service_local/ut/ya.make index ed9198c5e2d..4be778ef30c 100644 --- a/cloud/filestore/libs/service_local/ut/ya.make +++ b/cloud/filestore/libs/service_local/ut/ya.make @@ -7,6 +7,7 @@ PEERDIR( ) SRCS( + index_ut.cpp service_ut.cpp ) diff --git a/cloud/filestore/libs/service_local/ya.make b/cloud/filestore/libs/service_local/ya.make index 70f7b8a36fe..1fe9517f236 100644 --- a/cloud/filestore/libs/service_local/ya.make +++ b/cloud/filestore/libs/service_local/ya.make @@ -23,6 +23,7 @@ ENDIF() PEERDIR( cloud/filestore/config + cloud/filestore/libs/diagnostics cloud/filestore/libs/service cloud/storage/core/libs/common diff --git a/cloud/filestore/tests/fio/qemu-local-noserver-test/ya.make b/cloud/filestore/tests/fio/qemu-local-noserver-test/ya.make index 51a5c5212e1..0373e506281 100644 --- a/cloud/filestore/tests/fio/qemu-local-noserver-test/ya.make +++ b/cloud/filestore/tests/fio/qemu-local-noserver-test/ya.make @@ -16,6 +16,7 @@ TEST_SRCS( ) SET(QEMU_VIRTIO fs) +SET(VHOST_RESTART_INTERVAL 10) INCLUDE(${ARCADIA_ROOT}/cloud/filestore/tests/recipes/vhost-local-noserver.inc) INCLUDE(${ARCADIA_ROOT}/cloud/filestore/tests/recipes/vhost-endpoint.inc)