diff --git a/cloud/filestore/libs/storage/service/service_ut.cpp b/cloud/filestore/libs/storage/service/service_ut.cpp index 6d083d70a94..f6a4568b9fd 100644 --- a/cloud/filestore/libs/storage/service/service_ut.cpp +++ b/cloud/filestore/libs/storage/service/service_ut.cpp @@ -4520,7 +4520,7 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest) nodeId1, listNodesResponse.GetNodes(0).GetId()); - // checking DupCache logic - just in case + // checking DupCache logic service.SendCreateNodeRequest( headers, TCreateNodeArgs::File(RootNodeId, "file1"), @@ -4536,9 +4536,382 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest) createResponse->Record.GetNode().GetId()); } - // TODO(#1350): ShouldRetryNodeCreationInFollowerUponLeaderRestart - // TODO(#1350): ShouldRetryNodeCreationInFollowerUponCreateHandle - // TODO(#1350): ShouldRetryNodeCreationInFollowerUponCreateHandleUponLeaderRestart + Y_UNIT_TEST(ShouldRetryNodeCreationInFollowerUponLeaderRestart) + { + NProto::TStorageConfig config; + config.SetMultiTabletForwardingEnabled(true); + TTestEnv env({}, config); + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + + const TString fsId = "test"; + const auto shard1Id = fsId + "-f1"; + const auto shard2Id = fsId + "-f2"; + + TServiceClient service(env.GetRuntime(), nodeIdx); + + ui64 tabletId = -1; + env.GetRuntime().SetEventFilter( + [&] (auto& runtime, TAutoPtr& event) { + Y_UNUSED(runtime); + switch (event->GetTypeRewrite()) { + case TEvSSProxy::EvDescribeFileStoreResponse: { + using TDesc = TEvSSProxy::TEvDescribeFileStoreResponse; + const auto* msg = event->Get(); + const auto& desc = + msg->PathDescription.GetFileStoreDescription(); + if (desc.GetConfig().GetFileSystemId() == fsId) { + tabletId = desc.GetIndexTabletId(); + } + } + } + + return false; + }); + + service.CreateFileStore(fsId, 1'000); + service.CreateFileStore(shard1Id, 1'000); + service.CreateFileStore(shard2Id, 1'000); + + ConfigureFollowers(service, fsId, shard1Id, shard2Id); + + auto headers = service.InitSession(fsId, "client"); + + bool intercept = true; + bool intercepted = false; + env.GetRuntime().SetEventFilter( + [&] (auto& runtime, TAutoPtr& event) { + Y_UNUSED(runtime); + if (event->GetTypeRewrite() == TEvService::EvCreateNodeRequest) { + const auto* msg = + event->Get(); + if (intercept + && msg->Record.GetFileSystemId() == shard1Id) + { + intercepted = true; + return true; + } + } + return false; + }); + + const ui64 requestId = 111; + service.SendCreateNodeRequest( + headers, + TCreateNodeArgs::File(RootNodeId, "file1"), + requestId); + + ui32 iterations = 0; + while (!intercepted && iterations++ < 100) { + env.GetRuntime().DispatchEvents({}, TDuration::MilliSeconds(50)); + } + + UNIT_ASSERT(intercepted); + intercept = false; + + // TODO listNodes in leader? + + auto headers1 = headers; + headers1.FileSystemId = shard1Id; + + auto listNodesResponse = service.ListNodes( + headers1, + shard1Id, + RootNodeId)->Record; + + UNIT_ASSERT_VALUES_EQUAL(0, listNodesResponse.NamesSize()); + UNIT_ASSERT_VALUES_EQUAL(0, listNodesResponse.NodesSize()); + + TIndexTabletClient tablet(env.GetRuntime(), nodeIdx, tabletId); + tablet.RebootTablet(); + + auto createResponse = service.RecvCreateNodeResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + E_REJECTED, + createResponse->GetError().GetCode(), + createResponse->GetError().GetMessage()); + + // remaking session since CreateSessionActor doesn't do it by itself + // because EvWakeup never arrives because Scheduling doesn't work by + // default and RegistrationObservers get reset after RebootTablet + // restoreClientSession = true + headers = service.InitSession(fsId, "client", {}, true); + + listNodesResponse = service.ListNodes( + headers, + fsId, + RootNodeId)->Record; + + const ui64 nodeId1 = (1LU << 56U) + 2; + + UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NamesSize()); + UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NodesSize()); + UNIT_ASSERT_VALUES_EQUAL("file1", listNodesResponse.GetNames(0)); + UNIT_ASSERT_VALUES_EQUAL( + nodeId1, + listNodesResponse.GetNodes(0).GetId()); + + listNodesResponse = service.ListNodes( + headers1, + shard1Id, + RootNodeId)->Record; + + UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NamesSize()); + UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NodesSize()); + + // checking DupCache logic + service.SendCreateNodeRequest( + headers, + TCreateNodeArgs::File(RootNodeId, "file1"), + requestId); + + createResponse = service.RecvCreateNodeResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + S_OK, + createResponse->GetError().GetCode(), + createResponse->GetError().GetMessage()); + UNIT_ASSERT_VALUES_EQUAL( + nodeId1, + createResponse->Record.GetNode().GetId()); + } + + Y_UNIT_TEST(ShouldRetryNodeCreationInFollowerUponCreateHandle) + { + NProto::TStorageConfig config; + config.SetMultiTabletForwardingEnabled(true); + TTestEnv env({}, config); + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + + const TString fsId = "test"; + const auto shard1Id = fsId + "-f1"; + const auto shard2Id = fsId + "-f2"; + + TServiceClient service(env.GetRuntime(), nodeIdx); + service.CreateFileStore(fsId, 1'000); + service.CreateFileStore(shard1Id, 1'000); + service.CreateFileStore(shard2Id, 1'000); + + ConfigureFollowers(service, fsId, shard1Id, shard2Id); + + auto headers = service.InitSession(fsId, "client"); + + TAutoPtr followerCreateResponse; + bool intercept = true; + env.GetRuntime().SetEventFilter( + [&] (auto& runtime, TAutoPtr& event) { + Y_UNUSED(runtime); + if (event->GetTypeRewrite() == TEvService::EvCreateNodeRequest) { + const auto* msg = + event->Get(); + if (intercept + && msg->Record.GetFileSystemId() == shard1Id) + { + auto response = std::make_unique< + TEvService::TEvCreateNodeResponse>( + MakeError(E_REJECTED, "error")); + + followerCreateResponse = new IEventHandle( + event->Sender, + event->Recipient, + response.release(), + 0, // flags + event->Cookie); + + return true; + } + } + return false; + }); + + const ui64 requestId = 111; + service.SendCreateHandleRequest( + headers, + fsId, + RootNodeId, + "file1", + TCreateHandleArgs::CREATE, + "", // followerId + requestId); + + ui32 iterations = 0; + while (!followerCreateResponse && iterations++ < 100) { + env.GetRuntime().DispatchEvents({}, TDuration::MilliSeconds(50)); + } + + UNIT_ASSERT(followerCreateResponse); + intercept = false; + env.GetRuntime().Send(followerCreateResponse.Release()); + + auto createHandleResponse = service.RecvCreateHandleResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + S_OK, + createHandleResponse->GetError().GetCode(), + createHandleResponse->GetError().GetMessage()); + + const auto nodeId1 = createHandleResponse->Record.GetNodeAttr().GetId(); + UNIT_ASSERT_VALUES_EQUAL((1LU << 56U) + 2, nodeId1); + + auto listNodesResponse = service.ListNodes( + headers, + fsId, + RootNodeId)->Record; + + UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NamesSize()); + UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NodesSize()); + UNIT_ASSERT_VALUES_EQUAL("file1", listNodesResponse.GetNames(0)); + UNIT_ASSERT_VALUES_EQUAL( + nodeId1, + listNodesResponse.GetNodes(0).GetId()); + + // checking DupCache logic + service.SendCreateHandleRequest( + headers, + fsId, + RootNodeId, + "file1", + TCreateHandleArgs::CREATE, + "", // followerId + requestId); + + createHandleResponse = service.RecvCreateHandleResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + S_OK, + createHandleResponse->GetError().GetCode(), + createHandleResponse->GetError().GetMessage()); + UNIT_ASSERT_VALUES_EQUAL( + nodeId1, + createHandleResponse->Record.GetNodeAttr().GetId()); + } + + Y_UNIT_TEST(ShouldRetryNodeCreationInFollowerUponCreateHandleUponLeaderRestart) + { + NProto::TStorageConfig config; + config.SetMultiTabletForwardingEnabled(true); + TTestEnv env({}, config); + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + + const TString fsId = "test"; + const auto shard1Id = fsId + "-f1"; + const auto shard2Id = fsId + "-f2"; + + TServiceClient service(env.GetRuntime(), nodeIdx); + + ui64 tabletId = -1; + env.GetRuntime().SetEventFilter( + [&] (auto& runtime, TAutoPtr& event) { + Y_UNUSED(runtime); + switch (event->GetTypeRewrite()) { + case TEvSSProxy::EvDescribeFileStoreResponse: { + using TDesc = TEvSSProxy::TEvDescribeFileStoreResponse; + const auto* msg = event->Get(); + const auto& desc = + msg->PathDescription.GetFileStoreDescription(); + if (desc.GetConfig().GetFileSystemId() == fsId) { + tabletId = desc.GetIndexTabletId(); + } + } + } + + return false; + }); + + service.CreateFileStore(fsId, 1'000); + service.CreateFileStore(shard1Id, 1'000); + service.CreateFileStore(shard2Id, 1'000); + + ConfigureFollowers(service, fsId, shard1Id, shard2Id); + + auto headers = service.InitSession(fsId, "client"); + + bool intercept = true; + bool intercepted = false; + env.GetRuntime().SetEventFilter( + [&] (auto& runtime, TAutoPtr& event) { + Y_UNUSED(runtime); + if (event->GetTypeRewrite() == TEvService::EvCreateNodeRequest) { + const auto* msg = + event->Get(); + if (intercept + && msg->Record.GetFileSystemId() == shard1Id) + { + intercepted = true; + return true; + } + } + return false; + }); + + const ui64 requestId = 111; + service.SendCreateHandleRequest( + headers, + fsId, + RootNodeId, + "file1", + TCreateHandleArgs::CREATE, + "", // followerId + requestId); + + ui32 iterations = 0; + while (!intercepted && iterations++ < 100) { + env.GetRuntime().DispatchEvents({}, TDuration::MilliSeconds(50)); + } + + UNIT_ASSERT(intercepted); + intercept = false; + + TIndexTabletClient tablet(env.GetRuntime(), nodeIdx, tabletId); + tablet.RebootTablet(); + + auto createHandleResponse = service.RecvCreateHandleResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + E_REJECTED, + createHandleResponse->GetError().GetCode(), + createHandleResponse->GetError().GetMessage()); + + const auto nodeId1 = (1LU << 56U) + 2; + + // remaking session since CreateSessionActor doesn't do it by itself + // because EvWakeup never arrives because Scheduling doesn't work by + // default and RegistrationObservers get reset after RebootTablet + // restoreClientSession = true + headers = service.InitSession(fsId, "client", {}, true); + + auto listNodesResponse = service.ListNodes( + headers, + fsId, + RootNodeId)->Record; + + UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NamesSize()); + UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NodesSize()); + UNIT_ASSERT_VALUES_EQUAL("file1", listNodesResponse.GetNames(0)); + UNIT_ASSERT_VALUES_EQUAL( + nodeId1, + listNodesResponse.GetNodes(0).GetId()); + + // checking DupCache logic + service.SendCreateHandleRequest( + headers, + fsId, + RootNodeId, + "file1", + TCreateHandleArgs::CREATE, + "", // followerId + requestId); + + createHandleResponse = service.RecvCreateHandleResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + S_OK, + createHandleResponse->GetError().GetCode(), + createHandleResponse->GetError().GetMessage()); + UNIT_ASSERT_VALUES_EQUAL( + nodeId1, + createHandleResponse->Record.GetNodeAttr().GetId()); + } } } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.h b/cloud/filestore/libs/storage/tablet/tablet_actor.h index ccfc5860d93..e2d35435a72 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.h +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.h @@ -363,6 +363,14 @@ class TIndexTabletActor final bool CheckSessionForDestroy(const TSession* session, ui64 seqNo); + void RegisterCreateNodeInFollowerActor( + const NActors::TActorContext& ctx, + TRequestInfoPtr requestInfo, + NProto::TCreateNodeRequest request, + ui64 requestId, + ui64 opLogEntryId, + NProto::TCreateNodeResponse response); + void RegisterUnlinkNodeInFollowerActor( const NActors::TActorContext& ctx, TRequestInfoPtr requestInfo, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_createhandle.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_createhandle.cpp index e00283f2d62..eebd6261628 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_createhandle.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_createhandle.cpp @@ -43,28 +43,27 @@ NProto::TError ValidateRequest(const NProto::TCreateHandleRequest& request) //////////////////////////////////////////////////////////////////////////////// +// TODO(#1350): extract common code, see TCreateNodeInFollowerActor class TCreateNodeInFollowerUponCreateHandleActor final : public TActorBootstrapped { private: const TString LogTag; TRequestInfoPtr RequestInfo; - const TString FollowerId; - const TString FollowerName; const TActorId ParentId; - const NProto::TNode Attrs; - NProto::THeaders Headers; + const NProto::TCreateNodeRequest Request; + const ui64 RequestId; + const ui64 OpLogEntryId; NProto::TCreateHandleResponse Response; public: TCreateNodeInFollowerUponCreateHandleActor( TString logTag, TRequestInfoPtr requestInfo, - TString followerId, - TString followerName, const TActorId& parentId, - NProto::TNode attrs, - NProto::THeaders headers, + NProto::TCreateNodeRequest request, + ui64 requestId, + ui64 opLogEntryId, NProto::TCreateHandleResponse response); void Bootstrap(const TActorContext& ctx); @@ -90,19 +89,17 @@ class TCreateNodeInFollowerUponCreateHandleActor final TCreateNodeInFollowerUponCreateHandleActor::TCreateNodeInFollowerUponCreateHandleActor( TString logTag, TRequestInfoPtr requestInfo, - TString followerId, - TString followerName, const TActorId& parentId, - NProto::TNode attrs, - NProto::THeaders headers, + NProto::TCreateNodeRequest request, + ui64 requestId, + ui64 opLogEntryId, NProto::TCreateHandleResponse response) : LogTag(std::move(logTag)) , RequestInfo(std::move(requestInfo)) - , FollowerId(std::move(followerId)) - , FollowerName(std::move(followerName)) , ParentId(parentId) - , Attrs(std::move(attrs)) - , Headers(std::move(headers)) + , Request(std::move(request)) + , RequestId(requestId) + , OpLogEntryId(opLogEntryId) , Response(std::move(response)) {} @@ -115,22 +112,15 @@ void TCreateNodeInFollowerUponCreateHandleActor::Bootstrap(const TActorContext& void TCreateNodeInFollowerUponCreateHandleActor::SendRequest(const TActorContext& ctx) { auto request = std::make_unique(); - *request->Record.MutableHeaders() = std::move(Headers); - request->Record.MutableFile()->SetMode(Attrs.GetMode()); - request->Record.SetUid(Attrs.GetUid()); - request->Record.SetGid(Attrs.GetGid()); - request->Record.SetFileSystemId(FollowerId); - request->Record.SetNodeId(RootNodeId); - request->Record.SetName(FollowerName); - request->Record.ClearFollowerFileSystemId(); + request->Record = Request; LOG_INFO( ctx, TFileStoreComponents::TABLET_WORKER, "%s Sending CreateNodeRequest to follower %s, %s", LogTag.c_str(), - FollowerId.c_str(), - FollowerName.c_str()); + Request.GetFileSystemId().c_str(), + Request.GetName().c_str()); ctx.Send( MakeIndexTabletProxyServiceId(), @@ -143,27 +133,57 @@ void TCreateNodeInFollowerUponCreateHandleActor::HandleCreateNodeResponse( { auto* msg = ev->Get(); + if (msg->GetError().GetCode() == E_FS_EXIST) { + // EXIST can arrive after a successful operation is retried, it's ok + LOG_DEBUG( + ctx, + TFileStoreComponents::TABLET_WORKER, + "%s Follower node creation for %s, %s returned EEXIST %s", + LogTag.c_str(), + Request.GetFileSystemId().c_str(), + Request.GetName().c_str(), + FormatError(msg->GetError()).Quote().c_str()); + + msg->Record.ClearError(); + } + if (HasError(msg->GetError())) { + if (GetErrorKind(msg->GetError()) == EErrorKind::ErrorRetriable) { + LOG_WARN( + ctx, + TFileStoreComponents::TABLET_WORKER, + "%s Follower node creation failed for %s, %s with error %s" + ", retrying", + LogTag.c_str(), + Request.GetFileSystemId().c_str(), + Request.GetName().c_str(), + FormatError(msg->GetError()).Quote().c_str()); + + SendRequest(ctx); + return; + } + LOG_ERROR( ctx, TFileStoreComponents::TABLET_WORKER, - "%s Follower node creation failed for %s, %s with error %s", + "%s Follower node creation failed for %s, %s with error %s" + ", will not retry", LogTag.c_str(), - FollowerId.c_str(), - FollowerName.c_str(), + Request.GetFileSystemId().c_str(), + Request.GetName().c_str(), FormatError(msg->GetError()).Quote().c_str()); ReplyAndDie(ctx, msg->GetError()); return; } - LOG_INFO( + LOG_DEBUG( ctx, TFileStoreComponents::TABLET_WORKER, "%s Follower node created for %s, %s", LogTag.c_str(), - FollowerId.c_str(), - FollowerName.c_str()); + Request.GetFileSystemId().c_str(), + Request.GetName().c_str()); *Response.MutableNodeAttr() = std::move(*msg->Record.MutableNode()); @@ -183,18 +203,16 @@ void TCreateNodeInFollowerUponCreateHandleActor::ReplyAndDie( NProto::TError error) { if (HasError(error)) { - // TODO(#1350): properly retry node creation via the leader fs *Response.MutableError() = std::move(error); } - // TODO(#1350): reply directly to the client (not to the tablet) upon - // poisoning - // or keep this RequestInfo in the ActiveTransactions list until this event - // gets processed by the tablet using TResponse = TEvIndexTabletPrivate::TEvNodeCreatedInFollowerUponCreateHandle; ctx.Send(ParentId, std::make_unique( std::move(RequestInfo), + Request.GetHeaders().GetSessionId(), + RequestId, + OpLogEntryId, std::move(Response))); Die(ctx); @@ -509,6 +527,23 @@ void TIndexTabletActor::ExecuteTx_CreateHandle( args.Response.SetFollowerNodeName(args.FollowerName); } + if (args.IsNewFollowerNode) { + // OpLogEntryId doesn't have to be a CommitId - it's just convenient to + // use CommitId here in order not to generate some other unique ui64 + args.OpLogEntry.SetEntryId(args.WriteCommitId); + auto* followerRequest = args.OpLogEntry.MutableCreateNodeRequest(); + *followerRequest->MutableHeaders() = args.Request.GetHeaders(); + followerRequest->MutableFile()->SetMode(args.Mode); + followerRequest->SetUid(args.Uid); + followerRequest->SetGid(args.Gid); + followerRequest->SetFileSystemId(args.FollowerId); + followerRequest->SetNodeId(RootNodeId); + followerRequest->SetName(args.FollowerName); + followerRequest->ClearFollowerFileSystemId(); + + db.WriteOpLogEntry(args.OpLogEntry); + } + AddDupCacheEntry( db, session, @@ -523,8 +558,6 @@ void TIndexTabletActor::CompleteTx_CreateHandle( const TActorContext& ctx, TTxIndexTablet::TCreateHandle& args) { - RemoveTransaction(*args.RequestInfo); - if (args.Error.GetCode() == E_ARGUMENT) { // service actor sent something inappropriate, we'd better log it LOG_ERROR( @@ -535,11 +568,7 @@ void TIndexTabletActor::CompleteTx_CreateHandle( FormatError(args.Error).Quote().c_str()); } - if (!HasError(args.Error)) { - CommitDupCacheEntry(args.SessionId, args.RequestId); - } - - if (args.IsNewFollowerNode && !HasError(args.Error)) { + if (args.OpLogEntry.HasCreateNodeRequest() && !HasError(args.Error)) { LOG_INFO(ctx, TFileStoreComponents::TABLET, "%s Creating node in follower upon CreateHandle: %s, %s", LogTag.c_str(), @@ -549,11 +578,10 @@ void TIndexTabletActor::CompleteTx_CreateHandle( auto actor = std::make_unique( LogTag, args.RequestInfo, - args.FollowerId, - args.FollowerName, ctx.SelfID, - CreateRegularAttrs(args.Mode, args.Uid, args.Gid), - std::move(*args.Request.MutableHeaders()), + std::move(*args.OpLogEntry.MutableCreateNodeRequest()), + args.RequestId, + args.OpLogEntry.GetEntryId(), std::move(args.Response)); auto actorId = NCloud::Register(ctx, std::move(actor)); @@ -562,10 +590,13 @@ void TIndexTabletActor::CompleteTx_CreateHandle( return; } + RemoveTransaction(*args.RequestInfo); + auto response = std::make_unique(args.Error); if (!HasError(args.Error)) { + CommitDupCacheEntry(args.SessionId, args.RequestId); response->Record = std::move(args.Response); } @@ -588,6 +619,9 @@ void TIndexTabletActor::HandleNodeCreatedInFollowerUponCreateHandle( auto response = std::make_unique(); response->Record = std::move(msg->CreateHandleResponse); + RemoveTransaction(*msg->RequestInfo); + CommitDupCacheEntry(msg->SessionId, msg->RequestId); + CompleteResponse( response->Record, msg->RequestInfo->CallContext, @@ -596,6 +630,7 @@ void TIndexTabletActor::HandleNodeCreatedInFollowerUponCreateHandle( NCloud::Reply(ctx, *msg->RequestInfo, std::move(response)); WorkerActors.erase(ev->Sender); + ExecuteTx(ctx, msg->OpLogEntryId); } } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_createnode.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_createnode.cpp index c1f54bbabd3..cdccb8068d3 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_createnode.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_createnode.cpp @@ -451,7 +451,6 @@ void TIndexTabletActor::ExecuteTx_CreateNode( followerRequest->ClearFollowerFileSystemId(); db.WriteOpLogEntry(args.OpLogEntry); - } } else { // hard link @@ -531,18 +530,14 @@ void TIndexTabletActor::CompleteTx_CreateNode( args.FollowerId.c_str(), args.FollowerName.c_str()); - auto actor = std::make_unique( - LogTag, + RegisterCreateNodeInFollowerActor( + ctx, args.RequestInfo, - ctx.SelfID, std::move(*args.OpLogEntry.MutableCreateNodeRequest()), args.RequestId, args.OpLogEntry.GetEntryId(), std::move(args.Response)); - auto actorId = NCloud::Register(ctx, std::move(actor)); - WorkerActors.insert(actorId); - return; } @@ -601,13 +596,17 @@ void TIndexTabletActor::HandleNodeCreatedInFollower( auto response = std::make_unique(); response->Record = msg->CreateNodeResponse; - CompleteResponse( - response->Record, - msg->RequestInfo->CallContext, - ctx); + if (msg->RequestInfo) { + RemoveTransaction(*msg->RequestInfo); + + CompleteResponse( + response->Record, + msg->RequestInfo->CallContext, + ctx); - // replying before DupCacheEntry is committed to reduce response latency - NCloud::Reply(ctx, *msg->RequestInfo, std::move(response)); + // replying before DupCacheEntry is committed to reduce response latency + NCloud::Reply(ctx, *msg->RequestInfo, std::move(response)); + } WorkerActors.erase(ev->Sender); ExecuteTx( @@ -662,4 +661,27 @@ void TIndexTabletActor::CompleteTx_CommitNodeCreationInFollower( args.RequestId); } +//////////////////////////////////////////////////////////////////////////////// + +void TIndexTabletActor::RegisterCreateNodeInFollowerActor( + const NActors::TActorContext& ctx, + TRequestInfoPtr requestInfo, + NProto::TCreateNodeRequest request, + ui64 requestId, + ui64 opLogEntryId, + NProto::TCreateNodeResponse response) +{ + auto actor = std::make_unique( + LogTag, + std::move(requestInfo), + ctx.SelfID, + std::move(request), + requestId, + opLogEntryId, + std::move(response)); + + auto actorId = NCloud::Register(ctx, std::move(actor)); + WorkerActors.insert(actorId); +} + } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_oplog.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_oplog.cpp index 2e5797d8827..8e91f9e7822 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_oplog.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_oplog.cpp @@ -13,9 +13,23 @@ void TIndexTabletActor::ReplayOpLog( const NActors::TActorContext& ctx, const TVector& opLog) { + // TODO(#1350): requests to followers should be ordered, otherwise weird + // races are possible, e.g. create + unlink ops for the same node can arrive + // in opposite order (original requests won't come in opposite order, but + // if one of those requests gets rejected and is retried, the final order + // may change) for (const auto& op: opLog) { if (op.HasCreateNodeRequest()) { - // TODO + RegisterCreateNodeInFollowerActor( + ctx, + nullptr, // requestInfo + op.GetCreateNodeRequest(), + op.GetRequestId(), // needed for DupCache update (only for + // follower requests originating from + // CreateNode requests) + op.GetEntryId(), + {} // response + ); } else if (op.HasUnlinkNodeRequest()) { RegisterUnlinkNodeInFollowerActor( ctx, diff --git a/cloud/filestore/libs/storage/tablet/tablet_private.h b/cloud/filestore/libs/storage/tablet/tablet_private.h index 476df9c8e4d..55c0f5848a0 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_private.h +++ b/cloud/filestore/libs/storage/tablet/tablet_private.h @@ -535,12 +535,21 @@ struct TEvIndexTabletPrivate struct TNodeCreatedInFollowerUponCreateHandle { TRequestInfoPtr RequestInfo; + const TString SessionId; + const ui64 RequestId; + const ui64 OpLogEntryId; NProto::TCreateHandleResponse CreateHandleResponse; TNodeCreatedInFollowerUponCreateHandle( TRequestInfoPtr requestInfo, + TString sessionId, + ui64 requestId, + ui64 opLogEntryId, NProto::TCreateHandleResponse createHandleResponse) : RequestInfo(std::move(requestInfo)) + , SessionId(std::move(sessionId)) + , RequestId(requestId) + , OpLogEntryId(opLogEntryId) , CreateHandleResponse(std::move(createHandleResponse)) { } diff --git a/cloud/filestore/libs/storage/tablet/tablet_tx.h b/cloud/filestore/libs/storage/tablet/tablet_tx.h index 90120cc0cf0..62c039db831 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_tx.h +++ b/cloud/filestore/libs/storage/tablet/tablet_tx.h @@ -1092,6 +1092,8 @@ struct TTxIndexTablet TMaybe TargetNode; TMaybe ParentNode; + NProto::TOpLogEntry OpLogEntry; + NProto::TCreateHandleResponse Response; TCreateHandle( @@ -1121,6 +1123,8 @@ struct TTxIndexTablet TargetNode.Clear(); ParentNode.Clear(); + OpLogEntry.Clear(); + Response.Clear(); } };