Skip to content

Commit

Permalink
issue-1350: replaying CreateNode requests from OpLog upon tablet rest…
Browse files Browse the repository at this point in the history 
…art, retrying CreateNode requests happening upon CreateHandle requests, ut (#1597)

* issue-1350: replaying CreateNode requests from OpLog upon tablet restart + ut

* issue-1350: retrying CreateNode requests in followers upon CreateHandle requests in leader, preparation for replaying those CreateNode requests upon tablet restart

* issue-1350: replaying CreateNode requests originating from CreateHandle requests upon tablet restart + ut

* issue-1350: deleted TODOs that were implemented
  • Loading branch information
@qkrorlqr
qkrorlqr authored Jul 14, 2024
1 parent c9d9ba1 commit a33fbbf
Show file tree
Hide file tree
Showing 7 changed files with 531 additions and 66 deletions.
381 changes: 377 additions & 4 deletions cloud/filestore/libs/storage/service/service_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4520,7 +4520,7 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
nodeId1,
listNodesResponse.GetNodes(0).GetId());

// checking DupCache logic - just in case
// checking DupCache logic
service.SendCreateNodeRequest(
headers,
TCreateNodeArgs::File(RootNodeId, "file1"),
Expand All @@ -4536,9 +4536,382 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
createResponse->Record.GetNode().GetId());
}

// TODO(#1350): ShouldRetryNodeCreationInFollowerUponLeaderRestart
// TODO(#1350): ShouldRetryNodeCreationInFollowerUponCreateHandle
// TODO(#1350): ShouldRetryNodeCreationInFollowerUponCreateHandleUponLeaderRestart
Y_UNIT_TEST(ShouldRetryNodeCreationInFollowerUponLeaderRestart)
{
NProto::TStorageConfig config;
config.SetMultiTabletForwardingEnabled(true);
TTestEnv env({}, config);
env.CreateSubDomain("nfs");

ui32 nodeIdx = env.CreateNode("nfs");

const TString fsId = "test";
const auto shard1Id = fsId + "-f1";
const auto shard2Id = fsId + "-f2";

TServiceClient service(env.GetRuntime(), nodeIdx);

ui64 tabletId = -1;
env.GetRuntime().SetEventFilter(
[&] (auto& runtime, TAutoPtr<IEventHandle>& event) {
Y_UNUSED(runtime);
switch (event->GetTypeRewrite()) {
case TEvSSProxy::EvDescribeFileStoreResponse: {
using TDesc = TEvSSProxy::TEvDescribeFileStoreResponse;
const auto* msg = event->Get<TDesc>();
const auto& desc =
msg->PathDescription.GetFileStoreDescription();
if (desc.GetConfig().GetFileSystemId() == fsId) {
tabletId = desc.GetIndexTabletId();
}
}
}

return false;
});

service.CreateFileStore(fsId, 1'000);
service.CreateFileStore(shard1Id, 1'000);
service.CreateFileStore(shard2Id, 1'000);

ConfigureFollowers(service, fsId, shard1Id, shard2Id);

auto headers = service.InitSession(fsId, "client");

bool intercept = true;
bool intercepted = false;
env.GetRuntime().SetEventFilter(
[&] (auto& runtime, TAutoPtr<IEventHandle>& event) {
Y_UNUSED(runtime);
if (event->GetTypeRewrite() == TEvService::EvCreateNodeRequest) {
const auto* msg =
event->Get<TEvService::TEvCreateNodeRequest>();
if (intercept
&& msg->Record.GetFileSystemId() == shard1Id)
{
intercepted = true;
return true;
}
}
return false;
});

const ui64 requestId = 111;
service.SendCreateNodeRequest(
headers,
TCreateNodeArgs::File(RootNodeId, "file1"),
requestId);

ui32 iterations = 0;
while (!intercepted && iterations++ < 100) {
env.GetRuntime().DispatchEvents({}, TDuration::MilliSeconds(50));
}

UNIT_ASSERT(intercepted);
intercept = false;

// TODO listNodes in leader?

auto headers1 = headers;
headers1.FileSystemId = shard1Id;

auto listNodesResponse = service.ListNodes(
headers1,
shard1Id,
RootNodeId)->Record;

UNIT_ASSERT_VALUES_EQUAL(0, listNodesResponse.NamesSize());
UNIT_ASSERT_VALUES_EQUAL(0, listNodesResponse.NodesSize());

TIndexTabletClient tablet(env.GetRuntime(), nodeIdx, tabletId);
tablet.RebootTablet();

auto createResponse = service.RecvCreateNodeResponse();
UNIT_ASSERT_VALUES_EQUAL_C(
E_REJECTED,
createResponse->GetError().GetCode(),
createResponse->GetError().GetMessage());

// remaking session since CreateSessionActor doesn't do it by itself
// because EvWakeup never arrives because Scheduling doesn't work by
// default and RegistrationObservers get reset after RebootTablet
// restoreClientSession = true
headers = service.InitSession(fsId, "client", {}, true);

listNodesResponse = service.ListNodes(
headers,
fsId,
RootNodeId)->Record;

const ui64 nodeId1 = (1LU << 56U) + 2;

UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NamesSize());
UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NodesSize());
UNIT_ASSERT_VALUES_EQUAL("file1", listNodesResponse.GetNames(0));
UNIT_ASSERT_VALUES_EQUAL(
nodeId1,
listNodesResponse.GetNodes(0).GetId());

listNodesResponse = service.ListNodes(
headers1,
shard1Id,
RootNodeId)->Record;

UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NamesSize());
UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NodesSize());

// checking DupCache logic
service.SendCreateNodeRequest(
headers,
TCreateNodeArgs::File(RootNodeId, "file1"),
requestId);

createResponse = service.RecvCreateNodeResponse();
UNIT_ASSERT_VALUES_EQUAL_C(
S_OK,
createResponse->GetError().GetCode(),
createResponse->GetError().GetMessage());
UNIT_ASSERT_VALUES_EQUAL(
nodeId1,
createResponse->Record.GetNode().GetId());
}

Y_UNIT_TEST(ShouldRetryNodeCreationInFollowerUponCreateHandle)
{
NProto::TStorageConfig config;
config.SetMultiTabletForwardingEnabled(true);
TTestEnv env({}, config);
env.CreateSubDomain("nfs");

ui32 nodeIdx = env.CreateNode("nfs");

const TString fsId = "test";
const auto shard1Id = fsId + "-f1";
const auto shard2Id = fsId + "-f2";

TServiceClient service(env.GetRuntime(), nodeIdx);
service.CreateFileStore(fsId, 1'000);
service.CreateFileStore(shard1Id, 1'000);
service.CreateFileStore(shard2Id, 1'000);

ConfigureFollowers(service, fsId, shard1Id, shard2Id);

auto headers = service.InitSession(fsId, "client");

TAutoPtr<IEventHandle> followerCreateResponse;
bool intercept = true;
env.GetRuntime().SetEventFilter(
[&] (auto& runtime, TAutoPtr<IEventHandle>& event) {
Y_UNUSED(runtime);
if (event->GetTypeRewrite() == TEvService::EvCreateNodeRequest) {
const auto* msg =
event->Get<TEvService::TEvCreateNodeRequest>();
if (intercept
&& msg->Record.GetFileSystemId() == shard1Id)
{
auto response = std::make_unique<
TEvService::TEvCreateNodeResponse>(
MakeError(E_REJECTED, "error"));

followerCreateResponse = new IEventHandle(
event->Sender,
event->Recipient,
response.release(),
0, // flags
event->Cookie);

return true;
}
}
return false;
});

const ui64 requestId = 111;
service.SendCreateHandleRequest(
headers,
fsId,
RootNodeId,
"file1",
TCreateHandleArgs::CREATE,
"", // followerId
requestId);

ui32 iterations = 0;
while (!followerCreateResponse && iterations++ < 100) {
env.GetRuntime().DispatchEvents({}, TDuration::MilliSeconds(50));
}

UNIT_ASSERT(followerCreateResponse);
intercept = false;
env.GetRuntime().Send(followerCreateResponse.Release());

auto createHandleResponse = service.RecvCreateHandleResponse();
UNIT_ASSERT_VALUES_EQUAL_C(
S_OK,
createHandleResponse->GetError().GetCode(),
createHandleResponse->GetError().GetMessage());

const auto nodeId1 = createHandleResponse->Record.GetNodeAttr().GetId();
UNIT_ASSERT_VALUES_EQUAL((1LU << 56U) + 2, nodeId1);

auto listNodesResponse = service.ListNodes(
headers,
fsId,
RootNodeId)->Record;

UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NamesSize());
UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NodesSize());
UNIT_ASSERT_VALUES_EQUAL("file1", listNodesResponse.GetNames(0));
UNIT_ASSERT_VALUES_EQUAL(
nodeId1,
listNodesResponse.GetNodes(0).GetId());

// checking DupCache logic
service.SendCreateHandleRequest(
headers,
fsId,
RootNodeId,
"file1",
TCreateHandleArgs::CREATE,
"", // followerId
requestId);

createHandleResponse = service.RecvCreateHandleResponse();
UNIT_ASSERT_VALUES_EQUAL_C(
S_OK,
createHandleResponse->GetError().GetCode(),
createHandleResponse->GetError().GetMessage());
UNIT_ASSERT_VALUES_EQUAL(
nodeId1,
createHandleResponse->Record.GetNodeAttr().GetId());
}

Y_UNIT_TEST(ShouldRetryNodeCreationInFollowerUponCreateHandleUponLeaderRestart)
{
NProto::TStorageConfig config;
config.SetMultiTabletForwardingEnabled(true);
TTestEnv env({}, config);
env.CreateSubDomain("nfs");

ui32 nodeIdx = env.CreateNode("nfs");

const TString fsId = "test";
const auto shard1Id = fsId + "-f1";
const auto shard2Id = fsId + "-f2";

TServiceClient service(env.GetRuntime(), nodeIdx);

ui64 tabletId = -1;
env.GetRuntime().SetEventFilter(
[&] (auto& runtime, TAutoPtr<IEventHandle>& event) {
Y_UNUSED(runtime);
switch (event->GetTypeRewrite()) {
case TEvSSProxy::EvDescribeFileStoreResponse: {
using TDesc = TEvSSProxy::TEvDescribeFileStoreResponse;
const auto* msg = event->Get<TDesc>();
const auto& desc =
msg->PathDescription.GetFileStoreDescription();
if (desc.GetConfig().GetFileSystemId() == fsId) {
tabletId = desc.GetIndexTabletId();
}
}
}

return false;
});

service.CreateFileStore(fsId, 1'000);
service.CreateFileStore(shard1Id, 1'000);
service.CreateFileStore(shard2Id, 1'000);

ConfigureFollowers(service, fsId, shard1Id, shard2Id);

auto headers = service.InitSession(fsId, "client");

bool intercept = true;
bool intercepted = false;
env.GetRuntime().SetEventFilter(
[&] (auto& runtime, TAutoPtr<IEventHandle>& event) {
Y_UNUSED(runtime);
if (event->GetTypeRewrite() == TEvService::EvCreateNodeRequest) {
const auto* msg =
event->Get<TEvService::TEvCreateNodeRequest>();
if (intercept
&& msg->Record.GetFileSystemId() == shard1Id)
{
intercepted = true;
return true;
}
}
return false;
});

const ui64 requestId = 111;
service.SendCreateHandleRequest(
headers,
fsId,
RootNodeId,
"file1",
TCreateHandleArgs::CREATE,
"", // followerId
requestId);

ui32 iterations = 0;
while (!intercepted && iterations++ < 100) {
env.GetRuntime().DispatchEvents({}, TDuration::MilliSeconds(50));
}

UNIT_ASSERT(intercepted);
intercept = false;

TIndexTabletClient tablet(env.GetRuntime(), nodeIdx, tabletId);
tablet.RebootTablet();

auto createHandleResponse = service.RecvCreateHandleResponse();
UNIT_ASSERT_VALUES_EQUAL_C(
E_REJECTED,
createHandleResponse->GetError().GetCode(),
createHandleResponse->GetError().GetMessage());

const auto nodeId1 = (1LU << 56U) + 2;

// remaking session since CreateSessionActor doesn't do it by itself
// because EvWakeup never arrives because Scheduling doesn't work by
// default and RegistrationObservers get reset after RebootTablet
// restoreClientSession = true
headers = service.InitSession(fsId, "client", {}, true);

auto listNodesResponse = service.ListNodes(
headers,
fsId,
RootNodeId)->Record;

UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NamesSize());
UNIT_ASSERT_VALUES_EQUAL(1, listNodesResponse.NodesSize());
UNIT_ASSERT_VALUES_EQUAL("file1", listNodesResponse.GetNames(0));
UNIT_ASSERT_VALUES_EQUAL(
nodeId1,
listNodesResponse.GetNodes(0).GetId());

// checking DupCache logic
service.SendCreateHandleRequest(
headers,
fsId,
RootNodeId,
"file1",
TCreateHandleArgs::CREATE,
"", // followerId
requestId);

createHandleResponse = service.RecvCreateHandleResponse();
UNIT_ASSERT_VALUES_EQUAL_C(
S_OK,
createHandleResponse->GetError().GetCode(),
createHandleResponse->GetError().GetMessage());
UNIT_ASSERT_VALUES_EQUAL(
nodeId1,
createHandleResponse->Record.GetNodeAttr().GetId());
}
}

} // namespace NCloud::NFileStore::NStorage
Loading

0 comments on commit a33fbbf

Please sign in to comment.