Skip to content

Commit

Permalink
NBS-179: use backward compatible arguments to run vhost-server for lo…
Browse files Browse the repository at this point in the history 
…cal disks (#199)

We are extending the command-line arguments of blockstore-vhost-server to
support the RDMA backend but to ensure compatibility during version upgrades we
will use old local disk arguments for aio backend.

For aio backend vhost-server will be started with:

```
blockstore-vhost-server --serial nvme-disk-0 --socket-path /tmp/1.sock -q 2 --device /dev/nvme3n1:394062200832:5518414315520
```

For rdma backend vhost-server will be started with  additional arguments:

```
blockstore-vhost-server ... --client-id aaa --disk-id bbb --device-backend rdma --block-size 4096
```
  • Loading branch information
@budevg
budevg committed Jan 24, 2024
1 parent 73b30ae commit 328cd76
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 15 deletions.
18 changes: 12 additions & 6 deletions cloud/blockstore/libs/endpoints_vhost/external_vhost_server.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -837,18 +837,24 @@ class TExternalVhostEndpointListener final
: request.GetInstanceId();

TVector<TString> args {
"--client-id", clientId,
"--disk-id", request.GetDiskId(),
"--serial", deviceName,
"--socket-path", socketPath,
"-q", ToString(request.GetVhostQueuesCount())
};

args.emplace_back("--device-backend");
args.emplace_back(GetDeviceBackend(epType));
if (epType == EEndpointType::Rdma) {
args.emplace_back("--client-id");
args.emplace_back(clientId);

args.emplace_back("--block-size");
args.emplace_back(ToString(volume.GetBlockSize()));
args.emplace_back("--disk-id");
args.emplace_back(request.GetDiskId());

args.emplace_back("--device-backend");
args.emplace_back(GetDeviceBackend(epType));

args.emplace_back("--block-size");
args.emplace_back(ToString(volume.GetBlockSize()));
}

for (const auto& device: volume.GetDevices()) {
const ui64 size = device.GetBlockCount() * volume.GetBlockSize();
Expand Down
134 changes: 125 additions & 9 deletions cloud/blockstore/libs/endpoints_vhost/external_vhost_server_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,42 @@ struct TFixture
return volume;
} ();

const NProto::TVolume FastPathVolume = [&] {
NProto::TVolume volume;

volume.SetDiskId("vol0");
volume.SetBlocksCount(10'000);
volume.SetBlockSize(4_KB);
volume.SetStorageMediaKind(NProto::STORAGE_MEDIA_SSD_NONREPLICATED);
volume.SetIsFastPathEnabled(true);

{
auto* device = volume.AddDevices();
device->SetDeviceName("/dev/disk/by-path/pci-0000:00:16.0-sas-phy2-lun-0");
device->SetDeviceUUID("uuid1");
device->SetAgentId("host1");
device->SetBlockCount(4'000);
device->SetPhysicalOffset(32'000);
auto *rdma = device->MutableRdmaEndpoint();
rdma->SetHost("host1");
rdma->SetPort(1111);
}

{
auto* device = volume.AddDevices();
device->SetDeviceName("/dev/disk/by-path/pci-0000:00:16.0-sas-phy2-lun-0");
device->SetDeviceUUID("uuid2");
device->SetAgentId("host2");
device->SetBlockCount(6'000);
device->SetPhysicalOffset(0);
auto *rdma = device->MutableRdmaEndpoint();
rdma->SetHost("host2");
rdma->SetPort(2222);
}

return volume;
} ();

THistory History;

IEndpointListenerPtr Listener = CreateExternalVhostEndpointListener(
Expand Down Expand Up @@ -337,7 +373,7 @@ TVector<TString> GetArgN(const TVector<TString>& args, TStringBuf name)

Y_UNIT_TEST_SUITE(TExternalEndpointTest)
{
Y_UNIT_TEST_F(ShouldStartExternalEndpoint, TFixture)
Y_UNIT_TEST_F(ShouldStartAioExternalEndpoint, TFixture)
{
UNIT_ASSERT_VALUES_EQUAL(0, History.size());

Expand All @@ -356,11 +392,86 @@ Y_UNIT_TEST_SUITE(TExternalEndpointTest)
UNIT_ASSERT_VALUES_EQUAL(Volume.GetDiskId(), create->DiskId);

/*
--client-id ... 2
--disk-id ... 2
--serial local0 2
--socket-path /tmp/socket.vhost 2
-q 2 2
--device ... 2
--device ... 2
--read-only 1
11
*/

UNIT_ASSERT_VALUES_EQUAL(11, create->CmdArgs.size());
UNIT_ASSERT_VALUES_EQUAL("local0", GetArg(create->CmdArgs, "--serial"));

UNIT_ASSERT_VALUES_EQUAL(
"/tmp/socket.vhost",
GetArg(create->CmdArgs, "--socket-path"));

UNIT_ASSERT_VALUES_EQUAL("2", GetArg(create->CmdArgs, "-q"));
UNIT_ASSERT(FindPtr(create->CmdArgs, "--read-only"));

auto devices = GetArgN(create->CmdArgs, "--device");

UNIT_ASSERT_VALUES_EQUAL(2, devices.size());

UNIT_ASSERT_VALUES_EQUAL(TStringBuilder()
<< "/dev/disk/by-path/pci-0000:00:16.0-sas-phy2-lun-0:"
<< (4'000 * 4_KB)
<< ":32000",
devices[0]);

UNIT_ASSERT_VALUES_EQUAL(TStringBuilder()
<< "/dev/disk/by-path/pci-0000:00:16.0-sas-phy2-lun-0:"
<< (6'000 * 4_KB)
<< ":0",
devices[1]);

UNIT_ASSERT_VALUES_EQUAL(request.GetClientId(), create->ClientId);
UNIT_ASSERT_VALUES_EQUAL(2, create->Cgroups.size());
UNIT_ASSERT_VALUES_EQUAL("cg-1", create->Cgroups[0]);
UNIT_ASSERT_VALUES_EQUAL("cg-2", create->Cgroups[1]);

auto* start = std::get_if<TStartExternalEndpoint>(&History[1]);
UNIT_ASSERT_C(start, "actual entry: " << History[1].index());

History.clear();
}

{
auto error = Listener->StopEndpoint(SocketPath).GetValueSync();
UNIT_ASSERT_C(!HasError(error), error);

UNIT_ASSERT_VALUES_EQUAL(1, History.size());
auto* stop = std::get_if<TStopExternalEndpoint>(&History[0]);
UNIT_ASSERT_C(stop, "actual entry: " << History[0].index());
}
}

Y_UNIT_TEST_F(ShouldStartRdmaExternalEndpoint, TFixture)
{
UNIT_ASSERT_VALUES_EQUAL(0, History.size());

{
auto request = CreateDefaultStartEndpointRequest();

auto error = Listener->StartEndpoint(request, FastPathVolume, Session)
.GetValueSync();
UNIT_ASSERT_C(!HasError(error), error);

UNIT_ASSERT_VALUES_EQUAL(2, History.size());

auto* create = std::get_if<TCreateExternalEndpoint>(&History[0]);
UNIT_ASSERT_C(create, "actual entry: " << History[0].index());

UNIT_ASSERT_VALUES_EQUAL(Volume.GetDiskId(), create->DiskId);

/*
--serial local0 2
--socket-path /tmp/socket.vhost 2
-q 2 2
--client-id ... 2
--disk-id ... 2
--device-backend ... 2
--block-size ... 2
--device ... 2
Expand All @@ -370,31 +481,36 @@ Y_UNIT_TEST_SUITE(TExternalEndpointTest)
*/

UNIT_ASSERT_VALUES_EQUAL(19, create->CmdArgs.size());
UNIT_ASSERT_VALUES_EQUAL("client", GetArg(create->CmdArgs, "--client-id"));
UNIT_ASSERT_VALUES_EQUAL("vol0", GetArg(create->CmdArgs, "--disk-id"));
UNIT_ASSERT_VALUES_EQUAL("local0", GetArg(create->CmdArgs, "--serial"));

UNIT_ASSERT_VALUES_EQUAL(
"/tmp/socket.vhost",
GetArg(create->CmdArgs, "--socket-path"));

UNIT_ASSERT_VALUES_EQUAL("2", GetArg(create->CmdArgs, "-q"));
UNIT_ASSERT_VALUES_EQUAL("aio", GetArg(create->CmdArgs, "--device-backend"));

UNIT_ASSERT_VALUES_EQUAL("client", GetArg(create->CmdArgs, "--client-id"));

UNIT_ASSERT_VALUES_EQUAL("vol0", GetArg(create->CmdArgs, "--disk-id"));

UNIT_ASSERT_VALUES_EQUAL("rdma", GetArg(create->CmdArgs, "--device-backend"));

UNIT_ASSERT_VALUES_EQUAL("4096", GetArg(create->CmdArgs, "--block-size"));

UNIT_ASSERT(FindPtr(create->CmdArgs, "--read-only"));

auto devices = GetArgN(create->CmdArgs, "--device");

UNIT_ASSERT_VALUES_EQUAL(2, devices.size());

UNIT_ASSERT_VALUES_EQUAL(TStringBuilder()
<< "/dev/disk/by-path/pci-0000:00:16.0-sas-phy2-lun-0:"
<< "rdma://host1:1111/uuid1:"
<< (4'000 * 4_KB)
<< ":32000",
<< ":0",
devices[0]);

UNIT_ASSERT_VALUES_EQUAL(TStringBuilder()
<< "/dev/disk/by-path/pci-0000:00:16.0-sas-phy2-lun-0:"
<< "rdma://host2:2222/uuid2:"
<< (6'000 * 4_KB)
<< ":0",
devices[1]);
Expand Down

0 comments on commit 328cd76

Please sign in to comment.