NBS-3864: use volume block size for rdma backend

Don't assume that disk agent will expose devices with 512 block size. Instead use the volume block size which is supported by the disk agent devices.
ydb-platform · Jan 9, 2024 · d31e92c · d31e92c
1 parent af6cf43
commit d31e92c
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 17 deletions.
diff --git a/cloud/blockstore/libs/endpoints_vhost/external_vhost_server.cpp b/cloud/blockstore/libs/endpoints_vhost/external_vhost_server.cpp
@@ -847,6 +847,9 @@ class TExternalVhostEndpointListener final
         args.emplace_back("--device-backend");
         args.emplace_back(GetDeviceBackend(epType));
 
+        args.emplace_back("--block-size");
+        args.emplace_back(ToString(volume.GetBlockSize()));
+
         for (const auto& device: volume.GetDevices()) {
             const ui64 size = device.GetBlockCount() * volume.GetBlockSize();
 

diff --git a/cloud/blockstore/libs/endpoints_vhost/external_vhost_server_ut.cpp b/cloud/blockstore/libs/endpoints_vhost/external_vhost_server_ut.cpp
@@ -362,13 +362,14 @@ Y_UNIT_TEST_SUITE(TExternalEndpointTest)
                 --socket-path /tmp/socket.vhost     2
                 -q 2                                2
                 --device-backend ...                2
+                --block-size ...                    2
                 --device ...                        2
                 --device ...                        2
                 --read-only                         1
-                                                   17
+                                                   19
             */
 
-            UNIT_ASSERT_VALUES_EQUAL(17, create->CmdArgs.size());
+            UNIT_ASSERT_VALUES_EQUAL(19, create->CmdArgs.size());
             UNIT_ASSERT_VALUES_EQUAL("client", GetArg(create->CmdArgs, "--client-id"));
             UNIT_ASSERT_VALUES_EQUAL("vol0", GetArg(create->CmdArgs, "--disk-id"));
             UNIT_ASSERT_VALUES_EQUAL("local0", GetArg(create->CmdArgs, "--serial"));
@@ -379,6 +380,7 @@ Y_UNIT_TEST_SUITE(TExternalEndpointTest)
 
             UNIT_ASSERT_VALUES_EQUAL("2", GetArg(create->CmdArgs, "-q"));
             UNIT_ASSERT_VALUES_EQUAL("aio", GetArg(create->CmdArgs, "--device-backend"));
+            UNIT_ASSERT_VALUES_EQUAL("4096", GetArg(create->CmdArgs, "--block-size"));
             UNIT_ASSERT(FindPtr(create->CmdArgs, "--read-only"));
 
             auto devices = GetArgN(create->CmdArgs, "--device");

diff --git a/cloud/blockstore/vhost-server/backend_rdma.cpp b/cloud/blockstore/vhost-server/backend_rdma.cpp
@@ -39,7 +39,9 @@ class TRdmaBackend final: public IBackend
     TString ClientId;
     ICompletionStatsPtr CompletionStats;
     TSimpleStats CompletionStatsData;
-    bool ReadOnly;
+    bool ReadOnly = false;
+    ui32 BlockSize = 0;
+    ui32 SectorsToBlockShift = 0;
 
 public:
     explicit TRdmaBackend(ILoggingServicePtr logging);
@@ -76,6 +78,14 @@ vhd_bdev_info TRdmaBackend::Init(const TOptions& options)
     ClientId = options.ClientId;
     ReadOnly = options.ReadOnly;
 
+    BlockSize = options.BlockSize;
+    STORAGE_VERIFY(
+        BlockSize >= 512 && IsPowerOf2(BlockSize),
+        TWellKnownEntityTypes::ENDPOINT,
+        ClientId);
+
+    SectorsToBlockShift = MostSignificantBit(BlockSize) - VHD_SECTOR_SHIFT;
+
     auto rdmaClientConfig = std::make_shared<NRdma::TClientConfig>();
     rdmaClientConfig->QueueSize = options.RdmaClient.QueueSize;
     rdmaClientConfig->MaxBufferSize = options.RdmaClient.MaxBufferSize;
@@ -92,7 +102,7 @@ vhd_bdev_info TRdmaBackend::Init(const TOptions& options)
         NStorage::ERdmaTaskQueueOpt::DontUse);
 
     Volume.SetStorageMediaKind(NProto::STORAGE_MEDIA_SSD_NONREPLICATED);
-    Volume.SetBlockSize(VHD_SECTOR_SIZE);
+    Volume.SetBlockSize(BlockSize);
     Volume.SetDiskId(options.DiskId);
 
     ui64 totalBytes = 0;
@@ -120,29 +130,29 @@ vhd_bdev_info TRdmaBackend::Init(const TOptions& options)
                 << ", offset=" << chunk.Offset);
 
         STORAGE_VERIFY_C(
-            chunk.ByteCount % Volume.GetBlockSize() == 0,
+            chunk.ByteCount % BlockSize == 0,
             TWellKnownEntityTypes::ENDPOINT,
             ClientId,
             "device chunk size is not aligned to "
-                << Volume.GetBlockSize()
+                << BlockSize
                 << ", device=" << chunk.DevicePath
                 << ", byte_count=" << chunk.ByteCount);
 
-        device->SetBlockCount(chunk.ByteCount / Volume.GetBlockSize());
+        device->SetBlockCount(chunk.ByteCount / BlockSize);
         totalBytes += chunk.ByteCount;
     }
 
     STORAGE_INFO("Volume:"
         << " DiskId=" << Volume.GetDiskId()
-        << " TotalBlocks=" << totalBytes / Volume.GetBlockSize()
-        << " BlockSize=" << Volume.GetBlockSize());
+        << " TotalBlocks=" << totalBytes / BlockSize
+        << " BlockSize=" << BlockSize);
 
     return {
         .serial = options.Serial.c_str(),
         .socket_path = options.SocketPath.c_str(),
-        .block_size = Volume.GetBlockSize(),
+        .block_size = BlockSize,
         .num_queues = options.QueueCount,   // Max count of virtio queues
-        .total_blocks = totalBytes / Volume.GetBlockSize(),
+        .total_blocks = totalBytes / BlockSize,
         .features = ReadOnly ? VHD_BDEV_F_READONLY : 0};
 }
 
@@ -227,9 +237,9 @@ void TRdmaBackend::ProcessReadRequest(struct vhd_io* io, TCpuCycles startCycles)
     reqHeaders->SetTimestamp(TInstant::Now().MicroSeconds());
     reqHeaders->SetClientId(ClientId);
 
-    request->SetStartIndex(bio->first_sector);
-    request->SetBlocksCount(bio->total_sectors);
-    request->BlockSize = Volume.GetBlockSize();
+    request->SetStartIndex(bio->first_sector >> SectorsToBlockShift);
+    request->SetBlocksCount(bio->total_sectors >> SectorsToBlockShift);
+    request->BlockSize = BlockSize;
     request->Sglist.SetSgList(std::move(ConvertVhdSgList(bio->sglist)));
 
     STORAGE_DEBUG(
@@ -269,9 +279,9 @@ void TRdmaBackend::ProcessWriteRequest(
     reqHeaders->SetTimestamp(TInstant::Now().MicroSeconds());
     reqHeaders->SetClientId(ClientId);
 
-    request->SetStartIndex(bio->first_sector);
-    request->BlocksCount = bio->total_sectors;
-    request->BlockSize = Volume.GetBlockSize();
+    request->SetStartIndex(bio->first_sector >> SectorsToBlockShift);
+    request->BlocksCount = bio->total_sectors >> SectorsToBlockShift;
+    request->BlockSize = BlockSize;
     request->Sglist.SetSgList(std::move(ConvertVhdSgList(bio->sglist)));
 
     STORAGE_DEBUG(

diff --git a/cloud/blockstore/vhost-server/options.cpp b/cloud/blockstore/vhost-server/options.cpp
@@ -96,6 +96,10 @@ void TOptions::Parse(int argc, char** argv)
         .RequiredArgument("INT")
         .StoreResultDef(&BatchSize);
 
+    opts.AddLongOption("block-size", "size of block device")
+        .RequiredArgument("INT")
+        .StoreResultDef(&BlockSize);
+
     opts.AddLongOption('q', "queue-count")
         .RequiredArgument("INT")
         .StoreResult(&QueueCount);

diff --git a/cloud/blockstore/vhost-server/options.h b/cloud/blockstore/vhost-server/options.h
@@ -28,6 +28,7 @@ struct TOptions
     bool NoSync = false;
     bool NoChmod = false;
     ui32 BatchSize = 1024;
+    ui32 BlockSize = 512;
     ui32 QueueCount = 0;
 
     TString LogType = "json";