diff --git a/cloud/blockstore/libs/endpoints_grpc/socket_endpoint_listener.cpp b/cloud/blockstore/libs/endpoints_grpc/socket_endpoint_listener.cpp index 539b570a0fc..2c745f2d7ab 100644 --- a/cloud/blockstore/libs/endpoints_grpc/socket_endpoint_listener.cpp +++ b/cloud/blockstore/libs/endpoints_grpc/socket_endpoint_listener.cpp @@ -73,13 +73,16 @@ class TEndpointService final const std::shared_ptr LastMountResponsePtr; public: - TEndpointService( - TString diskId, - ui32 blockSize, - ISessionPtr session) + TEndpointService(TString diskId, ui32 blockSize, ISessionPtr session) : DiskId(std::move(diskId)) , BlockSize(blockSize) - , StorageAdapter(session, blockSize, true) + , StorageAdapter( + session, + blockSize, + true, // normalize, + TDuration::Zero(), // maxRequestDuration + TDuration::Zero() // shutdownTimeout + ) , Session(std::move(session)) , LastMountResponsePtr(std::make_shared()) { diff --git a/cloud/blockstore/libs/nbd/server_handler.cpp b/cloud/blockstore/libs/nbd/server_handler.cpp index b026e5099e8..18ee38c6d43 100644 --- a/cloud/blockstore/libs/nbd/server_handler.cpp +++ b/cloud/blockstore/libs/nbd/server_handler.cpp @@ -939,7 +939,6 @@ IServerHandlerFactoryPtr CreateServerHandlerFactory( std::move(storage), options.ClientId, options.BlockSize, - NBD_MAX_BUFFER_SIZE / options.BlockSize, options.UnalignedRequestsDisabled); return std::make_shared( diff --git a/cloud/blockstore/libs/rdma/iface/client.h b/cloud/blockstore/libs/rdma/iface/client.h index e8e851f2b80..0a94458a996 100644 --- a/cloud/blockstore/libs/rdma/iface/client.h +++ b/cloud/blockstore/libs/rdma/iface/client.h @@ -22,7 +22,13 @@ namespace NCloud::NBlockStore::NRdma { struct TClientConfig { ui32 QueueSize = 10; - ui32 MaxBufferSize = 1024*1024; + // Keep the value greater then MaxSubRequestSize, ProcessingRangeSize, + // ResyncRangeSize in cloud/blockstore/libs/service/device_handler.cpp + // cloud/blockstore/libs/storage/partition_nonrepl/model/processing_blocks.h + // cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_util.h + // Keep sync with MaxBufferSize in cloud/blockstore/vhost-server/options.h + // and cloud/blockstore/libs/rdma/iface/server.h + ui32 MaxBufferSize = 4_MB + 4_KB; EWaitMode WaitMode = EWaitMode::Poll; ui32 PollerThreads = 1; TDuration MaxReconnectDelay = TDuration::Seconds(60); diff --git a/cloud/blockstore/libs/rdma/iface/server.h b/cloud/blockstore/libs/rdma/iface/server.h index a83b6e365a1..e0f5b2e139d 100644 --- a/cloud/blockstore/libs/rdma/iface/server.h +++ b/cloud/blockstore/libs/rdma/iface/server.h @@ -20,7 +20,8 @@ struct TServerConfig { ui32 Backlog = 10; ui32 QueueSize = 10; - ui32 MaxBufferSize = 1024*1024; + // Keep sync with MaxBufferSize in cloud/blockstore/libs/rdma/iface/client.h + ui32 MaxBufferSize = 4_MB + 4_KB; TDuration KeepAliveTimeout = TDuration::Seconds(10); EWaitMode WaitMode = EWaitMode::Poll; ui32 PollerThreads = 1; diff --git a/cloud/blockstore/libs/rdma_test/rdma_test_environment.cpp b/cloud/blockstore/libs/rdma_test/rdma_test_environment.cpp index 0a21adf9453..61c68520bca 100644 --- a/cloud/blockstore/libs/rdma_test/rdma_test_environment.cpp +++ b/cloud/blockstore/libs/rdma_test/rdma_test_environment.cpp @@ -15,8 +15,13 @@ TRdmaTestEnvironment::TRdmaTestEnvironment(size_t deviceSize, ui32 poolSize) : Storage(std::make_shared(deviceSize)) { THashMap devices; - devices[Device_1] = - std::make_shared(Storage, 4_KB, true, 0); + devices[Device_1] = std::make_shared( + Storage, + 4_KB, // storageBlockSize + true, // normalize, + TDuration::Zero(), // maxRequestDuration + TDuration::Zero() // shutdownTimeout + ); TVector uuids; for (const auto& [key, value]: devices) { uuids.push_back(key); diff --git a/cloud/blockstore/libs/service/aligned_device_handler.cpp b/cloud/blockstore/libs/service/aligned_device_handler.cpp index 3f6afbe8eb9..705375d176a 100644 --- a/cloud/blockstore/libs/service/aligned_device_handler.cpp +++ b/cloud/blockstore/libs/service/aligned_device_handler.cpp @@ -132,11 +132,11 @@ TAlignedDeviceHandler::TAlignedDeviceHandler( IStoragePtr storage, TString clientId, ui32 blockSize, - ui32 maxBlockCount) + ui32 maxSubRequestSize) : Storage(std::move(storage)) , ClientId(std::move(clientId)) , BlockSize(blockSize) - , MaxBlockCount(maxBlockCount) + , MaxBlockCount(maxSubRequestSize / BlockSize) { Y_ABORT_UNLESS(MaxBlockCount > 0); } diff --git a/cloud/blockstore/libs/service/aligned_device_handler.h b/cloud/blockstore/libs/service/aligned_device_handler.h index 20feab0f3ca..f4793c076ee 100644 --- a/cloud/blockstore/libs/service/aligned_device_handler.h +++ b/cloud/blockstore/libs/service/aligned_device_handler.h @@ -57,7 +57,7 @@ class TAlignedDeviceHandler final IStoragePtr storage, TString clientId, ui32 blockSize, - ui32 maxBlockCount); + ui32 maxSubRequestSize); // implements IDeviceHandler NThreading::TFuture Read( diff --git a/cloud/blockstore/libs/service/device_handler.cpp b/cloud/blockstore/libs/service/device_handler.cpp index 2573a12f13c..e3c8d115fe9 100644 --- a/cloud/blockstore/libs/service/device_handler.cpp +++ b/cloud/blockstore/libs/service/device_handler.cpp @@ -13,16 +13,25 @@ namespace { constexpr ui32 MaxUnalignedRequestSize = 32_MB; +// Keep the value less than MaxBufferSize in +// cloud/blockstore/libs/rdma/iface/client.h +constexpr ui32 MaxSubRequestSize = 4_MB; + //////////////////////////////////////////////////////////////////////////////// struct TDefaultDeviceHandlerFactory final : public IDeviceHandlerFactory { + const ui32 MaxSubRequestSize; + + explicit TDefaultDeviceHandlerFactory(ui32 maxSubRequestSize) + : MaxSubRequestSize(maxSubRequestSize) + {} + IDeviceHandlerPtr CreateDeviceHandler( IStoragePtr storage, TString clientId, ui32 blockSize, - ui32 maxBlockCount, bool unalignedRequestsDisabled) override { if (unalignedRequestsDisabled) { @@ -30,14 +39,14 @@ struct TDefaultDeviceHandlerFactory final std::move(storage), std::move(clientId), blockSize, - maxBlockCount); + MaxSubRequestSize); } return std::make_shared( std::move(storage), std::move(clientId), blockSize, - maxBlockCount, + MaxSubRequestSize, MaxUnalignedRequestSize); } }; @@ -48,7 +57,12 @@ struct TDefaultDeviceHandlerFactory final IDeviceHandlerFactoryPtr CreateDefaultDeviceHandlerFactory() { - return std::make_shared(); + return std::make_shared(MaxSubRequestSize); +} + +IDeviceHandlerFactoryPtr CreateDeviceHandlerFactory(ui32 maxSubRequestSize) +{ + return std::make_shared(maxSubRequestSize); } } // namespace NCloud::NBlockStore diff --git a/cloud/blockstore/libs/service/device_handler.h b/cloud/blockstore/libs/service/device_handler.h index e4dcd328f8b..6b9bb86c09f 100644 --- a/cloud/blockstore/libs/service/device_handler.h +++ b/cloud/blockstore/libs/service/device_handler.h @@ -45,12 +45,12 @@ struct IDeviceHandlerFactory IStoragePtr storage, TString clientId, ui32 blockSize, - ui32 maxBlockCount, bool unalignedRequestsDisabled) = 0; }; //////////////////////////////////////////////////////////////////////////////// IDeviceHandlerFactoryPtr CreateDefaultDeviceHandlerFactory(); +IDeviceHandlerFactoryPtr CreateDeviceHandlerFactory(ui32 maxSubRequestSize); } // namespace NCloud::NBlockStore diff --git a/cloud/blockstore/libs/service/device_handler_ut.cpp b/cloud/blockstore/libs/service/device_handler_ut.cpp index 33b3b589cd9..de6699cb8ff 100644 --- a/cloud/blockstore/libs/service/device_handler_ut.cpp +++ b/cloud/blockstore/libs/service/device_handler_ut.cpp @@ -123,11 +123,11 @@ class TTestEnvironment }); }; - DeviceHandler = CreateDefaultDeviceHandlerFactory()->CreateDeviceHandler( + auto factory = CreateDeviceHandlerFactory(maxBlockCount * BlockSize); + DeviceHandler = factory->CreateDeviceHandler( std::move(testStorage), "testClientId", BlockSize, - maxBlockCount, unalignedRequestsDisabled); } @@ -335,11 +335,11 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest) auto storage = std::make_shared(); - auto deviceHandler = CreateDefaultDeviceHandlerFactory()->CreateDeviceHandler( + auto factory = CreateDeviceHandlerFactory(blocksCountLimit * blockSize); + auto deviceHandler = factory->CreateDeviceHandler( storage, clientId, blockSize, - blocksCountLimit, false); std::array zeroBlocks; @@ -402,7 +402,6 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest) storage, clientId, blockSize, - 1024, true); ui32 startIndex = 42; @@ -489,7 +488,6 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest) storage, clientId, blockSize, - 1024, true); { @@ -598,11 +596,11 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest) auto storage = std::make_shared(); - auto deviceHandler = CreateDefaultDeviceHandlerFactory()->CreateDeviceHandler( + auto factory = CreateDeviceHandlerFactory(blocksCountLimit * blockSize); + auto deviceHandler = factory->CreateDeviceHandler( storage, clientId, blockSize, - blocksCountLimit, unalignedRequestDisabled); storage->ZeroBlocksHandler = [&] ( @@ -717,7 +715,6 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest) storage, clientId, blockSize, - 1024, false); storage->WriteBlocksLocalHandler = [&] ( @@ -781,7 +778,6 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest) storage, clientId, blockSize, - 1024, false); storage->WriteBlocksLocalHandler = [&] ( diff --git a/cloud/blockstore/libs/service/storage.cpp b/cloud/blockstore/libs/service/storage.cpp index 8c4a25a6264..0079dff3c8b 100644 --- a/cloud/blockstore/libs/service/storage.cpp +++ b/cloud/blockstore/libs/service/storage.cpp @@ -21,6 +21,10 @@ namespace { //////////////////////////////////////////////////////////////////////////////// +constexpr ui32 MaxRequestSize = 32_MB; + +//////////////////////////////////////////////////////////////////////////////// + class TStorageStub final : public IStorage { @@ -173,7 +177,6 @@ class TStorageAdapter::TImpl const IStoragePtr Storage; const ui32 StorageBlockSize; const bool Normalize; - const ui32 MaxRequestSize; const TDuration MaxRequestDuration; std::shared_ptr InflightReads{ @@ -190,7 +193,6 @@ class TStorageAdapter::TImpl IStoragePtr storage, ui32 storageBlockSize, bool normalize, - ui32 maxRequestSize, TDuration maxRequestDuration); TFuture ReadBlocks( @@ -240,12 +242,10 @@ TStorageAdapter::TImpl::TImpl( IStoragePtr storage, ui32 storageBlockSize, bool normalize, - ui32 maxRequestSize, TDuration maxRequestDuration) : Storage(std::move(storage)) , StorageBlockSize(storageBlockSize) , Normalize(normalize) - , MaxRequestSize(maxRequestSize) , MaxRequestDuration(maxRequestDuration) {} @@ -666,14 +666,12 @@ TStorageAdapter::TStorageAdapter( IStoragePtr storage, ui32 storageBlockSize, bool normalize, - ui32 maxRequestSize, TDuration maxRequestDuration, TDuration shutdownTimeout) : Impl(std::make_unique( std::move(storage), storageBlockSize, normalize, - maxRequestSize, maxRequestDuration)) , ShutdownTimeout(shutdownTimeout) {} diff --git a/cloud/blockstore/libs/service/storage.h b/cloud/blockstore/libs/service/storage.h index 9b026ef1f25..77788401298 100644 --- a/cloud/blockstore/libs/service/storage.h +++ b/cloud/blockstore/libs/service/storage.h @@ -57,9 +57,8 @@ class TStorageAdapter ui32 storageBlockSize, bool normalize, // When true, we use StorageBlockSize when making // requests to the underlying storage - ui32 maxRequestSize = 0, - TDuration maxRequestDuration = TDuration::Zero(), - TDuration shutdownTimeout = TDuration::Zero()); + TDuration maxRequestDuration, + TDuration shutdownTimeout); ~TStorageAdapter(); diff --git a/cloud/blockstore/libs/service/storage_ut.cpp b/cloud/blockstore/libs/service/storage_ut.cpp index 10b9ac250c6..73cc18027e3 100644 --- a/cloud/blockstore/libs/service/storage_ut.cpp +++ b/cloud/blockstore/libs/service/storage_ut.cpp @@ -74,9 +74,11 @@ Y_UNIT_TEST_SUITE(TStorageTest) auto adapter = std::make_shared( storage, - 4_KB, // storageBlockSize - false, // normalize - 32_MB); // maxRequestSize + 4_KB, // storageBlockSize + false, // normalize, + TDuration::Zero(), // maxRequestDuration + TDuration::Zero() // shutdownTimeout + ); auto request = std::make_shared(); request->SetStartIndex(1000); @@ -143,9 +145,11 @@ Y_UNIT_TEST_SUITE(TStorageTest) auto adapter = std::make_shared( storage, - 4_KB, // storageBlockSize - true, // normalize - 32_MB); // maxRequestSize + 4_KB, // storageBlockSize + true, // normalize, + TDuration::Zero(), // maxRequestDuration + TDuration::Zero() // shutdownTimeout + ); auto request = std::make_shared(); request->SetStartIndex(1000); @@ -215,10 +219,10 @@ Y_UNIT_TEST_SUITE(TStorageTest) auto adapter = std::make_shared( storage, - 4_KB, // storageBlockSize - false, // normalize - 32_MB, // maxRequestSize - TDuration::Seconds(1) // timeout + 4_KB, // storageBlockSize + false, // normalize + TDuration::Seconds(1), // maxRequestDuration + TDuration::Zero() // shutdownTimeout ); auto now = TInstant::Seconds(1); @@ -337,7 +341,6 @@ Y_UNIT_TEST_SUITE(TStorageTest) storage, 4_KB, // storageBlockSize false, // normalize - 32_MB, // maxRequestSize TDuration::Seconds(1), // request timeout shutdownTimeout // shutdown timeout ); @@ -463,9 +466,11 @@ Y_UNIT_TEST_SUITE(TStorageTest) auto adapter = std::make_shared( storage, - 4096, // storageBlockSize + 4_KB, // storageBlockSize normalize, - 32_MB); // maxRequestSize + TDuration::Zero(), // request timeout + TDuration::Zero() // shutdown timeout + ); { // Requesting the reading of two blocks. Since the first block will // be filled with zeros, it should be optimized and will return an @@ -542,9 +547,11 @@ Y_UNIT_TEST_SUITE(TStorageTest) auto adapter = std::make_shared( storage, - 4096, // storageBlockSize + 4_KB, // storageBlockSize normalize, - 32_MB); // maxRequestSize + TDuration::Zero(), // request timeout + TDuration::Zero() // shutdown timeout + ); { auto request = std::make_shared(); diff --git a/cloud/blockstore/libs/service/unaligned_device_handler.cpp b/cloud/blockstore/libs/service/unaligned_device_handler.cpp index 97255c454c3..7846d5a88ae 100644 --- a/cloud/blockstore/libs/service/unaligned_device_handler.cpp +++ b/cloud/blockstore/libs/service/unaligned_device_handler.cpp @@ -470,13 +470,13 @@ TUnalignedDeviceHandler::TUnalignedDeviceHandler( IStoragePtr storage, TString clientId, ui32 blockSize, - ui32 maxBlockCount, + ui32 maxSubRequestSize, ui32 maxUnalignedRequestSize) : Backend(std::make_shared( std::move(storage), std::move(clientId), blockSize, - maxBlockCount)) + maxSubRequestSize)) , BlockSize(blockSize) , MaxUnalignedBlockCount(maxUnalignedRequestSize / BlockSize) {} diff --git a/cloud/blockstore/libs/service/unaligned_device_handler.h b/cloud/blockstore/libs/service/unaligned_device_handler.h index cc32d7e3e65..8e7fca365a8 100644 --- a/cloud/blockstore/libs/service/unaligned_device_handler.h +++ b/cloud/blockstore/libs/service/unaligned_device_handler.h @@ -45,7 +45,7 @@ class TUnalignedDeviceHandler final IStoragePtr storage, TString clientId, ui32 blockSize, - ui32 maxBlockCount, + ui32 maxSubRequestSize, ui32 maxUnalignedRequestSize); NThreading::TFuture Read( diff --git a/cloud/blockstore/libs/service_local/service_local.cpp b/cloud/blockstore/libs/service_local/service_local.cpp index 335308f7300..146541deb36 100644 --- a/cloud/blockstore/libs/service_local/service_local.cpp +++ b/cloud/blockstore/libs/service_local/service_local.cpp @@ -70,9 +70,12 @@ struct TMountSession , ClientId(std::move(clientId)) , Storage(storage) , StorageAdapter( - std::move(storage), - blockSize, - true) + std::move(storage), + blockSize, + true, // normalize, + TDuration::Zero(), // maxRequestDuration + TDuration::Zero() // shutdownTimeout + ) {} }; diff --git a/cloud/blockstore/libs/service_local/storage_spdk_ut.cpp b/cloud/blockstore/libs/service_local/storage_spdk_ut.cpp index 7c8a8bb653a..fc35a279d38 100644 --- a/cloud/blockstore/libs/service_local/storage_spdk_ut.cpp +++ b/cloud/blockstore/libs/service_local/storage_spdk_ut.cpp @@ -654,7 +654,13 @@ Y_UNIT_TEST_SUITE(TSpdkStorageTest) blockSize, CreateMonitoringServiceStub()); - TStorageAdapter storageAdapter(std::move(storage), blockSize, true); + TStorageAdapter storageAdapter( + std::move(storage), + blockSize, + true, // normalize, + TDuration::Zero(), // maxRequestDuration + TDuration::Zero() // shutdownTimeout + ); Y_DEFER { device.Stop(); }; diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp index 00b12e2d008..fb3a2761877 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp @@ -42,10 +42,6 @@ namespace { //////////////////////////////////////////////////////////////////////////////// -constexpr ui32 MaxRequestSize = 128*1024*1024; // TODO - -//////////////////////////////////////////////////////////////////////////////// - void ToDeviceStats( const TString& uuid, const TString& name, @@ -358,7 +354,6 @@ TFuture TDiskAgentState::InitSpdkStorage() std::move(r.Devices[i]), config.GetBlockSize(), false, // normalize - MaxRequestSize, ioTimeout, AgentConfig->GetShutdownTimeout()) }; @@ -405,7 +400,6 @@ TFuture TDiskAgentState::InitAioStorage() std::move(r.Devices[i]), config.GetBlockSize(), false, // normalize - MaxRequestSize, ioTimeout, AgentConfig->GetShutdownTimeout()), .Stats = std::move(r.Stats[i]) diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/model/processing_blocks.h b/cloud/blockstore/libs/storage/partition_nonrepl/model/processing_blocks.h index 23742bd92f3..f2ab6e9ab5a 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/model/processing_blocks.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/model/processing_blocks.h @@ -8,6 +8,8 @@ namespace NCloud::NBlockStore::NStorage { //////////////////////////////////////////////////////////////////////////////// // We process 4 MB of data at a time. +// Keep the value less than MaxBufferSize in +// cloud/blockstore/libs/rdma/iface/client.h constexpr ui64 ProcessingRangeSize = 4_MB; //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_util.h b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_util.h index 1f8ecec4856..41566f14c1b 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_util.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_util.h @@ -22,6 +22,8 @@ struct TReplicaDescriptor //////////////////////////////////////////////////////////////////////////////// // TODO: increase x4? +// Keep the value less than MaxBufferSize in +// cloud/blockstore/libs/rdma/iface/client.h constexpr ui64 ResyncRangeSize = 4_MB; //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/blockstore/libs/vhost/server.cpp b/cloud/blockstore/libs/vhost/server.cpp index 92663afd1bf..a79f64015fc 100644 --- a/cloud/blockstore/libs/vhost/server.cpp +++ b/cloud/blockstore/libs/vhost/server.cpp @@ -29,11 +29,6 @@ namespace { //////////////////////////////////////////////////////////////////////////////// -// Maximum size of the request data. -constexpr ui32 MaxRequestSize = 32_MB; - -//////////////////////////////////////////////////////////////////////////////// - struct TReadBlocksLocalMethod { static TFuture Execute( @@ -426,7 +421,6 @@ class TExecutor final std::move(storage), options.ClientId, options.BlockSize, - MaxRequestSize / options.BlockSize, options.UnalignedRequestsDisabled); auto endpoint = std::make_shared(