From bb3e226d35f001d9d3d46a4adb64101f037ee3fb Mon Sep 17 00:00:00 2001 From: sharpeye Date: Mon, 23 Sep 2024 21:13:22 +0000 Subject: [PATCH] issue-2086: introduce IFileIOServiceProvider --- .../file_io_service_provider.cpp | 115 ++++++++ .../service_local/file_io_service_provider.h | 31 ++ .../file_io_service_provider_ut.cpp | 269 ++++++++++++++++++ cloud/blockstore/libs/service_local/public.h | 9 + .../blockstore/libs/service_local/ut/ya.make | 3 +- cloud/blockstore/libs/service_local/ya.make | 7 +- 6 files changed, 430 insertions(+), 4 deletions(-) create mode 100644 cloud/blockstore/libs/service_local/file_io_service_provider.cpp create mode 100644 cloud/blockstore/libs/service_local/file_io_service_provider.h create mode 100644 cloud/blockstore/libs/service_local/file_io_service_provider_ut.cpp diff --git a/cloud/blockstore/libs/service_local/file_io_service_provider.cpp b/cloud/blockstore/libs/service_local/file_io_service_provider.cpp new file mode 100644 index 0000000000..2fabecc223 --- /dev/null +++ b/cloud/blockstore/libs/service_local/file_io_service_provider.cpp @@ -0,0 +1,115 @@ +#include "file_io_service_provider.h" + +#include + +#include +#include +#include + +namespace NCloud::NBlockStore::NServer { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +class TFileIOServiceProvider final + : public IFileIOServiceProvider +{ +private: + const ui32 PathsPerServices; + std::function Factory; + + TVector> PathToFileIOIndex; + TVector FileIOs; + +public: + explicit TFileIOServiceProvider( + ui32 pathsPerServices, + std::function factory) + : PathsPerServices{Max(pathsPerServices, 1U)} + , Factory(std::move(factory)) + {} + + void Start() override + {} + + void Stop() override + { + for (auto& fileIO: FileIOs) { + fileIO->Stop(); + } + } + + IFileIOServicePtr CreateFileIOService(TStringBuf filePath) override + { + if (auto* p = FindIfPtr( + PathToFileIOIndex, + [=](const auto& p) { return p.first == filePath; })) + { + return FileIOs[p->second]; + } + + if ((PathToFileIOIndex.size() + 1) > PathsPerServices * FileIOs.size()) + { + auto service = Factory(); + Y_DEBUG_ABORT_UNLESS(service); + service->Start(); + FileIOs.push_back(service); + } + + PathToFileIOIndex.emplace_back(filePath, FileIOs.size() - 1); + + return FileIOs.back(); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TFileIOServiceProviderStub final + : IFileIOServiceProvider +{ + IFileIOServicePtr FileIO; + + explicit TFileIOServiceProviderStub( + IFileIOServicePtr fileIO) + : FileIO{std::move(fileIO)} + {} + + void Start() override + { + FileIO->Start(); + } + + void Stop() override + { + FileIO->Stop(); + } + + IFileIOServicePtr CreateFileIOService(TStringBuf filePath) override + { + Y_UNUSED(filePath); + + return FileIO; + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +IFileIOServiceProviderPtr CreateFileIOServiceProviderStub( + IFileIOServicePtr fileIO) +{ + return std::make_shared(std::move(fileIO)); +} + +IFileIOServiceProviderPtr CreateFileIOServiceProvider( + ui32 filePathsPerServices, + std::function factory) +{ + return std::make_shared( + filePathsPerServices, + std::move(factory)); +} + +} // namespace NCloud::NBlockStore::NServer diff --git a/cloud/blockstore/libs/service_local/file_io_service_provider.h b/cloud/blockstore/libs/service_local/file_io_service_provider.h new file mode 100644 index 0000000000..e0feceeff6 --- /dev/null +++ b/cloud/blockstore/libs/service_local/file_io_service_provider.h @@ -0,0 +1,31 @@ +#pragma once + +#include "public.h" + +#include +#include + +#include + +#include + +namespace NCloud::NBlockStore::NServer { + +//////////////////////////////////////////////////////////////////////////////// + +struct IFileIOServiceProvider + : IStartable +{ + virtual IFileIOServicePtr CreateFileIOService(TStringBuf filePath) = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +IFileIOServiceProviderPtr CreateFileIOServiceProviderStub( + IFileIOServicePtr fileIO); + +IFileIOServiceProviderPtr CreateFileIOServiceProvider( + ui32 filePathsPerServices, + std::function factory); + +} // namespace NCloud::NBlockStore::NServer diff --git a/cloud/blockstore/libs/service_local/file_io_service_provider_ut.cpp b/cloud/blockstore/libs/service_local/file_io_service_provider_ut.cpp new file mode 100644 index 0000000000..73b585e263 --- /dev/null +++ b/cloud/blockstore/libs/service_local/file_io_service_provider_ut.cpp @@ -0,0 +1,269 @@ +#include "file_io_service_provider.h" + +#include + +#include + +namespace NCloud::NBlockStore::NServer { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +struct TTestFileIOService + : IFileIOService +{ + int Started = 0; + int Stopped = 0; + + void Start() override + { + ++Started; + } + + void Stop() override + { + ++Stopped; + } + + void AsyncRead( + TFileHandle& file, + i64 offset, + TArrayRef buffer, + TFileIOCompletion* completion) override + { + Y_UNUSED(file); + Y_UNUSED(offset); + Y_UNUSED(buffer); + Y_UNUSED(completion); + } + + void AsyncWrite( + TFileHandle& file, + i64 offset, + TArrayRef buffer, + TFileIOCompletion* completion) override + { + Y_UNUSED(file); + Y_UNUSED(offset); + Y_UNUSED(buffer); + Y_UNUSED(completion); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TTestProvider + : IFileIOServiceProvider +{ + int Started = 0; + int Stopped = 0; + + TVector> FileIOs; + + void Start() override + { + ++Started; + } + + void Stop() override + { + ++Stopped; + + for (auto& s: FileIOs) { + s->Stop(); + } + } + + IFileIOServicePtr CreateFileIOService(TStringBuf filePath) override + { + Y_UNUSED(filePath); + + auto service = std::make_shared(); + + FileIOs.push_back(service); + + return service; + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TFileIOServiceProviderTest) +{ + Y_UNIT_TEST(ShouldCreateSingleFileIOService) + { + auto fileIO = std::make_shared(); + UNIT_ASSERT_VALUES_EQUAL(0, fileIO->Started); + + auto provider = CreateFileIOServiceProviderStub(fileIO); + provider->Start(); + + UNIT_ASSERT_VALUES_EQUAL(1, fileIO->Started); + + auto io1 = provider->CreateFileIOService("foo"); + UNIT_ASSERT(io1 == fileIO); + UNIT_ASSERT_VALUES_EQUAL(1, fileIO->Started); + + auto io2 = provider->CreateFileIOService("bar"); + UNIT_ASSERT(io2 == fileIO); + auto io3 = provider->CreateFileIOService("foo"); + UNIT_ASSERT(io3 == fileIO); + auto io4 = provider->CreateFileIOService("buz"); + UNIT_ASSERT(io4 == fileIO); + + UNIT_ASSERT_VALUES_EQUAL(1, fileIO->Started); + UNIT_ASSERT_VALUES_EQUAL(0, fileIO->Stopped); + + provider->Stop(); + UNIT_ASSERT_VALUES_EQUAL(1, fileIO->Stopped); + } + + void ShouldCreateOneServicePerOnePathImpl(ui32 pathsToServices) + { + TTestProvider upstream; + + auto provider = CreateFileIOServiceProvider( + pathsToServices, + [&] { return upstream.CreateFileIOService({}); }); + + provider->Start(); + + UNIT_ASSERT_VALUES_EQUAL(0, upstream.FileIOs.size()); + + auto io1 = provider->CreateFileIOService("p1"); + UNIT_ASSERT_VALUES_EQUAL(1, upstream.FileIOs.size()); + UNIT_ASSERT(io1 == upstream.FileIOs[0]); + UNIT_ASSERT_VALUES_EQUAL(1, upstream.FileIOs[0]->Started); + + auto io2 = provider->CreateFileIOService("p2"); + UNIT_ASSERT_VALUES_EQUAL(2, upstream.FileIOs.size()); + UNIT_ASSERT_VALUES_EQUAL(1, upstream.FileIOs[0]->Started); + UNIT_ASSERT_VALUES_EQUAL(1, upstream.FileIOs[1]->Started); + UNIT_ASSERT(io2 == upstream.FileIOs[1]); + UNIT_ASSERT(io1 != io2); + + auto io3 = provider->CreateFileIOService("p3"); + UNIT_ASSERT_VALUES_EQUAL(3, upstream.FileIOs.size()); + UNIT_ASSERT(io2 != io3); + + auto io4 = provider->CreateFileIOService("p4"); + UNIT_ASSERT_VALUES_EQUAL(4, upstream.FileIOs.size()); + UNIT_ASSERT(io3 != io4); + + auto io5 = provider->CreateFileIOService("p1"); + UNIT_ASSERT_VALUES_EQUAL(4, upstream.FileIOs.size()); + UNIT_ASSERT(io1 == io5); + + auto io6 = provider->CreateFileIOService("p4"); + UNIT_ASSERT_VALUES_EQUAL(4, upstream.FileIOs.size()); + UNIT_ASSERT(io4 == io6); + + provider->Stop(); + for (const auto& s: upstream.FileIOs) { + UNIT_ASSERT_VALUES_EQUAL(1, s->Started); + UNIT_ASSERT_VALUES_EQUAL(1, s->Stopped); + } + } + + Y_UNIT_TEST(ShouldCreateOneServicePerOnePath) + { + ShouldCreateOneServicePerOnePathImpl(0); + ShouldCreateOneServicePerOnePathImpl(1); + } + + Y_UNIT_TEST(ShouldCreateOneServicePerTwoPaths) + { + const ui32 pathsToServices = 2; + + TTestProvider upstream; + + auto provider = CreateFileIOServiceProvider( + pathsToServices, + [&] { return upstream.CreateFileIOService({}); }); + + provider->Start(); + + UNIT_ASSERT_VALUES_EQUAL(0, upstream.FileIOs.size()); + + auto io1 = provider->CreateFileIOService("p1"); + UNIT_ASSERT_VALUES_EQUAL(1, upstream.FileIOs.size()); + UNIT_ASSERT(io1); + + auto io2 = provider->CreateFileIOService("p2"); + UNIT_ASSERT_VALUES_EQUAL(1, upstream.FileIOs.size()); + UNIT_ASSERT(io1 == io2); + + auto io3 = provider->CreateFileIOService("p3"); + UNIT_ASSERT_VALUES_EQUAL(2, upstream.FileIOs.size()); + + auto io4 = provider->CreateFileIOService("p4"); + UNIT_ASSERT_VALUES_EQUAL(2, upstream.FileIOs.size()); + + UNIT_ASSERT(io3 == io4); + UNIT_ASSERT(io1 != io4); + + auto io5 = provider->CreateFileIOService("p1"); + UNIT_ASSERT_VALUES_EQUAL(2, upstream.FileIOs.size()); + UNIT_ASSERT(io1 == io5); + + auto io6 = provider->CreateFileIOService("p6"); + UNIT_ASSERT_VALUES_EQUAL(3, upstream.FileIOs.size()); + UNIT_ASSERT(io1 != io6); + UNIT_ASSERT(io3 != io6); + + auto io7 = provider->CreateFileIOService("p6"); + UNIT_ASSERT_VALUES_EQUAL(3, upstream.FileIOs.size()); + UNIT_ASSERT(io6 == io7); + + provider->Stop(); + for (const auto& s: upstream.FileIOs) { + UNIT_ASSERT_VALUES_EQUAL(1, s->Started); + UNIT_ASSERT_VALUES_EQUAL(1, s->Stopped); + } + } + + Y_UNIT_TEST(ShouldCreateOneServicePerFourPaths) + { + const ui32 pathsToServices = 4; + const ui32 devicesPerPath = 64; + const ui32 pathCount = 6; + const ui32 expectedServiceCount = 2; + + TTestProvider upstream; + + auto provider = CreateFileIOServiceProvider( + pathsToServices, + [&] { return upstream.CreateFileIOService({}); }); + + provider->Start(); + + for (ui32 i = 0; i != pathCount; ++i) { + const TString path = + "/dev/disk/by-partlabel/NVMENBS0" + ToString(i); + + for (ui32 j = 0; j != devicesPerPath; ++j) { + auto fileIO = provider->CreateFileIOService(path); + UNIT_ASSERT(fileIO == upstream.FileIOs.back()); + } + } + + UNIT_ASSERT_VALUES_EQUAL(expectedServiceCount, upstream.FileIOs.size()); + + for (const auto& s: upstream.FileIOs) { + UNIT_ASSERT_VALUES_EQUAL(1, s->Started); + UNIT_ASSERT_VALUES_EQUAL(0, s->Stopped); + } + + provider->Stop(); + + for (const auto& s: upstream.FileIOs) { + UNIT_ASSERT_VALUES_EQUAL(1, s->Started); + UNIT_ASSERT_VALUES_EQUAL(1, s->Stopped); + } + } +} + +} // namespace NCloud::NBlockStore::NServer diff --git a/cloud/blockstore/libs/service_local/public.h b/cloud/blockstore/libs/service_local/public.h index 13db8e64b9..651550c9ac 100644 --- a/cloud/blockstore/libs/service_local/public.h +++ b/cloud/blockstore/libs/service_local/public.h @@ -8,4 +8,13 @@ namespace NProto { class TLocalServiceConfig; } // namespace NProto +namespace NServer { + +//////////////////////////////////////////////////////////////////////////////// + +struct IFileIOServiceProvider; +using IFileIOServiceProviderPtr = std::shared_ptr; + +} // namespace NServer + } // namespace NCloud::NBlockStore diff --git a/cloud/blockstore/libs/service_local/ut/ya.make b/cloud/blockstore/libs/service_local/ut/ya.make index ab89892858..a711812d40 100644 --- a/cloud/blockstore/libs/service_local/ut/ya.make +++ b/cloud/blockstore/libs/service_local/ut/ya.make @@ -6,9 +6,10 @@ TIMEOUT(180) SRCS( compound_storage_ut.cpp + file_io_service_provider_ut.cpp storage_aio_ut.cpp - storage_rdma_ut.cpp storage_null_ut.cpp + storage_rdma_ut.cpp storage_spdk_ut.cpp ) diff --git a/cloud/blockstore/libs/service_local/ya.make b/cloud/blockstore/libs/service_local/ya.make index 6684751887..19c15d3b8b 100644 --- a/cloud/blockstore/libs/service_local/ya.make +++ b/cloud/blockstore/libs/service_local/ya.make @@ -3,6 +3,7 @@ LIBRARY() SRCS( broken_storage.cpp compound_storage.cpp + file_io_service_provider.cpp rdma_protocol.cpp service_local.cpp storage_aio.cpp @@ -20,13 +21,13 @@ PEERDIR( cloud/blockstore/libs/service cloud/blockstore/libs/spdk/iface cloud/blockstore/libs/storage/protos - + cloud/storage/core/libs/common - + library/cpp/aio library/cpp/deprecated/atomic library/cpp/protobuf/util - + contrib/libs/protobuf )