diff --git a/README.md b/README.md index aa46283d920..7419306dfdc 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,16 @@ -## Network Block Store +## Network Block Store and Network File Store Network Block Device implementation over YDB BlobStorage or over our own storage nodes. Offers reliable thin-provisioned block devices which support snapshots. +Network File System implementation over YDB BlobStorage. Offers POSIX-compliant scalable filesystem which can be attached to virtual machines via virtiofs or simply mounted via FUSE. ### Quickstart Follow the instructions [here](example/README.md) to build and run NBS on your machine and to attach an NBS-based disk via NBD. NBS-based disks can be attached via vhost-user-blk as well. +Follow the instructions [here](cloud/filestore/README.md) to build and run Filestore on your machine and to attach it to a virtual machine via virtiofs or mount it on your host via FUSE. + Follow the instructions [here](CLANG-FORMAT.md) to install clang-format for formatting the code. ### How to Deploy -TODO +Follow the instructions [here](deploy/README.md) to deploy NBS and Filestore on a cluster. diff --git a/cloud/filestore/apps/client/lib/command.h b/cloud/filestore/apps/client/lib/command.h index 6c2983535c9..baa533cccc1 100644 --- a/cloud/filestore/apps/client/lib/command.h +++ b/cloud/filestore/apps/client/lib/command.h @@ -77,6 +77,11 @@ class TCommand return Opts; } + TLog& AccessLog() + { + return Log; + } + protected: virtual void Init(); @@ -177,16 +182,22 @@ class TFileStoreCommand class TSessionGuard final { TFileStoreCommand& FileStoreCmd; + TLog& Log; public: explicit TSessionGuard(TFileStoreCommand& fileStoreCmd) : FileStoreCmd(fileStoreCmd) + , Log(FileStoreCmd.AccessLog()) { } ~TSessionGuard() { - FileStoreCmd.DestroySession(); + try { + FileStoreCmd.DestroySession(); + } catch (...) { + STORAGE_ERROR("~TSessionGuard: " << CurrentExceptionMessage()); + } } }; diff --git a/cloud/filestore/apps/client/lib/ls.cpp b/cloud/filestore/apps/client/lib/ls.cpp index e1a91937b86..a29360e6476 100644 --- a/cloud/filestore/apps/client/lib/ls.cpp +++ b/cloud/filestore/apps/client/lib/ls.cpp @@ -94,6 +94,8 @@ TString NodeTypeToString(NProto::ENodeType nodeType) return "l"; case NProto::E_SOCK_NODE: return "s"; + case NProto::E_SYMLINK_NODE: + return "L"; default: ythrow yexception() << "must be unreachable"; } diff --git a/cloud/filestore/config/storage.proto b/cloud/filestore/config/storage.proto index bbb09da9690..e7438b7fa2a 100644 --- a/cloud/filestore/config/storage.proto +++ b/cloud/filestore/config/storage.proto @@ -416,4 +416,11 @@ message TStorageConfig // automatically recover after various races that may happen during index // tablet startup due to bugs. optional uint32 MaxBackpressurePeriodBeforeSuicide = 392; // in ms + + // settings for ydb config dispatcher service. + // optional NCloud.NProto.TConfigDispatcherSettings ConfigDispatcherSettings = 393; + + // If the number of blocks marked for deletion via large deletion markers + // exceeds this threshold, large truncate-like operations will be rejected. + optional uint64 LargeDeletionMarkersThresholdForBackpressure = 394; } diff --git a/cloud/filestore/libs/diagnostics/critical_events.h b/cloud/filestore/libs/diagnostics/critical_events.h index cbf49802045..cc00fb6706a 100644 --- a/cloud/filestore/libs/diagnostics/critical_events.h +++ b/cloud/filestore/libs/diagnostics/critical_events.h @@ -22,6 +22,7 @@ namespace NCloud::NFileStore{ xxx(AsyncDestroyHandleFailed) \ xxx(DuplicateRequestId) \ xxx(InvalidDupCacheEntry) \ + xxx(GeneratedOrphanNode) \ // FILESTORE_CRITICAL_EVENTS #define FILESTORE_IMPOSSIBLE_EVENTS(xxx) \ diff --git a/cloud/filestore/libs/storage/api/tablet.h b/cloud/filestore/libs/storage/api/tablet.h index 4d285c0ecdb..838fe363e55 100644 --- a/cloud/filestore/libs/storage/api/tablet.h +++ b/cloud/filestore/libs/storage/api/tablet.h @@ -32,6 +32,9 @@ namespace NCloud::NFileStore::NStorage { xxx(GetStorageConfig, __VA_ARGS__) \ xxx(GetNodeAttrBatch, __VA_ARGS__) \ xxx(WriteCompactionMap, __VA_ARGS__) \ + xxx(UnsafeDeleteNode, __VA_ARGS__) \ + xxx(UnsafeUpdateNode, __VA_ARGS__) \ + xxx(UnsafeGetNode, __VA_ARGS__) \ // FILESTORE_TABLET_REQUESTS //////////////////////////////////////////////////////////////////////////////// @@ -97,6 +100,15 @@ struct TEvIndexTablet EvWriteCompactionMapRequest = EvBegin + 33, EvWriteCompactionMapResponse, + EvUnsafeDeleteNodeRequest = EvBegin + 35, + EvUnsafeDeleteNodeResponse, + + EvUnsafeUpdateNodeRequest = EvBegin + 37, + EvUnsafeUpdateNodeResponse, + + EvUnsafeGetNodeRequest = EvBegin + 39, + EvUnsafeGetNodeResponse, + EvEnd }; diff --git a/cloud/filestore/libs/storage/core/config.cpp b/cloud/filestore/libs/storage/core/config.cpp index d25335010c0..b3cf7608c5b 100644 --- a/cloud/filestore/libs/storage/core/config.cpp +++ b/cloud/filestore/libs/storage/core/config.cpp @@ -50,11 +50,12 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases; xxx(MaxBlocksPerTruncateTx, ui32, 0 /*TODO: 32GiB/4KiB*/ )\ xxx(MaxTruncateTxInflight, ui32, 10 )\ \ - xxx(MaxFileBlocks, ui32, 300_GB / 4_KB )\ - xxx(LargeDeletionMarkersEnabled, bool, false )\ - xxx(LargeDeletionMarkerBlocks, ui64, 1_GB / 4_KB )\ - xxx(LargeDeletionMarkersThreshold, ui64, 128_GB / 4_KB )\ - xxx(LargeDeletionMarkersCleanupThreshold, ui64, 1_TB / 4_KB )\ + xxx(MaxFileBlocks, ui32, 300_GB / 4_KB )\ + xxx(LargeDeletionMarkersEnabled, bool, false )\ + xxx(LargeDeletionMarkerBlocks, ui64, 1_GB / 4_KB )\ + xxx(LargeDeletionMarkersThreshold, ui64, 128_GB / 4_KB )\ + xxx(LargeDeletionMarkersCleanupThreshold, ui64, 1_TB / 4_KB )\ + xxx(LargeDeletionMarkersThresholdForBackpressure, ui64, 10_TB / 4_KB )\ \ xxx(CompactionRetryTimeout, TDuration, TDuration::Seconds(1) )\ xxx(BlobIndexOpsPriority, \ diff --git a/cloud/filestore/libs/storage/core/config.h b/cloud/filestore/libs/storage/core/config.h index d77c8d27c07..45c54e55e28 100644 --- a/cloud/filestore/libs/storage/core/config.h +++ b/cloud/filestore/libs/storage/core/config.h @@ -266,6 +266,7 @@ class TStorageConfig ui64 GetLargeDeletionMarkerBlocks() const; ui64 GetLargeDeletionMarkersThreshold() const; ui64 GetLargeDeletionMarkersCleanupThreshold() const; + ui64 GetLargeDeletionMarkersThresholdForBackpressure() const; bool GetMultipleStageRequestThrottlingEnabled() const; }; diff --git a/cloud/filestore/libs/storage/service/CMakeLists.darwin-x86_64.txt b/cloud/filestore/libs/storage/service/CMakeLists.darwin-x86_64.txt index 9b37bfce981..7cdec8231d3 100644 --- a/cloud/filestore/libs/storage/service/CMakeLists.darwin-x86_64.txt +++ b/cloud/filestore/libs/storage/service/CMakeLists.darwin-x86_64.txt @@ -40,6 +40,7 @@ target_sources(filestore-libs-storage-service PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_reassign_tablet.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_unsafe_node_ops.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_write_compaction_map.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_alterfs.cpp diff --git a/cloud/filestore/libs/storage/service/CMakeLists.linux-aarch64.txt b/cloud/filestore/libs/storage/service/CMakeLists.linux-aarch64.txt index faab3139840..359a37bf00f 100644 --- a/cloud/filestore/libs/storage/service/CMakeLists.linux-aarch64.txt +++ b/cloud/filestore/libs/storage/service/CMakeLists.linux-aarch64.txt @@ -41,6 +41,7 @@ target_sources(filestore-libs-storage-service PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_reassign_tablet.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_unsafe_node_ops.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_write_compaction_map.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_alterfs.cpp diff --git a/cloud/filestore/libs/storage/service/CMakeLists.linux-x86_64.txt b/cloud/filestore/libs/storage/service/CMakeLists.linux-x86_64.txt index b563f2fde2b..2d175587201 100644 --- a/cloud/filestore/libs/storage/service/CMakeLists.linux-x86_64.txt +++ b/cloud/filestore/libs/storage/service/CMakeLists.linux-x86_64.txt @@ -41,6 +41,7 @@ target_sources(filestore-libs-storage-service PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_reassign_tablet.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_unsafe_node_ops.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_write_compaction_map.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_alterfs.cpp diff --git a/cloud/filestore/libs/storage/service/CMakeLists.windows-x86_64.txt b/cloud/filestore/libs/storage/service/CMakeLists.windows-x86_64.txt index 9b37bfce981..7cdec8231d3 100644 --- a/cloud/filestore/libs/storage/service/CMakeLists.windows-x86_64.txt +++ b/cloud/filestore/libs/storage/service/CMakeLists.windows-x86_64.txt @@ -40,6 +40,7 @@ target_sources(filestore-libs-storage-service PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_reassign_tablet.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_unsafe_node_ops.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions_write_compaction_map.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_actions.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_alterfs.cpp diff --git a/cloud/filestore/libs/storage/service/service_actor.h b/cloud/filestore/libs/storage/service/service_actor.h index aea9e4e8b0e..03ebce5aa86 100644 --- a/cloud/filestore/libs/storage/service/service_actor.h +++ b/cloud/filestore/libs/storage/service/service_actor.h @@ -189,6 +189,18 @@ class TStorageServiceActor final TRequestInfoPtr requestInfo, TString input); + NActors::IActorPtr CreateUnsafeDeleteNodeActionActor( + TRequestInfoPtr requestInfo, + TString input); + + NActors::IActorPtr CreateUnsafeUpdateNodeActionActor( + TRequestInfoPtr requestInfo, + TString input); + + NActors::IActorPtr CreateUnsafeGetNodeActionActor( + TRequestInfoPtr requestInfo, + TString input); + private: void RenderSessions(IOutputStream& out); void RenderLocalFileStores(IOutputStream& out); diff --git a/cloud/filestore/libs/storage/service/service_actor_actions.cpp b/cloud/filestore/libs/storage/service/service_actor_actions.cpp index 990dafb6a71..806be7146da 100644 --- a/cloud/filestore/libs/storage/service/service_actor_actions.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_actions.cpp @@ -72,6 +72,18 @@ void TStorageServiceActor::HandleExecuteAction( "writecompactionmap", &TStorageServiceActor::CreateWriteCompactionMapActionActor }, + { + "unsafedeletenode", + &TStorageServiceActor::CreateUnsafeDeleteNodeActionActor + }, + { + "unsafeupdatenode", + &TStorageServiceActor::CreateUnsafeUpdateNodeActionActor + }, + { + "unsafegetnode", + &TStorageServiceActor::CreateUnsafeGetNodeActionActor + }, }; auto it = actions.find(action); diff --git a/cloud/filestore/libs/storage/service/service_actor_actions_unsafe_node_ops.cpp b/cloud/filestore/libs/storage/service/service_actor_actions_unsafe_node_ops.cpp new file mode 100644 index 00000000000..c49e7ee5f98 --- /dev/null +++ b/cloud/filestore/libs/storage/service/service_actor_actions_unsafe_node_ops.cpp @@ -0,0 +1,164 @@ +#include "service_actor.h" + +#include +#include +#include +#include +#include + +#include + +#include + +namespace NCloud::NFileStore::NStorage { + +using namespace NActors; + +using namespace NKikimr; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +template +class TTabletActionActor final + : public TActorBootstrapped> +{ +private: + const TRequestInfoPtr RequestInfo; + const TString Input; + + using TBase = TActorBootstrapped>; + +public: + TTabletActionActor( + TRequestInfoPtr requestInfo, + TString input); + + void Bootstrap(const TActorContext& ctx); + +private: + void ReplyAndDie( + const TActorContext& ctx, + const TResponse::ProtoRecordType& responseRecord); + +private: + STFUNC(StateWork) + { + switch (ev->GetTypeRewrite()) { + HFunc(TResponse, HandleResponse); + + default: + HandleUnexpectedEvent(ev, TFileStoreComponents::SERVICE); + break; + } + } + + void HandleResponse(const TResponse::TPtr& ev, const TActorContext& ctx); +}; + +//////////////////////////////////////////////////////////////////////////////// + +template +TTabletActionActor::TTabletActionActor( + TRequestInfoPtr requestInfo, + TString input) + : RequestInfo(std::move(requestInfo)) + , Input(std::move(input)) +{} + +template +void TTabletActionActor::Bootstrap( + const TActorContext& ctx) +{ + typename TRequest::ProtoRecordType request; + if (!google::protobuf::util::JsonStringToMessage(Input, &request).ok()) { + ReplyAndDie( + ctx, + TErrorResponse(E_ARGUMENT, "Failed to parse input")); + return; + } + + if (!request.GetFileSystemId()) { + ReplyAndDie( + ctx, + TErrorResponse(E_ARGUMENT, "FileSystem id should be supplied")); + return; + } + + auto requestToTablet = std::make_unique(); + requestToTablet->Record = std::move(request); + + NCloud::Send( + ctx, + MakeIndexTabletProxyServiceId(), + std::move(requestToTablet)); + + TBase::Become(&TTabletActionActor::StateWork); +} + +template +void TTabletActionActor::ReplyAndDie( + const TActorContext& ctx, + const TResponse::ProtoRecordType& response) +{ + auto msg = std::make_unique( + response.GetError()); + + google::protobuf::util::MessageToJsonString( + response, + msg->Record.MutableOutput()); + + NCloud::Reply(ctx, *RequestInfo, std::move(msg)); + TBase::Die(ctx); +} + +//////////////////////////////////////////////////////////////////////////////// + +template +void TTabletActionActor::HandleResponse( + const TResponse::TPtr& ev, + const TActorContext& ctx) +{ + ReplyAndDie(ctx, ev->Get()->Record); +} + +} // namespace + +IActorPtr TStorageServiceActor::CreateUnsafeDeleteNodeActionActor( + TRequestInfoPtr requestInfo, + TString input) +{ + using TUnsafeDeleteNodeActor = TTabletActionActor< + TEvIndexTablet::TEvUnsafeDeleteNodeRequest, + TEvIndexTablet::TEvUnsafeDeleteNodeResponse>; + return std::make_unique( + std::move(requestInfo), + std::move(input)); +} + +IActorPtr TStorageServiceActor::CreateUnsafeUpdateNodeActionActor( + TRequestInfoPtr requestInfo, + TString input) +{ + using TUnsafeUpdateNodeActor = TTabletActionActor< + TEvIndexTablet::TEvUnsafeUpdateNodeRequest, + TEvIndexTablet::TEvUnsafeUpdateNodeResponse>; + return std::make_unique( + std::move(requestInfo), + std::move(input)); +} + +IActorPtr TStorageServiceActor::CreateUnsafeGetNodeActionActor( + TRequestInfoPtr requestInfo, + TString input) +{ + using TUnsafeGetNodeActor = TTabletActionActor< + TEvIndexTablet::TEvUnsafeGetNodeRequest, + TEvIndexTablet::TEvUnsafeGetNodeResponse>; + return std::make_unique( + std::move(requestInfo), + std::move(input)); +} + +} // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/service/service_actor_actions_ut.cpp b/cloud/filestore/libs/storage/service/service_actor_actions_ut.cpp index e99345d2f49..3d15bf88c1a 100644 --- a/cloud/filestore/libs/storage/service/service_actor_actions_ut.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_actions_ut.cpp @@ -4,6 +4,9 @@ #include #include #include +#include + +#include namespace NCloud::NFileStore::NStorage { @@ -81,7 +84,8 @@ Y_UNIT_TEST_SUITE(TStorageServiceActionsTest) ui32 nodeIdx = env.CreateNode("nfs"); TServiceClient service(env.GetRuntime(), nodeIdx); - auto response = service.AssertExecuteActionFailed("NonExistingAction", "{}"); + auto response = + service.AssertExecuteActionFailed("NonExistingAction", "{}"); UNIT_ASSERT_VALUES_UNEQUAL(S_OK, response->GetStatus()); } @@ -177,6 +181,117 @@ Y_UNIT_TEST_SUITE(TStorageServiceActionsTest) true); } } + + Y_UNIT_TEST(ShouldPerformUnsafeNodeManipulations) + { + NProto::TStorageConfig config; + TTestEnv env{{}, config}; + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + + TServiceClient service(env.GetRuntime(), nodeIdx); + + const TString fsId = "test"; + service.CreateFileStore(fsId, 1'000); + + auto headers = service.InitSession("test", "client"); + + service.CreateNode(headers, TCreateNodeArgs::File(RootNodeId, "file1")); + service.CreateNode(headers, TCreateNodeArgs::File(RootNodeId, "file2")); + + ui64 id1 = 0; + ui64 id2 = 0; + ui64 id3 = 0; + + { + auto r = service.ListNodes(headers, fsId, RootNodeId)->Record; + UNIT_ASSERT_VALUES_EQUAL(2, r.NodesSize()); + UNIT_ASSERT_VALUES_EQUAL(0, r.GetNodes(0).GetSize()); + UNIT_ASSERT_VALUES_EQUAL(0, r.GetNodes(1).GetSize()); + + id1 = r.GetNodes(0).GetId(); + id2 = r.GetNodes(1).GetId(); + + UNIT_ASSERT_VALUES_UNEQUAL(0, id1); + UNIT_ASSERT_VALUES_UNEQUAL(0, id2); + } + + id3 = Max(id1, id2) + 1; + + { + NProtoPrivate::TUnsafeUpdateNodeRequest request; + request.SetFileSystemId(fsId); + auto* node = request.MutableNode(); + node->SetId(id3); + node->SetSize(333); + TString buf; + google::protobuf::util::MessageToJsonString(request, &buf); + service.ExecuteAction("UnsafeUpdateNode", buf); + } + + { + NProtoPrivate::TUnsafeGetNodeRequest request; + request.SetFileSystemId(fsId); + request.SetId(id3); + TString buf; + google::protobuf::util::MessageToJsonString(request, &buf); + auto r = service.ExecuteAction("UnsafeGetNode", buf)->Record; + + NProtoPrivate::TUnsafeGetNodeResponse response; + UNIT_ASSERT(google::protobuf::util::JsonStringToMessage( + r.GetOutput(), + &response).ok()); + + UNIT_ASSERT_VALUES_EQUAL(id3, response.GetNode().GetId()); + UNIT_ASSERT_VALUES_EQUAL(333, response.GetNode().GetSize()); + } + + { + NProtoPrivate::TUnsafeUpdateNodeRequest request; + request.SetFileSystemId(fsId); + auto* node = request.MutableNode(); + node->SetId(id2); + node->SetSize(222); + TString buf; + google::protobuf::util::MessageToJsonString(request, &buf); + service.ExecuteAction("UnsafeUpdateNode", buf); + } + + { + auto r = service.ListNodes(headers, fsId, RootNodeId)->Record; + UNIT_ASSERT_VALUES_EQUAL(2, r.NodesSize()); + UNIT_ASSERT_VALUES_EQUAL(0, r.GetNodes(0).GetSize()); + UNIT_ASSERT_VALUES_EQUAL(222, r.GetNodes(1).GetSize()); + } + + { + NProtoPrivate::TUnsafeDeleteNodeRequest request; + request.SetFileSystemId(fsId); + request.SetId(id3); + TString buf; + google::protobuf::util::MessageToJsonString(request, &buf); + service.ExecuteAction("UnsafeDeleteNode", buf); + } + + { + NProtoPrivate::TUnsafeGetNodeRequest request; + request.SetFileSystemId(fsId); + request.SetId(id3); + TString buf; + google::protobuf::util::MessageToJsonString(request, &buf); + auto r = service.ExecuteAction("UnsafeGetNode", buf)->Record; + + NProtoPrivate::TUnsafeGetNodeResponse response; + UNIT_ASSERT(google::protobuf::util::JsonStringToMessage( + r.GetOutput(), + &response).ok()); + + UNIT_ASSERT(!response.HasNode()); + } + + service.DestroySession(headers); + } } } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/service/ya.make b/cloud/filestore/libs/storage/service/ya.make index c7a1b49d183..7984946192f 100644 --- a/cloud/filestore/libs/storage/service/ya.make +++ b/cloud/filestore/libs/storage/service/ya.make @@ -13,6 +13,7 @@ SRCS( service_actor_actions_get_storage_config_fields.cpp service_actor_actions_get_storage_config.cpp service_actor_actions_reassign_tablet.cpp + service_actor_actions_unsafe_node_ops.cpp service_actor_actions_write_compaction_map.cpp service_actor_actions.cpp service_actor_alterfs.cpp diff --git a/cloud/filestore/libs/storage/tablet/CMakeLists.darwin-x86_64.txt b/cloud/filestore/libs/storage/tablet/CMakeLists.darwin-x86_64.txt index 3096a07705a..c84bf453b10 100644 --- a/cloud/filestore/libs/storage/tablet/CMakeLists.darwin-x86_64.txt +++ b/cloud/filestore/libs/storage/tablet/CMakeLists.darwin-x86_64.txt @@ -117,6 +117,7 @@ target_sources(libs-storage-tablet PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_throttling.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_waitready.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_writebatch.cpp diff --git a/cloud/filestore/libs/storage/tablet/CMakeLists.linux-aarch64.txt b/cloud/filestore/libs/storage/tablet/CMakeLists.linux-aarch64.txt index 064d549aa8a..db6c0073329 100644 --- a/cloud/filestore/libs/storage/tablet/CMakeLists.linux-aarch64.txt +++ b/cloud/filestore/libs/storage/tablet/CMakeLists.linux-aarch64.txt @@ -118,6 +118,7 @@ target_sources(libs-storage-tablet PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_throttling.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_waitready.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_writebatch.cpp diff --git a/cloud/filestore/libs/storage/tablet/CMakeLists.linux-x86_64.txt b/cloud/filestore/libs/storage/tablet/CMakeLists.linux-x86_64.txt index 064d549aa8a..db6c0073329 100644 --- a/cloud/filestore/libs/storage/tablet/CMakeLists.linux-x86_64.txt +++ b/cloud/filestore/libs/storage/tablet/CMakeLists.linux-x86_64.txt @@ -118,6 +118,7 @@ target_sources(libs-storage-tablet PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_throttling.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_waitready.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_writebatch.cpp diff --git a/cloud/filestore/libs/storage/tablet/CMakeLists.windows-x86_64.txt b/cloud/filestore/libs/storage/tablet/CMakeLists.windows-x86_64.txt index 3096a07705a..c84bf453b10 100644 --- a/cloud/filestore/libs/storage/tablet/CMakeLists.windows-x86_64.txt +++ b/cloud/filestore/libs/storage/tablet/CMakeLists.windows-x86_64.txt @@ -117,6 +117,7 @@ target_sources(libs-storage-tablet PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_throttling.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_waitready.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/tablet_actor_writebatch.cpp diff --git a/cloud/filestore/libs/storage/tablet/helpers.cpp b/cloud/filestore/libs/storage/tablet/helpers.cpp index cf73eec0344..2f17e422d94 100644 --- a/cloud/filestore/libs/storage/tablet/helpers.cpp +++ b/cloud/filestore/libs/storage/tablet/helpers.cpp @@ -87,7 +87,10 @@ NProto::TNode CopyAttrs(const NProto::TNode& src, ui32 mode) return node; } -void ConvertNodeFromAttrs(NProto::TNodeAttr& dst, ui64 id, const NProto::TNode& src) +void ConvertNodeFromAttrs( + NProto::TNodeAttr& dst, + ui64 id, + const NProto::TNode& src) { dst.SetId(id); dst.SetType(src.GetType()); @@ -101,6 +104,19 @@ void ConvertNodeFromAttrs(NProto::TNodeAttr& dst, ui64 id, const NProto::TNode& dst.SetLinks(src.GetLinks()); } +void ConvertAttrsToNode(const NProto::TNodeAttr& src, NProto::TNode* dst) +{ + dst->SetType(src.GetType()); + dst->SetMode(src.GetMode()); + dst->SetUid(src.GetUid()); + dst->SetGid(src.GetGid()); + dst->SetATime(src.GetATime()); + dst->SetMTime(src.GetMTime()); + dst->SetCTime(src.GetCTime()); + dst->SetSize(src.GetSize()); + dst->SetLinks(src.GetLinks()); +} + //////////////////////////////////////////////////////////////////////////////// NProto::TError ValidateNodeName(const TString& name) diff --git a/cloud/filestore/libs/storage/tablet/helpers.h b/cloud/filestore/libs/storage/tablet/helpers.h index c026f2abc91..8b915c0f4f7 100644 --- a/cloud/filestore/libs/storage/tablet/helpers.h +++ b/cloud/filestore/libs/storage/tablet/helpers.h @@ -64,7 +64,8 @@ namespace NCloud::NFileStore::NStorage { //////////////////////////////////////////////////////////////////////////////// -enum ECopyAttrsMode { +enum ECopyAttrsMode +{ E_CM_CTIME = 1, // CTime E_CM_MTIME = 2, // MTime @@ -84,7 +85,12 @@ NProto::TNode CreateSocketAttrs(ui32 mode, ui32 uid, ui32 gid); NProto::TNode CopyAttrs(const NProto::TNode& src, ui32 mode = E_CM_CTIME); -void ConvertNodeFromAttrs(NProto::TNodeAttr& dst, ui64 id, const NProto::TNode& src); +void ConvertNodeFromAttrs( + NProto::TNodeAttr& dst, + ui64 id, + const NProto::TNode& src); + +void ConvertAttrsToNode(const NProto::TNodeAttr& src, NProto::TNode* dst); //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp index a148082bd22..2fcb5afa42c 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp @@ -509,10 +509,19 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const avgCleanupScore >= Config->GetCleanupThresholdAverage(); bool isPriority = false; - if (auto priorityRange = NextPriorityRangeForCleanup()) { - cleanupRangeId = priorityRange->RangeId; - cleanupScore = Max(); - isPriority = true; + TString dummy; + // if we are close to our write backpressure thresholds, it's better to + // clean up normal deletion markers first in order not to freeze write + // requests + // + // large deletion marker cleanup is a slower process and having too many + // large deletion markers affects a much smaller percentage of workloads + if (!IsCloseToBackpressureThresholds(&dummy)) { + if (auto priorityRange = NextPriorityRangeForCleanup()) { + cleanupRangeId = priorityRange->RangeId; + cleanupScore = Max(); + isPriority = true; + } } return { @@ -532,6 +541,17 @@ TCleanupInfo TIndexTabletActor::GetCleanupInfo() const || isPriority}; } +bool TIndexTabletActor::IsCloseToBackpressureThresholds(TString* message) const +{ + auto bpThresholds = BuildBackpressureThresholds(); + const double scale = + Config->GetBackpressurePercentageForFairBlobIndexOpsPriority() + / 100.; + bpThresholds.CompactionScore *= scale; + bpThresholds.CleanupScore *= scale; + return !IsWriteAllowed(bpThresholds, GetBackpressureValues(), message); +} + //////////////////////////////////////////////////////////////////////////////// void TIndexTabletActor::HandleWakeup( diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.h b/cloud/filestore/libs/storage/tablet/tablet_actor.h index ef2af6bf376..cfb0ab84f5f 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.h +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.h @@ -151,6 +151,8 @@ class TIndexTabletActor final std::atomic NodesOpenForReadingBySingleSession{0}; std::atomic NodesOpenForReadingByMultipleSessions{0}; + std::atomic OrphanNodesCount{0}; + NMetrics::TDefaultWindowCalculator MaxUsedQuota{0}; using TLatHistogram = NMetrics::THistogram; @@ -216,7 +218,8 @@ class TIndexTabletActor final const TChannelsStats& channelsStats, const TReadAheadCacheStats& readAheadStats, const TNodeIndexCacheStats& nodeIndexCacheStats, - const TNodeToSessionCounters& nodeToSessionCounters); + const TNodeToSessionCounters& nodeToSessionCounters, + const TMiscNodeStats& miscNodeStats); } Metrics; const IProfileLogPtr ProfileLog; @@ -523,6 +526,7 @@ class TIndexTabletActor final ui32 ScaleCompactionThreshold(ui32 t) const; TCompactionInfo GetCompactionInfo() const; TCleanupInfo GetCleanupInfo() const; + bool IsCloseToBackpressureThresholds(TString* message) const; void HandleWakeup( const NActors::TEvents::TEvWakeup::TPtr& ev, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_allocatedata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_allocatedata.cpp index 9e6a9b35116..93e380a7bea 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_allocatedata.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_allocatedata.cpp @@ -199,11 +199,15 @@ void TIndexTabletActor::ExecuteTx_AllocateData( if (args.CommitId == InvalidCommitId) { return RebootTabletOnCommitOverflow(ctx, "AllocateData"); } - ZeroRange( + auto e = ZeroRange( db, args.NodeId, args.CommitId, TByteRange(args.Offset, minBorder - args.Offset, GetBlockSize())); + if (HasError(e)) { + args.Error = std::move(e); + return; + } } if (!needExtend) { diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp index 2ea823b6427..617668a84cb 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp @@ -36,7 +36,8 @@ void TIndexTabletActor::HandleCleanup( return; } - auto response = std::make_unique(error); + auto response = + std::make_unique(error); NCloud::Reply(ctx, *ev, std::move(response)); }; diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp index 4f05c8288ea..8f388e7ca70 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp @@ -326,14 +326,8 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx) if (IsBlobIndexOpsQueueEmpty()) { auto blobIndexOpsPriority = Config->GetBlobIndexOpsPriority(); - auto bpThresholds = BuildBackpressureThresholds(); - const double scale = - Config->GetBackpressurePercentageForFairBlobIndexOpsPriority() - / 100.; - bpThresholds.CompactionScore *= scale; - bpThresholds.CleanupScore *= scale; TString message; - if (!IsWriteAllowed(bpThresholds, GetBackpressureValues(), &message)) { + if (IsCloseToBackpressureThresholds(&message)) { // if we are close to our backpressure thresholds, we should fall // back to fair scheduling so that all operations show some progress blobIndexOpsPriority = NProto::BIOP_FAIR; diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp index 1ebe52099f3..4db8cc72d1a 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp @@ -322,6 +322,8 @@ void TIndexTabletActor::TMetrics::Register( NodesOpenForReadingByMultipleSessions, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM(OrphanNodesCount, EMetricType::MT_ABSOLUTE); + // Throttling REGISTER_LOCAL(MaxReadBandwidth, EMetricType::MT_ABSOLUTE); REGISTER_LOCAL(MaxWriteBandwidth, EMetricType::MT_ABSOLUTE); @@ -400,7 +402,8 @@ void TIndexTabletActor::TMetrics::Update( const TChannelsStats& channelsStats, const TReadAheadCacheStats& readAheadStats, const TNodeIndexCacheStats& nodeIndexCacheStats, - const TNodeToSessionCounters& nodeToSessionCounters) + const TNodeToSessionCounters& nodeToSessionCounters, + const TMiscNodeStats& miscNodeStats) { const ui32 blockSize = fileSystem.GetBlockSize(); @@ -472,6 +475,8 @@ void TIndexTabletActor::TMetrics::Update( NodesOpenForReadingByMultipleSessions, nodeToSessionCounters.NodesOpenForReadingByMultipleSessions); + Store(OrphanNodesCount, miscNodeStats.OrphanNodesCount); + BusyIdleCalc.OnUpdateStats(); } @@ -520,7 +525,8 @@ void TIndexTabletActor::RegisterStatCounters() CalculateChannelsStats(), CalculateReadAheadCacheStats(), CalculateNodeIndexCacheStats(), - GetNodeToSessionCounters()); + GetNodeToSessionCounters(), + GetMiscNodeStats()); Metrics.Register(fsId, storageMediaKind); } @@ -566,7 +572,8 @@ void TIndexTabletActor::HandleUpdateCounters( CalculateChannelsStats(), CalculateReadAheadCacheStats(), CalculateNodeIndexCacheStats(), - GetNodeToSessionCounters()); + GetNodeToSessionCounters(), + GetMiscNodeStats()); SendMetricsToExecutor(ctx); UpdateCountersScheduled = false; diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_createhandle.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_createhandle.cpp index 12652022067..ef74c8a47b4 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_createhandle.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_createhandle.cpp @@ -309,12 +309,16 @@ void TIndexTabletActor::ExecuteTx_CreateHandle( } else if (args.FollowerId.Empty() && HasFlag(args.Flags, NProto::TCreateHandleRequest::E_TRUNCATE)) { - Truncate( + auto e = Truncate( db, args.TargetNodeId, args.WriteCommitId, args.TargetNode->Attrs.GetSize(), 0); + if (HasError(e)) { + args.Error = std::move(e); + return; + } auto attrs = CopyAttrs(args.TargetNode->Attrs, E_CM_CMTIME); attrs.SetSize(0); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_destroyhandle.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_destroyhandle.cpp index e88ea160713..a4395fca245 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_destroyhandle.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_destroyhandle.cpp @@ -104,11 +104,19 @@ void TIndexTabletActor::ExecuteTx_DestroyHandle( if (args.Node->Attrs.GetLinks() == 0 && !HasOpenHandles(args.Node->NodeId)) { - RemoveNode( + auto e = RemoveNode( db, *args.Node, args.Node->MinCommitId, commitId); + + if (HasError(e)) { + WriteOrphanNode(db, TStringBuilder() + << "DestroyHandle: " << args.SessionId + << ", Handle: " << args.Request.GetHandle() + << ", RemoveNode: " << args.Node->NodeId + << ", Error: " << FormatError(e), args.Node->NodeId); + } } EnqueueTruncateIfNeeded(ctx); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_destroysession.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_destroysession.cpp index c79608df8a2..1e46210ead7 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_destroysession.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_destroysession.cpp @@ -308,11 +308,18 @@ void TIndexTabletActor::ExecuteTx_DestroySession( auto it = args.Nodes.find(nodeId); if (it != args.Nodes.end() && !HasOpenHandles(nodeId)) { - RemoveNode( + auto e = RemoveNode( db, *it, it->MinCommitId, commitId); + + if (HasError(e)) { + WriteOrphanNode(db, TStringBuilder() + << "DestroySession: " << args.SessionId + << ", RemoveNode: " << nodeId + << ", Error: " << FormatError(e), nodeId); + } } } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_loadstate.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_loadstate.cpp index 576b5c1e6b1..4c53ec35f09 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_loadstate.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_loadstate.cpp @@ -112,6 +112,7 @@ bool TIndexTabletActor::PrepareTx_LoadState( db.ReadSessionHistoryEntries(args.SessionHistory), db.ReadOpLog(args.OpLog), db.ReadLargeDeletionMarkers(args.LargeDeletionMarkers), + db.ReadOrphanNodes(args.OrphanNodeIds), }; bool ready = std::accumulate( @@ -231,9 +232,16 @@ void TIndexTabletActor::CompleteTx_LoadState( LOG_INFO_S(ctx, TFileStoreComponents::TABLET, LogTag << " Initializing tablet state"); - LOG_INFO_S(ctx, TFileStoreComponents::TABLET, - LogTag << " Read " << args.LargeDeletionMarkers.size() - << " large deletion markers"); + if (args.LargeDeletionMarkers) { + LOG_INFO_S(ctx, TFileStoreComponents::TABLET, + LogTag << " Read " << args.LargeDeletionMarkers.size() + << " large deletion markers"); + } + if (args.OrphanNodeIds) { + LOG_INFO_S(ctx, TFileStoreComponents::TABLET, + LogTag << " Read " << args.OrphanNodeIds.size() + << " orphan nodes"); + } LoadState( Executor()->Generation(), @@ -242,6 +250,7 @@ void TIndexTabletActor::CompleteTx_LoadState( args.FileSystemStats, args.TabletStorageInfo, args.LargeDeletionMarkers, + args.OrphanNodeIds, config); UpdateLogTag(); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_renamenode.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_renamenode.cpp index b4a16465a75..451adf5f318 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_renamenode.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_renamenode.cpp @@ -322,13 +322,22 @@ void TIndexTabletActor::ExecuteTx_RenameNode( } // remove target ref and unlink target node - UnlinkNode( + auto e = UnlinkNode( db, args.NewParentNode->NodeId, args.NewName, *args.NewChildNode, args.NewChildRef->MinCommitId, args.CommitId); + + if (HasError(e)) { + const auto nodeId = args.NewChildNode->NodeId; + WriteOrphanNode(db, TStringBuilder() + << "RenameNode: " << args.SessionId + << ", ParentNodeId: " << args.NewParentNode->NodeId + << ", NodeId: " << nodeId + << ", Error: " << FormatError(e), nodeId); + } } else { // remove target ref UnlinkExternalNode( diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_resetsession.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_resetsession.cpp index 4445cde0808..21d6cf98c4d 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_resetsession.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_resetsession.cpp @@ -142,11 +142,18 @@ void TIndexTabletActor::ExecuteTx_ResetSession( nodeId, it->Attrs.GetSize()); - RemoveNode( + auto e = RemoveNode( db, *it, it->MinCommitId, commitId); + + if (HasError(e)) { + WriteOrphanNode(db, TStringBuilder() + << "DestroySession: " << args.SessionId + << ", RemoveNode: " << nodeId + << ", Error: " << FormatError(e), nodeId); + } } } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_setnodeattr.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_setnodeattr.cpp index d1b49ddbfe5..bc7db996d84 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_setnodeattr.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_setnodeattr.cpp @@ -167,13 +167,18 @@ void TIndexTabletActor::ExecuteTx_SetNodeAttr( attrs.SetCTime(update.GetCTime()); } if (HasFlag(flags, NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE)) { - Truncate( + auto e = Truncate( db, args.NodeId, args.CommitId, attrs.GetSize(), update.GetSize()); + if (HasError(e)) { + args.Error = e; + return; + } + attrs.SetSize(update.GetSize()); } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp index 7a2f116cb75..f19b44227f4 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp @@ -322,7 +322,7 @@ void TIndexTabletActor::ExecuteTx_TruncateRange( args.Range.Length, args.ProfileLogRequest); - TruncateRange(db, args.NodeId, commitId, args.Range); + args.Error = TruncateRange(db, args.NodeId, commitId, args.Range); } void TIndexTabletActor::CompleteTx_TruncateRange( @@ -334,7 +334,7 @@ void TIndexTabletActor::CompleteTx_TruncateRange( std::move(args.ProfileLogRequest), ctx.Now(), GetFileSystemId(), - {}, + args.Error, ProfileLog); LOG_DEBUG(ctx, TFileStoreComponents::TABLET, @@ -343,7 +343,9 @@ void TIndexTabletActor::CompleteTx_TruncateRange( args.NodeId, args.Range.Describe().c_str()); - auto response = std::make_unique(); + auto response = + std::make_unique( + std::move(args.Error)); NCloud::Reply(ctx, *args.RequestInfo, std::move(response)); EnqueueCollectGarbageIfNeeded(ctx); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp index a03677707ab..27ac03b21e5 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp @@ -383,13 +383,18 @@ void TIndexTabletActor::ExecuteTx_UnlinkNode( db.WriteOpLogEntry(args.OpLogEntry); } else { - UnlinkNode( + auto e = UnlinkNode( db, args.ParentNodeId, args.Name, *args.ChildNode, args.ChildRef->MinCommitId, args.CommitId); + + if (HasError(e)) { + args.Error = std::move(e); + return; + } } auto* session = FindSession(args.SessionId); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp new file mode 100644 index 00000000000..cef98ff1bdb --- /dev/null +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp @@ -0,0 +1,276 @@ +#include "tablet_actor.h" + +#include "helpers.h" + +namespace NCloud::NFileStore::NStorage { + +using namespace NActors; + +using namespace NKikimr; +using namespace NKikimr::NTabletFlatExecutor; + +//////////////////////////////////////////////////////////////////////////////// + +void TIndexTabletActor::HandleUnsafeDeleteNode( + const TEvIndexTablet::TEvUnsafeDeleteNodeRequest::TPtr& ev, + const TActorContext& ctx) +{ + auto* msg = ev->Get(); + + auto requestInfo = CreateRequestInfo( + ev->Sender, + ev->Cookie, + msg->CallContext); + + AddTransaction(*requestInfo); + + LOG_WARN(ctx, TFileStoreComponents::TABLET, + "%s UnsafeDeleteNode: %s", + LogTag.c_str(), + msg->Record.DebugString().Quote().c_str()); + + ExecuteTx( + ctx, + std::move(requestInfo), + std::move(msg->Record)); +} + +//////////////////////////////////////////////////////////////////////////////// + +bool TIndexTabletActor::PrepareTx_UnsafeDeleteNode( + const TActorContext& ctx, + TTransactionContext& tx, + TTxIndexTablet::TUnsafeDeleteNode& args) +{ + Y_UNUSED(ctx); + + auto commitId = GetCurrentCommitId(); + + TIndexTabletDatabaseProxy db(tx.DB, args.NodeUpdates); + return ReadNode(db, args.Request.GetId(), commitId, args.Node); +} + +void TIndexTabletActor::ExecuteTx_UnsafeDeleteNode( + const TActorContext& ctx, + TTransactionContext& tx, + TTxIndexTablet::TUnsafeDeleteNode& args) +{ + TIndexTabletDatabaseProxy db(tx.DB, args.NodeUpdates); + + auto commitId = GenerateCommitId(); + if (commitId == InvalidCommitId) { + return RebootTabletOnCommitOverflow(ctx, "UnsafeDeleteNode"); + } + + if (!args.Node) { + LOG_WARN(ctx, TFileStoreComponents::TABLET, + "%s UnsafeDeleteNode: %s - node not found", + LogTag.c_str(), + args.Request.DebugString().Quote().c_str()); + return; + } + + args.Error = RemoveNode(db, *args.Node, args.Node->MinCommitId, commitId); +} + +void TIndexTabletActor::CompleteTx_UnsafeDeleteNode( + const TActorContext& ctx, + TTxIndexTablet::TUnsafeDeleteNode& args) +{ + RemoveTransaction(*args.RequestInfo); + + LOG_WARN(ctx, TFileStoreComponents::TABLET, + "%s UnsafeDeleteNode: %s, status: %s", + LogTag.c_str(), + args.Request.DebugString().Quote().c_str(), + FormatError(args.Error).c_str()); + + auto response = + std::make_unique( + std::move(args.Error)); + + NCloud::Reply(ctx, *args.RequestInfo, std::move(response)); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TIndexTabletActor::HandleUnsafeUpdateNode( + const TEvIndexTablet::TEvUnsafeUpdateNodeRequest::TPtr& ev, + const TActorContext& ctx) +{ + auto* msg = ev->Get(); + + auto requestInfo = CreateRequestInfo( + ev->Sender, + ev->Cookie, + msg->CallContext); + + AddTransaction(*requestInfo); + + LOG_WARN(ctx, TFileStoreComponents::TABLET, + "%s UnsafeUpdateNode: %s", + LogTag.c_str(), + msg->Record.DebugString().Quote().c_str()); + + ExecuteTx( + ctx, + std::move(requestInfo), + std::move(msg->Record)); +} + +//////////////////////////////////////////////////////////////////////////////// + +bool TIndexTabletActor::PrepareTx_UnsafeUpdateNode( + const TActorContext& ctx, + TTransactionContext& tx, + TTxIndexTablet::TUnsafeUpdateNode& args) +{ + Y_UNUSED(ctx); + + auto commitId = GetCurrentCommitId(); + + TIndexTabletDatabaseProxy db(tx.DB, args.NodeUpdates); + return ReadNode(db, args.Request.GetNode().GetId(), commitId, args.Node); +} + +void TIndexTabletActor::ExecuteTx_UnsafeUpdateNode( + const TActorContext& ctx, + TTransactionContext& tx, + TTxIndexTablet::TUnsafeUpdateNode& args) +{ + TIndexTabletDatabaseProxy db(tx.DB, args.NodeUpdates); + + auto commitId = GenerateCommitId(); + if (commitId == InvalidCommitId) { + return RebootTabletOnCommitOverflow(ctx, "UnsafeUpdateNode"); + } + + NProto::TNode prevNode; + NProto::TNode node; + ui64 nodeCommitId = commitId; + + if (args.Node) { + prevNode = args.Node->Attrs; + node = args.Node->Attrs; + nodeCommitId = args.Node->MinCommitId; + } else { + LOG_WARN(ctx, TFileStoreComponents::TABLET, + "%s UnsafeUpdateNode: %s - node not found, creating", + LogTag.c_str(), + args.Request.DebugString().Quote().c_str()); + + // creation here is still done via UpdateNode since CreateNode generates + // a new nodeId + } + + ConvertAttrsToNode(args.Request.GetNode(), &node); + UpdateNode( + db, + args.Request.GetNode().GetId(), + nodeCommitId, + commitId, + node, + prevNode); +} + +void TIndexTabletActor::CompleteTx_UnsafeUpdateNode( + const TActorContext& ctx, + TTxIndexTablet::TUnsafeUpdateNode& args) +{ + RemoveTransaction(*args.RequestInfo); + + TString oldNode; + if (args.Node) { + oldNode = args.Node->Attrs.Utf8DebugString(); + } + LOG_WARN(ctx, TFileStoreComponents::TABLET, + "%s UnsafeUpdateNode: %s, old node: %s", + LogTag.c_str(), + args.Request.DebugString().Quote().c_str(), + oldNode.Quote().c_str()); + + auto response = + std::make_unique(); + + NCloud::Reply(ctx, *args.RequestInfo, std::move(response)); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TIndexTabletActor::HandleUnsafeGetNode( + const TEvIndexTablet::TEvUnsafeGetNodeRequest::TPtr& ev, + const TActorContext& ctx) +{ + auto* msg = ev->Get(); + + auto requestInfo = CreateRequestInfo( + ev->Sender, + ev->Cookie, + msg->CallContext); + + AddTransaction(*requestInfo); + + LOG_WARN(ctx, TFileStoreComponents::TABLET, + "%s UnsafeGetNode: %s", + LogTag.c_str(), + msg->Record.DebugString().Quote().c_str()); + + ExecuteTx( + ctx, + std::move(requestInfo), + std::move(msg->Record)); +} + +//////////////////////////////////////////////////////////////////////////////// + +bool TIndexTabletActor::ValidateTx_UnsafeGetNode( + const TActorContext& ctx, + TTxIndexTablet::TUnsafeGetNode& args) +{ + Y_UNUSED(ctx); + Y_UNUSED(args); + + return true; +} + +bool TIndexTabletActor::PrepareTx_UnsafeGetNode( + const NActors::TActorContext& ctx, + IIndexTabletDatabase& db, + TTxIndexTablet::TUnsafeGetNode& args) +{ + Y_UNUSED(ctx); + + return ReadNode( + db, + args.Request.GetId(), + GetCurrentCommitId(), + args.Node); +} + +void TIndexTabletActor::CompleteTx_UnsafeGetNode( + const TActorContext& ctx, + TTxIndexTablet::TUnsafeGetNode& args) +{ + RemoveTransaction(*args.RequestInfo); + + TString node; + if (args.Node) { + node = args.Node->Attrs.Utf8DebugString(); + } + LOG_WARN(ctx, TFileStoreComponents::TABLET, + "%s UnsafeGetNode: %s, node: %s", + LogTag.c_str(), + args.Request.DebugString().Quote().c_str(), + node.Quote().c_str()); + + auto response = + std::make_unique(); + if (args.Node) { + auto& attrs = *response->Record.MutableNode(); + ConvertNodeFromAttrs(attrs, args.Node->NodeId, args.Node->Attrs); + } + + NCloud::Reply(ctx, *args.RequestInfo, std::move(response)); +} + +} // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_zerorange.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_zerorange.cpp index f0a6baf1c87..6574e32caf2 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_zerorange.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_zerorange.cpp @@ -69,8 +69,6 @@ void TIndexTabletActor::ExecuteTx_ZeroRange( TTransactionContext& tx, TTxIndexTablet::TZeroRange& args) { - Y_UNUSED(ctx); - TIndexTabletDatabase db(tx.DB); ui64 commitId = GenerateCommitId(); @@ -84,7 +82,7 @@ void TIndexTabletActor::ExecuteTx_ZeroRange( args.Range.Length, args.ProfileLogRequest); - ZeroRange(db, args.NodeId, commitId, args.Range); + args.Error = ZeroRange(db, args.NodeId, commitId, args.Range); } void TIndexTabletActor::CompleteTx_ZeroRange( @@ -96,16 +94,19 @@ void TIndexTabletActor::CompleteTx_ZeroRange( std::move(args.ProfileLogRequest), ctx.Now(), GetFileSystemId(), - {}, + args.Error, ProfileLog); LOG_DEBUG(ctx, TFileStoreComponents::TABLET, - "%s ZeroRange %lu %s completed", + "%s ZeroRange %lu %s completed: %s", LogTag.c_str(), args.NodeId, - args.Range.Describe().c_str()); + args.Range.Describe().c_str(), + FormatError(args.Error).Quote().c_str()); - auto response = std::make_unique(); + auto response = + std::make_unique( + std::move(args.Error)); NCloud::Reply(ctx, *args.RequestInfo, std::move(response)); EnqueueCollectGarbageIfNeeded(ctx); diff --git a/cloud/filestore/libs/storage/tablet/tablet_database.cpp b/cloud/filestore/libs/storage/tablet/tablet_database.cpp index 27bcece8ec2..cc5f9318869 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_database.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_database.cpp @@ -1404,6 +1404,49 @@ bool TIndexTabletDatabase::ReadLargeDeletionMarkers( return true; } +//////////////////////////////////////////////////////////////////////////////// +// OrphanNodes + +void TIndexTabletDatabase::WriteOrphanNode(ui64 nodeId) +{ + using TTable = TIndexTabletSchema::OrphanNodes; + + Table() + .Key(nodeId) + .Update(); +} + +void TIndexTabletDatabase::DeleteOrphanNode(ui64 nodeId) +{ + using TTable = TIndexTabletSchema::OrphanNodes; + + Table() + .Key(nodeId) + .Delete(); +} + +bool TIndexTabletDatabase::ReadOrphanNodes(TVector& nodeIds) +{ + using TTable = TIndexTabletSchema::OrphanNodes; + + auto it = Table() + .Select(); + + if (!it.IsReady()) { + return false; // not ready + } + + while (it.IsValid()) { + nodeIds.emplace_back(it.GetValue()); + + if (!it.Next()) { + return false; // not ready + } + } + + return true; +} + //////////////////////////////////////////////////////////////////////////////// // NewBlobs diff --git a/cloud/filestore/libs/storage/tablet/tablet_database.h b/cloud/filestore/libs/storage/tablet/tablet_database.h index 6a99be794f5..7a841690a13 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_database.h +++ b/cloud/filestore/libs/storage/tablet/tablet_database.h @@ -427,6 +427,14 @@ FILESTORE_FILESYSTEM_STATS(FILESTORE_DECLARE_STATS) bool ReadLargeDeletionMarkers(TVector& deletionMarkers); + // + // OrphanNodes + // + + void WriteOrphanNode(ui64 nodeId); + void DeleteOrphanNode(ui64 nodeId); + bool ReadOrphanNodes(TVector& nodeIds); + // // NewBlobs // diff --git a/cloud/filestore/libs/storage/tablet/tablet_database_ut.cpp b/cloud/filestore/libs/storage/tablet/tablet_database_ut.cpp index 66f064c7047..5020a24f845 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_database_ut.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_database_ut.cpp @@ -511,6 +511,41 @@ Y_UNIT_TEST_SUITE(TIndexTabletDatabaseTest) UNIT_ASSERT_VALUES_EQUAL(toString(entries), toString(markers)); }); } + + Y_UNIT_TEST(ShouldStoreOrphanNodes) + { + TTestExecutor executor; + executor.WriteTx([&] (TIndexTabletDatabase db) { + db.InitSchema(false); + }); + + executor.WriteTx([&] (TIndexTabletDatabase db) { + db.WriteOrphanNode(111); + db.WriteOrphanNode(222); + db.WriteOrphanNode(333); + }); + + executor.ReadTx([&] (TIndexTabletDatabase db) { + TVector nodeIds; + UNIT_ASSERT(db.ReadOrphanNodes(nodeIds)); + UNIT_ASSERT_VALUES_EQUAL(3, nodeIds.size()); + UNIT_ASSERT_VALUES_EQUAL(111, nodeIds[0]); + UNIT_ASSERT_VALUES_EQUAL(222, nodeIds[1]); + UNIT_ASSERT_VALUES_EQUAL(333, nodeIds[2]); + }); + + executor.WriteTx([&] (TIndexTabletDatabase db) { + db.DeleteOrphanNode(222); + }); + + executor.ReadTx([&] (TIndexTabletDatabase db) { + TVector nodeIds; + UNIT_ASSERT(db.ReadOrphanNodes(nodeIds)); + UNIT_ASSERT_VALUES_EQUAL(2, nodeIds.size()); + UNIT_ASSERT_VALUES_EQUAL(111, nodeIds[0]); + UNIT_ASSERT_VALUES_EQUAL(333, nodeIds[1]); + }); + } } } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/tablet_schema.h b/cloud/filestore/libs/storage/tablet/tablet_schema.h index 6cec54933d2..5b997fcd051 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_schema.h +++ b/cloud/filestore/libs/storage/tablet/tablet_schema.h @@ -533,6 +533,19 @@ struct TIndexTabletSchema using CompactionPolicy = TCompactionPolicy; }; + struct OrphanNodes: TTableSchema<27> + { + struct NodeId : Column<1, NKikimr::NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey; + + using TColumns = TableColumns< + NodeId + >; + + using StoragePolicy = TStoragePolicy; + }; + using TTables = SchemaTables< FileSystem, Sessions, @@ -559,7 +572,8 @@ struct TIndexTabletSchema TruncateQueue, SessionHistory, OpLog, - LargeDeletionMarkers + LargeDeletionMarkers, + OrphanNodes >; using TSettings = SchemaSettings< diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.cpp b/cloud/filestore/libs/storage/tablet/tablet_state.cpp index bda81044bcf..bedcc447452 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_state.cpp @@ -83,6 +83,7 @@ void TIndexTabletState::LoadState( const NProto::TFileSystemStats& fileSystemStats, const NCloud::NProto::TTabletStorageInfo& tabletStorageInfo, const TVector& largeDeletionMarkers, + const TVector& orphanNodeIds, const TThrottlerConfig& throttlerConfig) { Generation = generation; @@ -102,6 +103,8 @@ void TIndexTabletState::LoadState( LargeDeletionMarkersThreshold = config.GetLargeDeletionMarkersThreshold(); LargeDeletionMarkersCleanupThreshold = config.GetLargeDeletionMarkersCleanupThreshold(); + LargeDeletionMarkersThresholdForBackpressure = + config.GetLargeDeletionMarkersThresholdForBackpressure(); FileSystem.CopyFrom(fileSystem); FileSystemStats.CopyFrom(fileSystemStats); @@ -130,6 +133,8 @@ void TIndexTabletState::LoadState( for (const auto& deletionMarker: largeDeletionMarkers) { Impl->LargeBlocks.AddDeletionMarker(deletionMarker); } + + Impl->OrphanNodeIds.insert(orphanNodeIds.begin(), orphanNodeIds.end()); } void TIndexTabletState::UpdateConfig( @@ -170,4 +175,11 @@ void TIndexTabletState::DumpStats(IOutputStream& os) const ); } +TMiscNodeStats TIndexTabletState::GetMiscNodeStats() const +{ + return { + .OrphanNodesCount = static_cast(Impl->OrphanNodeIds.size()), + }; +} + } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.h b/cloud/filestore/libs/storage/tablet/tablet_state.h index 4adf124f2cd..286295e45f2 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state.h +++ b/cloud/filestore/libs/storage/tablet/tablet_state.h @@ -143,6 +143,11 @@ struct TNodeToSessionCounters i64 NodesOpenForReadingByMultipleSessions{0}; }; +struct TMiscNodeStats +{ + i64 OrphanNodesCount{0}; +}; + //////////////////////////////////////////////////////////////////////////////// class TIndexTabletState @@ -171,6 +176,7 @@ class TIndexTabletState /*const*/ ui64 LargeDeletionMarkerBlocks = 0; /*const*/ ui64 LargeDeletionMarkersThreshold = 0; /*const*/ ui64 LargeDeletionMarkersCleanupThreshold = 0; + /*const*/ ui64 LargeDeletionMarkersThresholdForBackpressure = 0; bool StateLoaded = false; @@ -190,6 +196,7 @@ class TIndexTabletState const NProto::TFileSystemStats& fileSystemStats, const NCloud::NProto::TTabletStorageInfo& tabletStorageInfo, const TVector& largeDeletionMarkers, + const TVector& orphanNodeIds, const TThrottlerConfig& throttlerConfig); bool IsStateLoaded() const @@ -275,6 +282,8 @@ class TIndexTabletState return NodeToSessionCounters; } + TMiscNodeStats GetMiscNodeStats() const; + const NProto::TFileStorePerformanceProfile& GetPerformanceProfile() const; const TFileStoreAllocRegistry& GetFileStoreProfilingRegistry() const @@ -380,13 +389,13 @@ FILESTORE_FILESYSTEM_STATS(FILESTORE_DECLARE_COUNTER) const NProto::TNode& attrs, const NProto::TNode& prevAttrs); - void RemoveNode( + [[nodiscard]] NProto::TError RemoveNode( TIndexTabletDatabase& db, const IIndexTabletDatabase::TNode& node, ui64 minCommitId, ui64 maxCommitId); - void UnlinkNode( + [[nodiscard]] NProto::TError UnlinkNode( TIndexTabletDatabase& db, ui64 parentNodeId, const TString& name, @@ -423,6 +432,11 @@ FILESTORE_FILESYSTEM_STATS(FILESTORE_DECLARE_COUNTER) bool HasBlocksLeft( ui32 blocks) const; + void WriteOrphanNode( + TIndexTabletDatabase& db, + const TString& message, + ui64 nodeId); + private: void UpdateUsedBlocksCount( TIndexTabletDatabase& db, @@ -1207,7 +1221,7 @@ FILESTORE_DUPCACHE_REQUESTS(FILESTORE_DECLARE_DUPCACHE) void AddTruncate(TIndexTabletDatabase& db, ui64 nodeId, TByteRange range); void DeleteTruncate(TIndexTabletDatabase& db, ui64 nodeId); - void Truncate( + [[nodiscard]] NProto::TError Truncate( TIndexTabletDatabase& db, ui64 nodeId, ui64 commitId, @@ -1220,7 +1234,7 @@ FILESTORE_DUPCACHE_REQUESTS(FILESTORE_DECLARE_DUPCACHE) // - aligns up range in the tail; // - deletes all blocks in NEW range; // - writes fresh bytes (zeroes) on unaligned head, if range.Offset != 0. - void TruncateRange( + [[nodiscard]] NProto::TError TruncateRange( TIndexTabletDatabase& db, ui64 nodeId, ui64 commitId, @@ -1230,14 +1244,14 @@ FILESTORE_DUPCACHE_REQUESTS(FILESTORE_DECLARE_DUPCACHE) // resizing the node. This function: // - writes fresh bytes (zeroes) on unaligned head, if any; // - writes fresh bytes (zeroes) on unaligned tail, if any. - void ZeroRange( + [[nodiscard]] NProto::TError ZeroRange( TIndexTabletDatabase& db, ui64 nodeId, ui64 commitId, TByteRange range); private: - void DeleteRange( + [[nodiscard]] NProto::TError DeleteRange( TIndexTabletDatabase& db, ui64 nodeId, ui64 commitId, diff --git a/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp b/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp index ef9728d3bd5..7d0f08439de 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp @@ -74,7 +74,7 @@ bool TIndexTabletState::GenerateBlobId( return true; } -void TIndexTabletState::Truncate( +NProto::TError TIndexTabletState::Truncate( TIndexTabletDatabase& db, ui64 nodeId, ui64 commitId, @@ -82,20 +82,20 @@ void TIndexTabletState::Truncate( ui64 targetSize) { if (currentSize <= targetSize) { - return; + return {}; } TByteRange range(targetSize, currentSize - targetSize, GetBlockSize()); if (TruncateBlocksThreshold && range.BlockCount() > TruncateBlocksThreshold) { EnqueueTruncateOp(nodeId, range); - return; + return {}; } - TruncateRange(db, nodeId, commitId, range); + return TruncateRange(db, nodeId, commitId, range); } -void TIndexTabletState::TruncateRange( +NProto::TError TIndexTabletState::TruncateRange( TIndexTabletDatabase& db, ui64 nodeId, ui64 commitId, @@ -106,7 +106,10 @@ void TIndexTabletState::TruncateRange( AlignUp(range.End(), range.BlockSize) - range.Offset, range.BlockSize); - DeleteRange(db, nodeId, commitId, tailAlignedRange); + auto e = DeleteRange(db, nodeId, commitId, tailAlignedRange); + if (HasError(e)) { + return e; + } const TByteRange headBound( range.Offset, @@ -123,15 +126,20 @@ void TIndexTabletState::TruncateRange( } InvalidateReadAheadCache(nodeId); + + return {}; } -void TIndexTabletState::ZeroRange( +NProto::TError TIndexTabletState::ZeroRange( TIndexTabletDatabase& db, ui64 nodeId, ui64 commitId, TByteRange range) { - DeleteRange(db, nodeId, commitId, range); + auto e = DeleteRange(db, nodeId, commitId, range); + if (HasError(e)) { + return e; + } const TByteRange headBound( range.Offset, @@ -160,9 +168,11 @@ void TIndexTabletState::ZeroRange( // FIXME: do not allocate each time TString(tailBound.Length, 0)); } + + return {}; } -void TIndexTabletState::DeleteRange( +NProto::TError TIndexTabletState::DeleteRange( TIndexTabletDatabase& db, ui64 nodeId, ui64 commitId, @@ -170,16 +180,16 @@ void TIndexTabletState::DeleteRange( { const ui64 deletedBlockCount = range.AlignedBlockCount(); if (deletedBlockCount) { - MarkFreshBlocksDeleted( - db, - nodeId, - commitId, - range.FirstAlignedBlock(), - deletedBlockCount); - const bool useLargeDeletionMarkers = LargeDeletionMarkersEnabled && deletedBlockCount >= LargeDeletionMarkersThreshold; if (useLargeDeletionMarkers) { + const auto t = LargeDeletionMarkersThresholdForBackpressure; + if (GetLargeDeletionMarkersCount() >= t) { + return MakeError(E_REJECTED, TStringBuilder() + << "too many large deletion markers: " + << GetLargeDeletionMarkersCount() << " >= " << t); + } + SplitRange( range.FirstAlignedBlock(), deletedBlockCount, @@ -213,6 +223,13 @@ void TIndexTabletState::DeleteRange( blocksCount); }); } + + MarkFreshBlocksDeleted( + db, + nodeId, + commitId, + range.FirstAlignedBlock(), + deletedBlockCount); } WriteFreshBytesDeletionMarker( @@ -221,6 +238,8 @@ void TIndexTabletState::DeleteRange( commitId, range.Offset, range.Length); + + return {}; } void TIndexTabletState::EnqueueTruncateOp(ui64 nodeId, TByteRange range) diff --git a/cloud/filestore/libs/storage/tablet/tablet_state_impl.h b/cloud/filestore/libs/storage/tablet/tablet_state_impl.h index bfdf07c22ab..e4847015859 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state_impl.h +++ b/cloud/filestore/libs/storage/tablet/tablet_state_impl.h @@ -59,6 +59,7 @@ struct TIndexTabletState::TImpl TReadAheadCache ReadAheadCache; TNodeIndexCache NodeIndexCache; TInMemoryIndexState InMemoryIndexState; + TSet OrphanNodeIds; TCheckpointStore Checkpoints; TChannels Channels; diff --git a/cloud/filestore/libs/storage/tablet/tablet_state_nodes.cpp b/cloud/filestore/libs/storage/tablet/tablet_state_nodes.cpp index 331d739b53e..10bc8b89276 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state_nodes.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_state_nodes.cpp @@ -98,12 +98,27 @@ void TIndexTabletState::UpdateNode( InvalidateNodeIndexCache(nodeId); } -void TIndexTabletState::RemoveNode( +NProto::TError TIndexTabletState::RemoveNode( TIndexTabletDatabase& db, const IIndexTabletDatabase::TNode& node, ui64 minCommitId, ui64 maxCommitId) { + // SymLinks have size (equal to TargetPath) but store no real data so there + // is no need to write deletion markers upon SymLink removal + if (!node.Attrs.GetSymLink()) { + auto e = Truncate( + db, + node.NodeId, + maxCommitId, + node.Attrs.GetSize(), + 0); + + if (HasError(e)) { + return e; + } + } + db.DeleteNode(node.NodeId); DecrementUsedNodesCount(db); @@ -116,21 +131,12 @@ void TIndexTabletState::RemoveNode( AddCheckpointNode(db, checkpointId, node.NodeId); } - // SymLinks have size (equal to TargetPath) but store no real data so there - // is no need to write deletion markers upon SymLink removal - if (!node.Attrs.GetSymLink()) { - Truncate( - db, - node.NodeId, - maxCommitId, - node.Attrs.GetSize(), - 0); - } - InvalidateNodeIndexCache(node.NodeId); + + return {}; } -void TIndexTabletState::UnlinkNode( +NProto::TError TIndexTabletState::UnlinkNode( TIndexTabletDatabase& db, ui64 parentNodeId, const TString& name, @@ -148,11 +154,15 @@ void TIndexTabletState::UnlinkNode( attrs, node.Attrs); } else { - RemoveNode( + auto e = RemoveNode( db, node, minCommitId, maxCommitId); + + if (HasError(e)) { + return e; + } } RemoveNodeRef( @@ -165,6 +175,8 @@ void TIndexTabletState::UnlinkNode( "", // followerId "" // followerName ); + + return {}; } void TIndexTabletState::UnlinkExternalNode( @@ -232,6 +244,16 @@ void TIndexTabletState::RewriteNode( InvalidateNodeIndexCache(nodeId); } +void TIndexTabletState::WriteOrphanNode( + TIndexTabletDatabase& db, + const TString& message, + ui64 nodeId) +{ + ReportGeneratedOrphanNode(message); + db.WriteOrphanNode(nodeId); + Impl->OrphanNodeIds.insert(nodeId); +} + //////////////////////////////////////////////////////////////////////////////// // NodeAttrs diff --git a/cloud/filestore/libs/storage/tablet/tablet_tx.h b/cloud/filestore/libs/storage/tablet/tablet_tx.h index 240cd262f64..ccc3f80b320 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_tx.h +++ b/cloud/filestore/libs/storage/tablet/tablet_tx.h @@ -75,6 +75,8 @@ namespace NCloud::NFileStore::NStorage { xxx(GetNodeAttrBatch, __VA_ARGS__) \ xxx(GetNodeXAttr, __VA_ARGS__) \ xxx(ListNodeXAttr, __VA_ARGS__) \ + \ + xxx(UnsafeGetNode, __VA_ARGS__) \ // FILESTORE_TABLET_RO_TRANSACTIONS #define FILESTORE_TABLET_RW_TRANSACTIONS(xxx, ...) \ @@ -131,6 +133,9 @@ namespace NCloud::NFileStore::NStorage { \ xxx(DeleteOpLogEntry, __VA_ARGS__) \ xxx(CommitNodeCreationInFollower, __VA_ARGS__) \ + \ + xxx(UnsafeDeleteNode, __VA_ARGS__) \ + xxx(UnsafeUpdateNode, __VA_ARGS__) \ // FILESTORE_TABLET_RW_TRANSACTIONS #define FILESTORE_TABLET_TRANSACTIONS(xxx, ...) \ @@ -170,7 +175,8 @@ struct TIndexStateNodeUpdates TVector NodeUpdates; }; -struct TProfileAware { +struct TProfileAware +{ NProto::TProfileLogRequestInfo ProfileLogRequest; explicit TProfileAware(EFileStoreSystemRequest requestType) noexcept @@ -308,6 +314,7 @@ struct TTxIndexTablet TVector SessionHistory; TVector OpLog; TVector LargeDeletionMarkers; + TVector OrphanNodeIds; NProto::TError Error; @@ -331,6 +338,7 @@ struct TTxIndexTablet SessionHistory.clear(); OpLog.clear(); LargeDeletionMarkers.clear(); + OrphanNodeIds.clear(); } }; @@ -1778,6 +1786,8 @@ struct TTxIndexTablet const ui64 NodeId; const TByteRange Range; + NProto::TError Error; + TTruncateRange( TRequestInfoPtr requestInfo, ui64 nodeId, @@ -1791,6 +1801,7 @@ struct TTxIndexTablet void Clear() { TProfileAware::Clear(); + Error.Clear(); } }; @@ -1826,6 +1837,8 @@ struct TTxIndexTablet const ui64 NodeId; const TByteRange Range; + NProto::TError Error; + TZeroRange( TRequestInfoPtr requestInfo, ui64 nodeId, @@ -1839,6 +1852,7 @@ struct TTxIndexTablet void Clear() { TProfileAware::Clear(); + Error.Clear(); } }; @@ -1956,7 +1970,7 @@ struct TTxIndexTablet NProto::TCreateNodeResponse Response; const ui64 EntryId; - explicit TCommitNodeCreationInFollower( + TCommitNodeCreationInFollower( TString sessionId, ui64 requestId, NProto::TCreateNodeResponse response, @@ -1971,6 +1985,72 @@ struct TTxIndexTablet { } }; + + // + // UnsafeNodeOps + // + + struct TUnsafeDeleteNode: TIndexStateNodeUpdates + { + const TRequestInfoPtr RequestInfo; + const NProtoPrivate::TUnsafeDeleteNodeRequest Request; + + TMaybe Node; + NProto::TError Error; + + TUnsafeDeleteNode( + TRequestInfoPtr requestInfo, + NProtoPrivate::TUnsafeDeleteNodeRequest request) + : RequestInfo(std::move(requestInfo)) + , Request(std::move(request)) + {} + + void Clear() + { + Node.Clear(); + Error.Clear(); + } + }; + + struct TUnsafeUpdateNode: TIndexStateNodeUpdates + { + const TRequestInfoPtr RequestInfo; + const NProtoPrivate::TUnsafeUpdateNodeRequest Request; + + TMaybe Node; + + TUnsafeUpdateNode( + TRequestInfoPtr requestInfo, + NProtoPrivate::TUnsafeUpdateNodeRequest request) + : RequestInfo(std::move(requestInfo)) + , Request(std::move(request)) + {} + + void Clear() + { + Node.Clear(); + } + }; + + struct TUnsafeGetNode + { + const TRequestInfoPtr RequestInfo; + const NProtoPrivate::TUnsafeGetNodeRequest Request; + + TMaybe Node; + + TUnsafeGetNode( + TRequestInfoPtr requestInfo, + NProtoPrivate::TUnsafeGetNodeRequest request) + : RequestInfo(std::move(requestInfo)) + , Request(std::move(request)) + {} + + void Clear() + { + Node.Clear(); + } + }; }; } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp index 478586de12f..83419df0e6a 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp @@ -6086,6 +6086,8 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) storageConfig.SetLargeDeletionMarkerBlocks(1_GB / block); storageConfig.SetLargeDeletionMarkersThreshold(128_GB / block); storageConfig.SetLargeDeletionMarkersCleanupThreshold(3_TB / block); + storageConfig.SetLargeDeletionMarkersThresholdForBackpressure( + 10_TB / block); const auto blobSize = 2 * block; storageConfig.SetWriteBlobThreshold(blobSize); @@ -6217,6 +6219,112 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) } } + TABLET_TEST(ShouldRejectLargeFileTruncationIfLargeDeletionMarkerCountIsTooHigh) + { + const auto block = tabletConfig.BlockSize; + + NProto::TStorageConfig storageConfig; + storageConfig.SetMaxFileBlocks(5_TB / block); + storageConfig.SetLargeDeletionMarkersEnabled(true); + storageConfig.SetLargeDeletionMarkerBlocks(1_GB / block); + storageConfig.SetLargeDeletionMarkersThreshold(128_GB / block); + storageConfig.SetLargeDeletionMarkersCleanupThreshold(20_TB / block); + storageConfig.SetLargeDeletionMarkersThresholdForBackpressure( + 4_TB / block); + const auto blobSize = 2 * block; + storageConfig.SetWriteBlobThreshold(blobSize); + + TTestEnv env({}, storageConfig); + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + tabletConfig.BlockCount = 10_TB / block; + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + tabletConfig); + tablet.InitSession("client", "session"); + + auto id1 = + CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test1")); + auto id2 = + CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test2")); + CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test3")); + + for (const auto id: {id1, id2}) { + TSetNodeAttrArgs args(id); + args.SetFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); + args.SetSize(5_TB); + tablet.SetNodeAttr(args); + UNIT_ASSERT_VALUES_EQUAL(5_TB, GetNodeAttrs(tablet, id).GetSize()); + } + + { + tablet.SendUnlinkNodeRequest(RootNodeId, "test1", false); + auto response = tablet.RecvUnlinkNodeResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + S_OK, + response->GetStatus(), + response->GetErrorReason()); + } + + { + tablet.SendUnlinkNodeRequest(RootNodeId, "test2", false); + auto response = tablet.RecvUnlinkNodeResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + E_REJECTED, + response->GetStatus(), + response->GetErrorReason()); + } + + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL( + 5_TB / block, + stats.GetLargeDeletionMarkersCount()); + UNIT_ASSERT_VALUES_EQUAL(2, stats.GetUsedNodesCount()); + } + + tablet.AdvanceTime(TDuration::Seconds(15)); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(5)); + + auto registry = env.GetRegistry(); + { + TTestRegistryVisitor visitor; + registry->Visit(TInstant::Zero(), visitor); + visitor.ValidateExpectedCounters({ + {{{"sensor", "OrphanNodesCount"}, {"filesystem", "test"}}, 0}, + }); + } + + tablet.RenameNode(RootNodeId, "test3", RootNodeId, "test2"); + + tablet.AdvanceTime(TDuration::Seconds(15)); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(5)); + + { + TTestRegistryVisitor visitor; + registry->Visit(TInstant::Zero(), visitor); + visitor.ValidateExpectedCounters({ + {{{"sensor", "OrphanNodesCount"}, {"filesystem", "test"}}, 1}, + }); + } + + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL( + 5_TB / block, + stats.GetLargeDeletionMarkersCount()); + UNIT_ASSERT_VALUES_EQUAL(2, stats.GetUsedNodesCount()); + } + } + TABLET_TEST_4K_ONLY(ShouldHandleRangeIdCollisionsInCompactionMapStats) { const auto block = tabletConfig.BlockSize; diff --git a/cloud/filestore/libs/storage/tablet/ya.make b/cloud/filestore/libs/storage/tablet/ya.make index 1fa863c0cb6..348e0ffa87e 100644 --- a/cloud/filestore/libs/storage/tablet/ya.make +++ b/cloud/filestore/libs/storage/tablet/ya.make @@ -64,6 +64,7 @@ SRCS( tablet_actor_throttling.cpp tablet_actor_truncate.cpp tablet_actor_unlinknode.cpp + tablet_actor_unsafe_node_ops.cpp tablet_actor_updateconfig.cpp tablet_actor_waitready.cpp tablet_actor_writebatch.cpp diff --git a/cloud/filestore/libs/vfs_fuse/CMakeLists.darwin-x86_64.txt b/cloud/filestore/libs/vfs_fuse/CMakeLists.darwin-x86_64.txt index 23d6e7ca940..95cb033a932 100644 --- a/cloud/filestore/libs/vfs_fuse/CMakeLists.darwin-x86_64.txt +++ b/cloud/filestore/libs/vfs_fuse/CMakeLists.darwin-x86_64.txt @@ -31,6 +31,7 @@ target_link_libraries(filestore-libs-vfs_fuse PUBLIC target_sources(filestore-libs-vfs_fuse PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/cache.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/config.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl_attr.cpp diff --git a/cloud/filestore/libs/vfs_fuse/CMakeLists.linux-aarch64.txt b/cloud/filestore/libs/vfs_fuse/CMakeLists.linux-aarch64.txt index 94d43d7b3c6..40a721a60da 100644 --- a/cloud/filestore/libs/vfs_fuse/CMakeLists.linux-aarch64.txt +++ b/cloud/filestore/libs/vfs_fuse/CMakeLists.linux-aarch64.txt @@ -32,6 +32,7 @@ target_link_libraries(filestore-libs-vfs_fuse PUBLIC target_sources(filestore-libs-vfs_fuse PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/cache.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/config.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl_attr.cpp diff --git a/cloud/filestore/libs/vfs_fuse/CMakeLists.linux-x86_64.txt b/cloud/filestore/libs/vfs_fuse/CMakeLists.linux-x86_64.txt index edf5cf58287..46343a852f6 100644 --- a/cloud/filestore/libs/vfs_fuse/CMakeLists.linux-x86_64.txt +++ b/cloud/filestore/libs/vfs_fuse/CMakeLists.linux-x86_64.txt @@ -33,6 +33,7 @@ target_link_libraries(filestore-libs-vfs_fuse PUBLIC target_sources(filestore-libs-vfs_fuse PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/cache.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/config.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl_attr.cpp diff --git a/cloud/filestore/libs/vfs_fuse/CMakeLists.windows-x86_64.txt b/cloud/filestore/libs/vfs_fuse/CMakeLists.windows-x86_64.txt index 23d6e7ca940..95cb033a932 100644 --- a/cloud/filestore/libs/vfs_fuse/CMakeLists.windows-x86_64.txt +++ b/cloud/filestore/libs/vfs_fuse/CMakeLists.windows-x86_64.txt @@ -31,6 +31,7 @@ target_link_libraries(filestore-libs-vfs_fuse PUBLIC target_sources(filestore-libs-vfs_fuse PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/cache.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/config.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl_attr.cpp diff --git a/cloud/filestore/libs/vfs_fuse/file_ring_buffer.cpp b/cloud/filestore/libs/vfs_fuse/file_ring_buffer.cpp new file mode 100644 index 00000000000..e2d79050771 --- /dev/null +++ b/cloud/filestore/libs/vfs_fuse/file_ring_buffer.cpp @@ -0,0 +1,313 @@ +#include "file_ring_buffer.h" + +#include + +#include +#include +#include +#include + +#include + +namespace NCloud::NFileStore { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +constexpr ui32 VERSION = 1; +constexpr ui32 INVALID_POS = Max(); +constexpr TStringBuf INVALID_MARKER = "invalid_entry_marker"; + +//////////////////////////////////////////////////////////////////////////////// + +struct THeader +{ + ui32 Version = 0; + ui32 Capacity = 0; + ui32 ReadPos = 0; + ui32 WritePos = 0; +}; + +struct Y_PACKED TEntryHeader +{ + ui32 Size = 0; + ui32 Checksum = 0; +}; + +void WriteEntry(IOutputStream& os, TStringBuf data) +{ + TEntryHeader eh; + eh.Size = data.Size(); + eh.Checksum = Crc32c(data.Data(), data.Size()); + os.Write(&eh, sizeof(eh)); + os.Write(data); +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +class TFileRingBuffer::TImpl +{ +private: + TFileMap Map; + const ui32 MaxEntrySize; + + char* Data = nullptr; + ui32 Count = 0; + const char* End = nullptr; + +private: + THeader* Header() + { + return reinterpret_cast(Map.Ptr()); + } + + const THeader* Header() const + { + return reinterpret_cast(Map.Ptr()); + } + + void SkipSlackSpace() + { + if (Header()->ReadPos == Header()->WritePos) { + Header()->ReadPos = 0; + Header()->WritePos = 0; + return; + } + + const auto* b = Data + Header()->ReadPos; + const auto* eh = reinterpret_cast(b); + if (eh->Size == 0) { + Header()->ReadPos = 0; + } + } + + using TVisitor = std::function; + + ui32 VisitEntry(const TVisitor& visitor, ui32 pos) const + { + const auto* b = Data + pos; + if (b + sizeof(TEntryHeader) > End) { + visitor(0, INVALID_MARKER); + return INVALID_POS; + } + + const auto* eh = reinterpret_cast(b); + if (eh->Size == 0) { + return 0; + } + + TStringBuf entry(b + sizeof(TEntryHeader), eh->Size); + if (entry.Data() + entry.Size() > End) { + visitor(eh->Checksum, INVALID_MARKER); + return INVALID_POS; + } + visitor(eh->Checksum, {b + sizeof(TEntryHeader), eh->Size}); + return pos + sizeof(TEntryHeader) + eh->Size; + } + + void Visit(const TVisitor& visitor) const + { + ui32 pos = Header()->ReadPos; + while (pos > Header()->WritePos && pos != INVALID_POS) { + pos = VisitEntry(visitor, pos); + } + + while (pos < Header()->WritePos && pos != INVALID_POS) { + pos = VisitEntry(visitor, pos); + if (!pos) { + // can happen if the buffer is corrupted + break; + } + } + } + +public: + TImpl(const TString& filePath, ui32 capacity, ui32 maxEntrySize) + : Map(filePath, TMemoryMapCommon::oRdWr) + , MaxEntrySize(maxEntrySize) + { + Y_ABORT_UNLESS(MaxEntrySize + sizeof(TEntryHeader) <= capacity); + + const ui32 realSize = sizeof(THeader) + capacity; + if (Map.Length() < realSize) { + Map.ResizeAndRemap(0, realSize); + } else { + Map.Map(0, realSize); + } + + if (Header()->Version) { + Y_ABORT_UNLESS(Header()->Version == VERSION); + Y_ABORT_UNLESS(Header()->Capacity == capacity); + } else { + Header()->Capacity = capacity; + Header()->Version = VERSION; + } + + Data = static_cast(Map.Ptr()) + sizeof(THeader); + End = Data + capacity; + + SkipSlackSpace(); + Visit([this] (ui32 checksum, TStringBuf entry) { + Y_UNUSED(checksum); + Y_UNUSED(entry); + ++Count; + }); + } + +public: + bool Push(TStringBuf data) + { + if (data.Empty() || data.Size() > MaxEntrySize) { + return false; + } + + const auto sz = data.Size() + sizeof(TEntryHeader); + auto* ptr = Data + Header()->WritePos; + + if (!Empty()) { + // checking that we have a contiguous chunk of sz + 1 bytes + // 1 extra byte is needed to distinguish between an empty buffer + // and a buffer which is completely full + if (Header()->ReadPos < Header()->WritePos) { + // we have a single contiguous occupied region + ui32 freeSpace = Header()->Capacity - Header()->WritePos; + if (freeSpace <= sz) { + if (Header()->ReadPos <= sz) { + // out of space + return false; + } + + memset(ptr, 0, freeSpace); + ptr = Data; + } + } else { + // we have two occupied regions + ui32 freeSpace = Header()->ReadPos - Header()->WritePos; + if (freeSpace <= sz) { + // out of space + return false; + } + } + } + + TMemoryOutput mo(ptr, sz); + WriteEntry(mo, data); + + Header()->WritePos = ptr - Data + sz; + ++Count; + + return true; + } + + TStringBuf Front() const + { + if (Empty()) { + return {}; + } + + const auto* b = Data + Header()->ReadPos; + if (b + sizeof(TEntryHeader) > End) { + // corruption + // TODO: report? + return {}; + } + + const auto* eh = reinterpret_cast(b); + TStringBuf result{b + sizeof(TEntryHeader), eh->Size}; + if (result.Data() + result.Size() > End) { + // corruption + // TODO: report? + return {}; + } + + return result; + } + + void Pop() + { + auto data = Front(); + if (!data) { + return; + } + + Header()->ReadPos += sizeof(TEntryHeader) + data.Size(); + --Count; + + SkipSlackSpace(); + } + + ui32 Size() const + { + return Count; + } + + bool Empty() const + { + const bool result = Header()->ReadPos == Header()->WritePos; + Y_DEBUG_ABORT_UNLESS(result == (Count == 0)); + return result; + } + + auto Validate() const + { + TVector entries; + + Visit([&] (ui32 checksum, TStringBuf entry) { + const ui32 actualChecksum = Crc32c(entry.Data(), entry.Size()); + if (actualChecksum != checksum) { + entries.push_back({ + TString(entry), + checksum, + actualChecksum}); + } + }); + + return entries; + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +TFileRingBuffer::TFileRingBuffer( + const TString& filePath, + ui32 capacity, + ui32 maxEntrySize) + : Impl(new TImpl(filePath, capacity, maxEntrySize)) +{} + +TFileRingBuffer::~TFileRingBuffer() = default; + +bool TFileRingBuffer::Push(TStringBuf data) +{ + return Impl->Push(data); +} + +TStringBuf TFileRingBuffer::Front() const +{ + return Impl->Front(); +} + +void TFileRingBuffer::Pop() +{ + Impl->Pop(); +} + +ui32 TFileRingBuffer::Size() const +{ + return Impl->Size(); +} + +bool TFileRingBuffer::Empty() const +{ + return Impl->Empty(); +} + +TVector TFileRingBuffer::Validate() const +{ + return Impl->Validate(); +} + +} // namespace NCloud::NFileStore diff --git a/cloud/filestore/libs/vfs_fuse/file_ring_buffer.h b/cloud/filestore/libs/vfs_fuse/file_ring_buffer.h new file mode 100644 index 00000000000..8da9fa935ae --- /dev/null +++ b/cloud/filestore/libs/vfs_fuse/file_ring_buffer.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +namespace NCloud::NFileStore { + +//////////////////////////////////////////////////////////////////////////////// + +struct TBrokenFileRingBufferEntry +{ + TString Data; + ui32 ExpectedChecksum = 0; + ui32 ActualChecksum = 0; +}; + +class TFileRingBuffer +{ +private: + class TImpl; + std::unique_ptr Impl; + +public: + TFileRingBuffer(const TString& filePath, ui32 capacity, ui32 maxEntrySize); + ~TFileRingBuffer(); + +public: + bool Push(TStringBuf data); + TStringBuf Front() const; + void Pop(); + ui32 Size() const; + bool Empty() const; + TVector Validate() const; +}; + +} // namespace NCloud::NFileStore diff --git a/cloud/filestore/libs/vfs_fuse/file_ring_buffer_ut.cpp b/cloud/filestore/libs/vfs_fuse/file_ring_buffer_ut.cpp new file mode 100644 index 00000000000..7e789467bc8 --- /dev/null +++ b/cloud/filestore/libs/vfs_fuse/file_ring_buffer_ut.cpp @@ -0,0 +1,338 @@ +#include "file_ring_buffer.h" + +#include + +#include +#include +#include +#include +#include + +namespace NCloud::NFileStore { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TString Dump(const TVector& entries) +{ + TStringBuilder sb; + + for (ui32 i = 0; i < entries.size(); ++i) { + if (i) { + sb << ", "; + } + + sb << "data=" << entries[i].Data + << " ecsum=" << entries[i].ExpectedChecksum + << " csum=" << entries[i].ActualChecksum; + } + + return sb; +} + +TString PopAll(TFileRingBuffer& rb) +{ + TStringBuilder sb; + + while (!rb.Empty()) { + if (sb.Size()) { + sb << ", "; + } + + sb << rb.Front(); + rb.Pop(); + } + + return sb; +} + +//////////////////////////////////////////////////////////////////////////////// + +struct TReferenceImplementation +{ + static constexpr ui32 EntryOverhead = 8; + + const ui32 MaxWeight; + const ui32 MaxEntrySize; + + TDeque Q; + ui32 ReadPos = 0; + ui32 WritePos = 0; + ui32 SlackSpace = 0; + + explicit TReferenceImplementation(ui32 maxWeight, ui32 maxEntrySize) + : MaxWeight(maxWeight) + , MaxEntrySize(maxEntrySize) + {} + + bool Push(TStringBuf data) + { + if (data.Empty() || data.Size() > MaxEntrySize) { + return false; + } + + const ui32 sz = EntryOverhead + data.Size(); + + if (!Empty()) { + if (ReadPos < WritePos) { + const auto avail = MaxWeight - WritePos; + if (avail <= sz) { + if (ReadPos <= sz) { + // out of space + return false; + } + + SlackSpace = avail; + WritePos = 0; + } + } else { + const auto avail = ReadPos - WritePos; + if (avail <= sz) { + // out of space + return false; + } + } + } + + WritePos += sz; + Q.emplace_back(data); + return true; + } + + TStringBuf Front() const + { + if (!Q) { + return {}; + } + + return Q.front(); + } + + void Pop() + { + if (!Q) { + return; + } + + const ui32 sz = Q.front().Size() + EntryOverhead; + ReadPos += sz; + if (MaxWeight - ReadPos <= SlackSpace) { + UNIT_ASSERT_VALUES_EQUAL(SlackSpace, MaxWeight - ReadPos); + if (ReadPos == WritePos) { + WritePos = 0; + } + ReadPos = 0; + SlackSpace = 0; + } + + Q.pop_front(); + } + + bool Empty() const + { + return Q.empty(); + } + + ui32 Size() const + { + return Q.size(); + } + + auto Validate() const + { + return TVector(); + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TFileRingBufferTest) +{ + template + void DoTestShouldPushPop(TRingBuffer& rb) + { + UNIT_ASSERT_VALUES_EQUAL(0, rb.Size()); + UNIT_ASSERT(rb.Empty()); + + UNIT_ASSERT(!rb.Push("longvasya11")); // too long + UNIT_ASSERT(!rb.Push("")); // empty + UNIT_ASSERT(rb.Push("vasya")); + UNIT_ASSERT(rb.Push("petya")); + UNIT_ASSERT(rb.Push("vasya2")); + UNIT_ASSERT(rb.Push("petya2")); + UNIT_ASSERT(!rb.Push("vasya3")); // out of space + + UNIT_ASSERT_VALUES_EQUAL("", Dump(rb.Validate())); + UNIT_ASSERT_VALUES_EQUAL(4, rb.Size()); + UNIT_ASSERT_VALUES_EQUAL("vasya", rb.Front()); + rb.Pop(); + + UNIT_ASSERT_VALUES_EQUAL("", Dump(rb.Validate())); + UNIT_ASSERT_VALUES_EQUAL(3, rb.Size()); + UNIT_ASSERT(!rb.Push("vasya3")); + + UNIT_ASSERT_VALUES_EQUAL("petya", rb.Front()); + rb.Pop(); + + UNIT_ASSERT_VALUES_EQUAL("", Dump(rb.Validate())); + UNIT_ASSERT_VALUES_EQUAL(2, rb.Size()); + UNIT_ASSERT(rb.Push("vasya3")); + + UNIT_ASSERT_VALUES_EQUAL("", Dump(rb.Validate())); + UNIT_ASSERT_VALUES_EQUAL(3, rb.Size()); + UNIT_ASSERT_VALUES_EQUAL("vasya2", rb.Front()); + rb.Pop(); + + UNIT_ASSERT_VALUES_EQUAL("", Dump(rb.Validate())); + UNIT_ASSERT_VALUES_EQUAL(2, rb.Size()); + UNIT_ASSERT_VALUES_EQUAL("petya2", rb.Front()); + rb.Pop(); + + UNIT_ASSERT_VALUES_EQUAL("", Dump(rb.Validate())); + UNIT_ASSERT_VALUES_EQUAL(1, rb.Size()); + UNIT_ASSERT_VALUES_EQUAL("vasya3", rb.Front()); + rb.Pop(); + + UNIT_ASSERT_VALUES_EQUAL("", Dump(rb.Validate())); + UNIT_ASSERT_VALUES_EQUAL(0, rb.Size()); + UNIT_ASSERT(rb.Empty()); + } + + Y_UNIT_TEST(ShouldPushPop) + { + const auto f = TTempFileHandle(); + const ui32 len = 64; + const ui32 maxEntrySize = 10; + TFileRingBuffer rb(f.GetName(), len, maxEntrySize); + + DoTestShouldPushPop(rb); + } + + Y_UNIT_TEST(ShouldPushPopReferenceImplementation) + { + const ui32 len = 64; + const ui32 maxEntrySize = 10; + TReferenceImplementation rb(len, maxEntrySize); + + DoTestShouldPushPop(rb); + } + + Y_UNIT_TEST(ShouldRestore) + { + const auto f = TTempFileHandle(); + const ui32 len = 64; + const ui32 maxEntrySize = 10; + auto rb = std::make_unique( + f.GetName(), + len, + maxEntrySize); + + UNIT_ASSERT(rb->Push("vasya")); + UNIT_ASSERT(rb->Push("petya")); + UNIT_ASSERT(rb->Push("vasya2")); + UNIT_ASSERT(rb->Push("petya2")); + rb->Pop(); + rb->Pop(); + UNIT_ASSERT(rb->Push("vasya3")); + UNIT_ASSERT(rb->Push("xxx")); + + rb = std::make_unique( + f.GetName(), + len, + maxEntrySize); + + UNIT_ASSERT_VALUES_EQUAL("", Dump(rb->Validate())); + UNIT_ASSERT_VALUES_EQUAL(4, rb->Size()); + + UNIT_ASSERT_VALUES_EQUAL("vasya2, petya2, vasya3, xxx", PopAll(*rb)); + } + + Y_UNIT_TEST(ShouldValidate) + { + const auto f = TTempFileHandle(); + const ui32 len = 64; + const ui32 maxEntrySize = 10; + TFileRingBuffer rb(f.GetName(), len, maxEntrySize); + + UNIT_ASSERT(rb.Push("vasya")); + UNIT_ASSERT(rb.Push("petya")); + UNIT_ASSERT(rb.Push("vasya2")); + UNIT_ASSERT(rb.Push("petya2")); + + UNIT_ASSERT_VALUES_EQUAL("", Dump(rb.Validate())); + TFileMap m(f.GetName(), TMemoryMapCommon::oRdWr); + m.Map(0, len); + char* data = static_cast(m.Ptr()); + data[10] = 'A'; + + UNIT_ASSERT_VALUES_EQUAL( + "data=invalid_entry_marker ecsum=0 csum=11034342", + Dump(rb.Validate())); + } + + TString GenerateData(ui32 sz) + { + TString s(sz, 0); + for (ui32 i = 0; i < sz; ++i) { + s[i] = 'a' + RandomNumber('z' - 'a' + 1); + } + return s; + } + + Y_UNIT_TEST(RandomizedPushPopRestore) + { + const auto f = TTempFileHandle(); + const ui32 len = 1_MB; + const ui32 testBytes = 16_MB; + const ui32 maxEntrySize = 4_KB; + const ui32 testUpToEntrySize = 5_KB; + const double restoreProbability = 0.05; + std::unique_ptr rb; + TReferenceImplementation ri(len, maxEntrySize); + + auto restore = [&] () { + rb = std::make_unique( + f.GetName(), + len, + maxEntrySize); + }; + + restore(); + + ui32 remainingBytes = testBytes; + while (remainingBytes || !ri.Empty()) { + const bool shouldPush = remainingBytes && RandomNumber(); + if (shouldPush) { + const ui32 entrySize = + RandomNumber(Min(remainingBytes + 1, testUpToEntrySize)); + const auto data = GenerateData(entrySize); + const bool pushed = ri.Push(data); + UNIT_ASSERT_VALUES_EQUAL(pushed, rb->Push(data)); + if (pushed) { + remainingBytes -= entrySize; + // Cerr << "PUSH\t" << data << Endl; + } + } else { + UNIT_ASSERT_VALUES_EQUAL(ri.Front(), rb->Front()); + // Cerr << "POP\t" << ri.Front() << Endl; + ri.Pop(); + rb->Pop(); + } + + // Cerr << ri.Size() << " " << remainingBytes << Endl; + + if (RandomNumber() < restoreProbability) { + restore(); + } + + UNIT_ASSERT_VALUES_EQUAL(ri.Size(), rb->Size()); + UNIT_ASSERT_VALUES_EQUAL(ri.Empty(), rb->Empty()); + UNIT_ASSERT_VALUES_EQUAL("", Dump(rb->Validate())); + } + } +} + +} // namespace NCloud::NFileStore diff --git a/cloud/filestore/libs/vfs_fuse/handle_ops_queue.cpp b/cloud/filestore/libs/vfs_fuse/handle_ops_queue.cpp index 9ec05a7338d..a4ac62ba5db 100644 --- a/cloud/filestore/libs/vfs_fuse/handle_ops_queue.cpp +++ b/cloud/filestore/libs/vfs_fuse/handle_ops_queue.cpp @@ -6,30 +6,30 @@ namespace NCloud::NFileStore::NFuse { void THandleOpsQueue::AddDestroyRequest(ui64 nodeId, ui64 handle) { - NProto::TQueueEntry request; - request.MutableDestroyHandleRequest()->SetHandle(handle); - request.MutableDestroyHandleRequest()->SetNodeId(nodeId); - Requests.push(request); + NProto::TQueueEntry request; + request.MutableDestroyHandleRequest()->SetHandle(handle); + request.MutableDestroyHandleRequest()->SetNodeId(nodeId); + Requests.push(request); } const NProto::TQueueEntry& THandleOpsQueue::Front() { - return Requests.front(); + return Requests.front(); } bool THandleOpsQueue::Empty() const { - return Requests.empty(); + return Requests.empty(); } void THandleOpsQueue::Pop() { - Requests.pop(); + Requests.pop(); } ui64 THandleOpsQueue::Size() const { - return Requests.size(); + return Requests.size(); } } // namespace NCloud::NFileStore::NFuse diff --git a/cloud/filestore/libs/vfs_fuse/handle_ops_queue.h b/cloud/filestore/libs/vfs_fuse/handle_ops_queue.h index 849e16bd9bd..2b3a8af430f 100644 --- a/cloud/filestore/libs/vfs_fuse/handle_ops_queue.h +++ b/cloud/filestore/libs/vfs_fuse/handle_ops_queue.h @@ -1,3 +1,5 @@ +#pragma once + #include #include @@ -9,14 +11,14 @@ namespace NCloud::NFileStore::NFuse { class THandleOpsQueue { private: - TQueue Requests; + TQueue Requests; public: - void AddDestroyRequest(ui64 nodeId, ui64 handle); - const NProto::TQueueEntry& Front(); - void Pop(); - ui64 Size() const; - bool Empty() const; + void AddDestroyRequest(ui64 nodeId, ui64 handle); + const NProto::TQueueEntry& Front(); + void Pop(); + ui64 Size() const; + bool Empty() const; }; } // namespace NCloud::NFileStore::NFuse diff --git a/cloud/filestore/libs/vfs_fuse/loop.cpp b/cloud/filestore/libs/vfs_fuse/loop.cpp index fc41789be7f..c0ebb8c54fc 100644 --- a/cloud/filestore/libs/vfs_fuse/loop.cpp +++ b/cloud/filestore/libs/vfs_fuse/loop.cpp @@ -1007,7 +1007,7 @@ class TFileSystemLoop final // ops.statfs = [] (fuse_req_t req, fuse_ino_t ino) { - CALL(StatFs, EFileStoreRequest::GetFileStoreInfo, 0, req, ino); + CALL(StatFs, EFileStoreRequest::StatFileStore, 0, req, ino); }; // diff --git a/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.darwin-x86_64.txt b/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.darwin-x86_64.txt index a6906d8cce9..71ef4a61853 100644 --- a/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.darwin-x86_64.txt +++ b/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.darwin-x86_64.txt @@ -25,6 +25,7 @@ target_link_options(cloud-filestore-libs-vfs_fuse-ut PRIVATE CoreFoundation ) target_sources(cloud-filestore-libs-vfs_fuse-ut PRIVATE + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer_ut.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_ut.cpp ) set_property( diff --git a/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.linux-aarch64.txt b/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.linux-aarch64.txt index 2b27e93f90e..7e2fff15300 100644 --- a/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.linux-aarch64.txt +++ b/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.linux-aarch64.txt @@ -28,6 +28,7 @@ target_link_options(cloud-filestore-libs-vfs_fuse-ut PRIVATE -ldl ) target_sources(cloud-filestore-libs-vfs_fuse-ut PRIVATE + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer_ut.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_ut.cpp ) set_property( diff --git a/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.linux-x86_64.txt b/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.linux-x86_64.txt index 8c2b35eef01..942ab76a0af 100644 --- a/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.linux-x86_64.txt +++ b/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.linux-x86_64.txt @@ -29,6 +29,7 @@ target_link_options(cloud-filestore-libs-vfs_fuse-ut PRIVATE -ldl ) target_sources(cloud-filestore-libs-vfs_fuse-ut PRIVATE + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer_ut.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_ut.cpp ) set_property( diff --git a/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.windows-x86_64.txt b/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.windows-x86_64.txt index 8287aecf90d..a1c55759c71 100644 --- a/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.windows-x86_64.txt +++ b/cloud/filestore/libs/vfs_fuse/ut/CMakeLists.windows-x86_64.txt @@ -18,6 +18,7 @@ target_link_libraries(cloud-filestore-libs-vfs_fuse-ut PUBLIC filestore-libs-vhost ) target_sources(cloud-filestore-libs-vfs_fuse-ut PRIVATE + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer_ut.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_ut.cpp ) set_property( diff --git a/cloud/filestore/libs/vfs_fuse/ut/ya.make b/cloud/filestore/libs/vfs_fuse/ut/ya.make index d92df2d4e6c..37b10750f8b 100644 --- a/cloud/filestore/libs/vfs_fuse/ut/ya.make +++ b/cloud/filestore/libs/vfs_fuse/ut/ya.make @@ -5,6 +5,7 @@ INCLUDE(${ARCADIA_ROOT}/cloud/filestore/tests/recipes/medium.inc) SRCDIR(cloud/filestore/libs/vfs_fuse) SRCS( + file_ring_buffer_ut.cpp fs_ut.cpp ) diff --git a/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.darwin-x86_64.txt b/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.darwin-x86_64.txt index 03363f6de8e..9b59bf00a07 100644 --- a/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.darwin-x86_64.txt +++ b/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.darwin-x86_64.txt @@ -30,6 +30,7 @@ target_link_libraries(libs-vfs_fuse-vhost PUBLIC target_sources(libs-vfs_fuse-vhost PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/cache.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/config.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl_attr.cpp diff --git a/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.linux-aarch64.txt b/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.linux-aarch64.txt index 0ea8ccccb98..0412eaded12 100644 --- a/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.linux-aarch64.txt +++ b/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.linux-aarch64.txt @@ -32,6 +32,7 @@ target_link_libraries(libs-vfs_fuse-vhost PUBLIC target_sources(libs-vfs_fuse-vhost PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/cache.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/config.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl_attr.cpp diff --git a/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.linux-x86_64.txt b/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.linux-x86_64.txt index 0ea8ccccb98..0412eaded12 100644 --- a/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.linux-x86_64.txt +++ b/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.linux-x86_64.txt @@ -32,6 +32,7 @@ target_link_libraries(libs-vfs_fuse-vhost PUBLIC target_sources(libs-vfs_fuse-vhost PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/cache.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/config.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl_attr.cpp diff --git a/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.windows-x86_64.txt b/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.windows-x86_64.txt index 03363f6de8e..9b59bf00a07 100644 --- a/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.windows-x86_64.txt +++ b/cloud/filestore/libs/vfs_fuse/vhost/CMakeLists.windows-x86_64.txt @@ -30,6 +30,7 @@ target_link_libraries(libs-vfs_fuse-vhost PUBLIC target_sources(libs-vfs_fuse-vhost PRIVATE ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/cache.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/config.cpp + ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/file_ring_buffer.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl.cpp ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/vfs_fuse/fs_impl_attr.cpp diff --git a/cloud/filestore/libs/vfs_fuse/ya.make.inc b/cloud/filestore/libs/vfs_fuse/ya.make.inc index 0d9604a29c5..e01bfd59275 100644 --- a/cloud/filestore/libs/vfs_fuse/ya.make.inc +++ b/cloud/filestore/libs/vfs_fuse/ya.make.inc @@ -1,6 +1,7 @@ SRCS( cache.cpp config.cpp + file_ring_buffer.cpp fs.cpp fs_impl.cpp fs_impl_attr.cpp diff --git a/cloud/filestore/private/api/protos/tablet.proto b/cloud/filestore/private/api/protos/tablet.proto index 42e6440a190..c4cb014fe15 100644 --- a/cloud/filestore/private/api/protos/tablet.proto +++ b/cloud/filestore/private/api/protos/tablet.proto @@ -601,3 +601,43 @@ message TWriteCompactionMapResponse // Optional error, set only if error happened. NCloud.NProto.TError Error = 1; } + +//////////////////////////////////////////////////////////////////////////////// +// Direct inode manipulation. + +message TUnsafeDeleteNodeRequest +{ + string FileSystemId = 1; + uint64 Id = 2; +} + +message TUnsafeDeleteNodeResponse +{ + // Optional error, set only if error happened. + NCloud.NProto.TError Error = 1; +} + +message TUnsafeUpdateNodeRequest +{ + string FileSystemId = 1; + NProto.TNodeAttr Node = 2; +} + +message TUnsafeUpdateNodeResponse +{ + // Optional error, set only if error happened. + NCloud.NProto.TError Error = 1; +} + +message TUnsafeGetNodeRequest +{ + string FileSystemId = 1; + uint64 Id = 2; +} + +message TUnsafeGetNodeResponse +{ + // Optional error, set only if error happened. + NCloud.NProto.TError Error = 1; + NProto.TNodeAttr Node = 2; +} diff --git a/cloud/filestore/tests/client/canondata/result.json b/cloud/filestore/tests/client/canondata/result.json index a90d991861a..1bb59bd5bb0 100644 --- a/cloud/filestore/tests/client/canondata/result.json +++ b/cloud/filestore/tests/client/canondata/result.json @@ -11,6 +11,9 @@ "test.test_describe_sessions": { "uri": "file://test.test_describe_sessions/results.txt" }, + "test.test_large_file": { + "uri": "file://test.test_large_file/results.txt" + }, "test.test_list_filestores": { "uri": "file://test.test_list_filestores/results.txt" }, diff --git a/cloud/filestore/tests/client/canondata/test.test_large_file/results.txt b/cloud/filestore/tests/client/canondata/test.test_large_file/results.txt new file mode 100644 index 00000000000..b62d5e3351d Binary files /dev/null and b/cloud/filestore/tests/client/canondata/test.test_large_file/results.txt differ diff --git a/cloud/filestore/tests/client/nfs-storage.txt b/cloud/filestore/tests/client/nfs-storage.txt index 862fa383148..b141fad8f57 100644 --- a/cloud/filestore/tests/client/nfs-storage.txt +++ b/cloud/filestore/tests/client/nfs-storage.txt @@ -1 +1,3 @@ MultiTabletForwardingEnabled: true +LargeDeletionMarkersEnabled: true +MaxFileBlocks: 536870912 diff --git a/cloud/filestore/tests/client/test.py b/cloud/filestore/tests/client/test.py index 954d9e94a0c..d745af69705 100644 --- a/cloud/filestore/tests/client/test.py +++ b/cloud/filestore/tests/client/test.py @@ -19,19 +19,25 @@ def __init_test(): return client, results_path +def __process_stat(node): + def d(k): + if k in node: + del node[k] + + d("ATime") + d("MTime") + d("CTime") + d("FollowerNodeName") + + return node + + def __exec_ls(client, *args): output = str(client.ls(*args, "--json"), 'utf-8') nodes: list = json.loads(output)['content'] for node in nodes: - def d(k): - if k in node: - del node[k] - - d("ATime") - d("MTime") - d("CTime") - d("FollowerNodeName") + __process_stat(node) return json.dumps(nodes, indent=4).encode('utf-8') @@ -428,3 +434,61 @@ def test_multitablet_findgarbage(): ret = common.canonical_file(results_path, local=True) return ret + + +def test_large_file(): + TiB = 1024 * 1024 * 1024 * 1024 + + data_file = os.path.join(common.output_path(), "data.txt") + with open(data_file, "w") as f: + f.write("some data") + + client, results_path = __init_test() + client.create( + "fs0", + "test_cloud", + "test_folder", + BLOCK_SIZE, + TiB // BLOCK_SIZE) + client.mkdir("fs0", "/aaa") + client.touch("fs0", "/aaa/bbb") + + out = client.stat("fs0", "/aaa/bbb") + stat = json.loads(out) + node_id = stat["Id"] + result = json.dumps(__process_stat(stat)) + result += "\n" + + client.write("fs0", "/aaa/bbb", "--data", data_file) + + client.set_node_attr("fs0", node_id, "--size", str(TiB)) + result += client.read("fs0", "/aaa/bbb", "--length", "9").decode("utf8") + result += "\n" + + out = client.stat("fs0", "/aaa/bbb") + stat = json.loads(out) + assert stat["Size"] == TiB + result += json.dumps(__process_stat(stat)) + result += "\n" + + client.set_node_attr("fs0", node_id, "--size", "1024") + result += client.read("fs0", "/aaa/bbb", "--length", "0").decode("utf8") + result += "\n" + + out = client.stat("fs0", "/aaa/bbb") + new_stat = json.loads(out) + assert stat["Size"] == TiB + result += json.dumps(__process_stat(new_stat)) + result += "\n" + result += client.read("fs0", "/aaa/bbb", "--length", "0").decode("utf8") + result += "\n" + + client.rm("fs0", "/aaa/bbb") + + client.destroy("fs0") + + with open(results_path, "w") as results_file: + results_file.write(result) + + ret = common.canonical_file(results_path, local=True) + return ret diff --git a/cloud/filestore/tests/python/lib/client.py b/cloud/filestore/tests/python/lib/client.py index 3d17f9b2701..977e708754e 100644 --- a/cloud/filestore/tests/python/lib/client.py +++ b/cloud/filestore/tests/python/lib/client.py @@ -261,6 +261,10 @@ def mkdir(self, cmd): def write(self, cmd): return common.execute(cmd).stdout + @standard_command("read") + def read(self, cmd): + return common.execute(cmd).stdout + @standard_command("touch") def touch(self, cmd): return common.execute(cmd).stdout diff --git a/cloud/storage/core/tools/testing/unstable-process/__main__.py b/cloud/storage/core/tools/testing/unstable-process/__main__.py index 4fe4e14d68f..4300147731b 100644 --- a/cloud/storage/core/tools/testing/unstable-process/__main__.py +++ b/cloud/storage/core/tools/testing/unstable-process/__main__.py @@ -114,6 +114,7 @@ def start_process(): if process.poll() is not None: if process.poll() in [0, 1, 100]: logging.info(f'subprocess failed to start, code {process.poll()}') + logging.info(os.system("ss -tpn")) process = start_process() else: logging.fatal(f'unexpected exit code {process.poll()}') diff --git a/deploy/8_node_cluster/config_cluster_template.yaml b/deploy/8_node_cluster/config_cluster_template.yaml new file mode 100644 index 00000000000..5f95ab3b8fb --- /dev/null +++ b/deploy/8_node_cluster/config_cluster_template.yaml @@ -0,0 +1,133 @@ +use_walle: False +use_fixed_tablet_types: True +require_address: False +cluster_uuid: "2452520" +static_pdisk_type: SSD +static_erasure: block-4-2 +fail_domain_type: body +features: + enable_separate_solomon_shard_for_PDisk: true + enable_VPatch: true +monitoring_address: "::" +hosts: + - name: my.host1 + body: '1' + datacenter: 'my_dc' + rack: 'lab-1:1' + drives: &disk_set_nvme3_hdd1 + - path: /dev/disk/by-partlabel/NVMEKIKIMR01 + type: SSD + - path: /dev/disk/by-partlabel/NVMEKIKIMR02 + type: SSD + - path: /dev/disk/by-partlabel/NVMEKIKIMR03 + type: SSD + - path: /dev/disk/by-partlabel/ROTKIKIMR01 + type: ROT + - name: my.host2 + body: '2' + datacenter: 'my_dc' + rack: 'lab-2:1' + drives: *disk_set_nvme3_hdd1 + - name: my.host3 + body: '3' + datacenter: 'my_dc' + rack: 'lab-3:1' + drives: *disk_set_nvme3_hdd1 + - name: my.host4 + body: '4' + datacenter: 'my_dc' + rack: 'lab-4:1' + drives: *disk_set_nvme3_hdd1 + - name: my.host5 + body: '5' + datacenter: 'my_dc' + rack: 'lab-5:1' + drives: *disk_set_nvme3_hdd1 + - name: my.host6 + body: '6' + datacenter: 'my_dc' + rack: 'lab-6:1' + drives: *disk_set_nvme3_hdd1 + - name: my.host7 + body: '7' + datacenter: 'my_dc' + rack: 'lab-7:1' + drives: *disk_set_nvme3_hdd1 + - name: my.host8 + body: '8' + datacenter: 'my_dc' + rack: 'lab-8:1' + drives: *disk_set_nvme3_hdd1 +ic: + max_inflight_amount_of_data_in_KB: 10240 + handshake_timeout_duration: + seconds: 1 +kqp: + result_rows_limit: 10000 +log: + default: 3 +domains: + - domain_name: my_cluster + storage_pool_kinds: + - kind: "ssd" + erasure: block-4-2 + fail_domain_type: body + filter_properties: + type: SSD + - kind: "rot" + erasure: block-4-2 + fail_domain_type: body + filter_properties: + type: ROT + - kind: "ssdmirror" + erasure: mirror-3of4 + fail_domain_type: body + filter_properties: + type: SSD + - kind: "rotmirror" + erasure: mirror-3of4 + fail_domain_type: body + filter_properties: + type: ROT + databases: + - name: NBS + storage_units: + - kind: ssd + count: 10 + - kind: rot + count: 5 + - kind: rotmirror + count: 5 + - kind: ssdmirror + count: 10 + overridden_configs: + nbs: + enable: true + names: + suppress_version_check: True + nbs_control: + enable: true + - name: NFS + storage_units: + - kind: ssd + count: 20 + - kind: rot + count: 1 + overridden_configs: + nfs: + enable: true +sys: + scheduler: + resolution: 256 + executors: + user: + threads: 8 + spin_threshold: 10 + system: + threads: 8 + spin_threshold: 10 + io: + threads: 1 + batch: + threads: 8 + diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-auth.txt b/deploy/8_node_cluster/nbs/cfg/nbs-auth.txt new file mode 100644 index 00000000000..cc7987d39b9 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-auth.txt @@ -0,0 +1,4 @@ +UseBlackBox: false +UseAccessService: true +UseStaff: false +UseUserAccountService: false diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-breakpad-sender.json b/deploy/8_node_cluster/nbs/cfg/nbs-breakpad-sender.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-breakpad-sender.json @@ -0,0 +1 @@ +{} diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-breakpad.json b/deploy/8_node_cluster/nbs/cfg/nbs-breakpad.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-breakpad.json @@ -0,0 +1 @@ +{} diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-client.txt b/deploy/8_node_cluster/nbs/cfg/nbs-client.txt new file mode 100644 index 00000000000..b829ebe0341 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-client.txt @@ -0,0 +1,49 @@ +ClientConfig { + Host: "localhost" + Port: 9767 + RetryTimeout: 86400000 + ThrottlingConfig { + NonreplThrottlingConfig { + ReadBandwidthPerCpuUnit: 1 + ReadIopsPerCpuUnit: 100 + WriteBandwidthPerCpuUnit: 1 + WriteIopsPerCpuUnit: 100 + MaxReadBandwidth: 1000 + MaxReadIops: 100000 + MaxWriteBandwidth: 1000 + MaxWriteIops: 100000 + } + Mirror2ThrottlingConfig { + ReadBandwidthPerCpuUnit: 1 + ReadIopsPerCpuUnit: 100 + WriteBandwidthPerCpuUnit: 1 + WriteIopsPerCpuUnit: 100 + MaxReadBandwidth: 1000 + MaxReadIops: 100000 + MaxWriteBandwidth: 1000 + MaxWriteIops: 100000 + } + Mirror3ThrottlingConfig { + ReadBandwidthPerCpuUnit: 1 + ReadIopsPerCpuUnit: 100 + WriteBandwidthPerCpuUnit: 1 + WriteIopsPerCpuUnit: 100 + MaxReadBandwidth: 1000 + MaxReadIops: 100000 + MaxWriteBandwidth: 1000 + MaxWriteIops: 100000 + } + MaxIopsPerHost: 126000 + MaxBandwidthPerHost: 2831155200 + NetworkThroughputPercentage: 50 + DefaultHostCpuCount: 100 + DefaultNetworkMbitThroughput: 25000 + } + IpcType: IPC_NBD + NbdSocketSuffix: "_nbd" + NbdUseNbsErrors: true +} +LogConfig { + SysLogService: "NBS_CLIENT" + LogLevel: 6 +} diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-diag.txt b/deploy/8_node_cluster/nbs/cfg/nbs-diag.txt new file mode 100644 index 00000000000..321af5a3801 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-diag.txt @@ -0,0 +1,89 @@ +HostNameScheme: HOSTNAME_YDBVIEWER +UseAsyncLogger: true +SsdPerfSettings { + Write { + Iops: 4200 + Bandwidth: 342000000 + } + Read { + Iops: 4200 + Bandwidth: 342000000 + } +} +HddPerfSettings { + Write { + Iops: 1023 + Bandwidth: 129000000 + } + Read { + Iops: 1023 + Bandwidth: 129000000 + } +} +NonreplPerfSettings { + Write { + Iops: 35114 + Bandwidth: 843000000 + } + Read { + Iops: 35114 + Bandwidth: 843000000 + } +} +Mirror2PerfSettings { + Write { + Iops: 17557 + Bandwidth: 422000000 + } + Read { + Iops: 35114 + Bandwidth: 843000000 + } +} +Mirror3PerfSettings { + Write { + Iops: 11709 + Bandwidth: 281000000 + } + Read { + Iops: 35114 + Bandwidth: 843000000 + } +} +LWTraceShuttleCount: 50000 +RequestThresholds { + Default: 1000 +} +RequestThresholds { + Default: 1000 + MediaKind: STORAGE_MEDIA_HDD +} +RequestThresholds { + Default: 20 + MediaKind: STORAGE_MEDIA_SSD_NONREPLICATED +} +RequestThresholds { + Default: 20 + MediaKind: STORAGE_MEDIA_SSD_MIRROR2 +} +RequestThresholds { + Default: 20 + MediaKind: STORAGE_MEDIA_SSD_MIRROR3 +} +RequestThresholds { + Default: 100 + ByRequestType { + key: "Compaction" + value: 1000 + } + MediaKind: STORAGE_MEDIA_SSD +} +MonitoringUrlData { + MonitoringClusterName: "my_lab" + MonitoringUrl: "https://my.monitoring" + MonitoringProject: "nbs.cloud" + MonitoringVolumeDashboard: "my_volume_dashboard" + MonitoringPartitionDashboard: "my_partition_dashboard" + MonitoringNBSAlertsDashboard: "my_alerts_dashboard" + MonitoringNBSTVDashboard: "my_tv_dashboard" +} diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-discovery.txt b/deploy/8_node_cluster/nbs/cfg/nbs-discovery.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-discovery.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-disk-agent.txt b/deploy/8_node_cluster/nbs/cfg/nbs-disk-agent.txt new file mode 100755 index 00000000000..119f33bd76d --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-disk-agent.txt @@ -0,0 +1,83 @@ +Enabled: true +NvmeTarget { + Nqn: "nqn.2018-09.io.spdk:cnode1" +} +Backend: DISK_AGENT_BACKEND_AIO +AcquireRequired: true +DeviceEraseMethod: DEVICE_ERASE_METHOD_DEALLOCATE +SeqNumber: 1 +DedicatedDiskAgent: true +RdmaTarget { + Endpoint { + Port: 10020 + } + Server { + Backlog: 64 + QueueSize: 512 + MaxBufferSize: 4198400 + WaitMode: WAIT_MODE_ADAPTIVE_WAIT + MaxInflightBytes: 536870912 + } +} +StorageDiscoveryConfig { + PathConfigs { + PathRegExp: "/dev/disk/by-partlabel/NVMENBS([0-9]{2})" + PoolConfigs { + MinSize: 3097678053376 + MaxSize: 3200631791616 + Layout { + HeaderSize: 1073741824 + DevicePadding: 33554432 + DeviceSize: 99857989632 + } + } + } + PathConfigs { + PathRegExp: "/dev/disk/by-partlabel/DEVNBS([0-9]{2})" + PoolConfigs { + PoolName: "dev" + MinSize: 3097678053376 + MaxSize: 3200631791616 + Layout { + HeaderSize: 1073741824 + DevicePadding: 33554432 + DeviceSize: 99857989632 + } + HashSuffix: "-dev" + } + } + PathConfigs { + PathRegExp: "/dev/disk/by-partlabel/NVMELOCAL([0-9]{2})" + PoolConfigs { + PoolName: "chunk-367G" + MinSize: 3097678053376 + MaxSize: 3200631791616 + Layout { + HeaderSize: 1073741824 + DevicePadding: 33554432 + DeviceSize: 394062200832 + } + HashSuffix: "-local" + } + BlockSize: 512 + } + PathConfigs { + PathRegExp: "/dev/disk/by-partlabel/ROTNBS([0-9]{2})" + MaxDeviceCount: 100 + PoolConfigs { + PoolName: "rot" + MinSize: 8000488079360 + MaxSize: 15999825870848 + Layout { + HeaderSize: 1073741824 + DevicePadding: 33554432 + DeviceSize: 99857989632 + } + HashSuffix: "-rot" + } + } +} +CachedConfigPath: "/var/log/nbs-disk-agent/config.cache.txt" +CachedSessionsPath: "/var/log/nbs-disk-agent/sessions.cache.txt" +IOParserActorCount: 1 +OffloadAllIORequestsParsingEnabled: true diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-disk-registry-proxy.txt b/deploy/8_node_cluster/nbs/cfg/nbs-disk-registry-proxy.txt new file mode 100644 index 00000000000..2bfe41c1474 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-disk-registry-proxy.txt @@ -0,0 +1,2 @@ +Owner: 156142163 +OwnerIdx: 1 diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-domains.txt b/deploy/8_node_cluster/nbs/cfg/nbs-domains.txt new file mode 100644 index 00000000000..199358ecec0 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-domains.txt @@ -0,0 +1,128 @@ +Domain { + DomainId: 1 + SchemeRoot: 72057594046678944 + SSId: 1 + HiveUid: 1 + PlanResolution: 10 + Name: "my_cluster" + StoragePoolTypes { + Kind: "ssd" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssd" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "ssdmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssdmirror" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "rot" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rot" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + StoragePoolTypes { + Kind: "rotmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rotmirror" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + ExplicitMediators: 72057594046382081 + ExplicitMediators: 72057594046382082 + ExplicitMediators: 72057594046382083 + ExplicitCoordinators: 72057594046316545 + ExplicitCoordinators: 72057594046316546 + ExplicitCoordinators: 72057594046316547 + ExplicitAllocators: 72057594046447617 + ExplicitAllocators: 72057594046447618 + ExplicitAllocators: 72057594046447619 +} +StateStorage { + SSId: 1 + Ring { + NToSelect: 5 + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + } +} +HiveConfig { + HiveUid: 1 + Hive: 72057594037968897 +} + diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-features.txt b/deploy/8_node_cluster/nbs/cfg/nbs-features.txt new file mode 100644 index 00000000000..3dbc0d949f9 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-features.txt @@ -0,0 +1,12 @@ +Features { + Name: "ChangeThrottlingPolicy" + Whitelist { + CloudIds: "my_cloud" + } +} +Features { + Name: "ReplaceDevice" + Whitelist { + CloudIds: "my_cloud" + } +} diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-http-proxy.txt b/deploy/8_node_cluster/nbs/cfg/nbs-http-proxy.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-http-proxy.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-iam.txt b/deploy/8_node_cluster/nbs/cfg/nbs-iam.txt new file mode 100755 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-iam.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-ic.txt b/deploy/8_node_cluster/nbs/cfg/nbs-ic.txt new file mode 100644 index 00000000000..e17991f6fdc --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-ic.txt @@ -0,0 +1,6 @@ +StartTcp: true +MaxInflightAmountOfDataInKB: 10240 +MergePerPeerCounters: true +HandshakeTimeoutDuration { + Seconds: 1 +} diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-log.txt b/deploy/8_node_cluster/nbs/cfg/nbs-log.txt new file mode 100644 index 00000000000..f6fc7826054 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-log.txt @@ -0,0 +1,104 @@ +Entry { + Component: "BLOCKSTORE_SERVICE" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_AUTH" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_DISK_REGISTRY" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_SCHEMESHARD" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_PARTITION" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_PARTITION_WORKER" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_DISK_AGENT" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_CLIENT" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_VOLUME" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_NBD" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_VOLUME_PROXY" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_SERVICE_PROXY" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_HIVE_PROXY" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_TRACE" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_SCHEDULER" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_SS_PROXY" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_DISK_REGISTRY_PROXY" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_DISK_AGENT_WORKER" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_SERVER" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_BOOTSTRAPPER" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_DISK_REGISTRY_WORKER" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_VHOST" + Level: 6 +} +Entry { + Component: "BLOCKSTORE_RDMA" + Level: 6 +} +Entry { + Component: "INTERCONNECT_NETWORK" + Level: 5 +} +Entry { + Component: "BLOCKSTORE_SPDK" + Level: 6 +} +SysLog: true +DefaultLevel: 4 +SysLogService: "NBS_SERVER" +TimeThresholdMs: 10000 diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-logbroker.txt b/deploy/8_node_cluster/nbs/cfg/nbs-logbroker.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-logbroker.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-names.txt b/deploy/8_node_cluster/nbs/cfg/nbs-names.txt new file mode 100644 index 00000000000..a845df4182a --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-names.txt @@ -0,0 +1,52 @@ +Node { + NodeId: 1 + Port: 19001 + Host: "my.host1" + InterconnectHost: "my.host1" +} +Node { + NodeId: 2 + Port: 19001 + Host: "my.host2" + InterconnectHost: "my.host2" +} +Node { + NodeId: 3 + Port: 19001 + Host: "my.host3" + InterconnectHost: "my.host3" +} +Node { + NodeId: 4 + Port: 19001 + Host: "my.host4" + InterconnectHost: "my.host4" +} +Node { + NodeId: 5 + Port: 19001 + Host: "my.host5" + InterconnectHost: "my.host5" +} +Node { + NodeId: 6 + Port: 19001 + Host: "my.host6" + InterconnectHost: "my.host6" +} +Node { + NodeId: 7 + Port: 19001 + Host: "my.host7" + InterconnectHost: "my.host7" +} +Node { + NodeId: 8 + Port: 19001 + Host: "my.host8" + InterconnectHost: "my.host8" +} +ClusterUUID: "2452520" +AcceptUUID: "2452520" +SuppressVersionCheck: true + diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-notify.txt b/deploy/8_node_cluster/nbs/cfg/nbs-notify.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-notify.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-rdma.txt b/deploy/8_node_cluster/nbs/cfg/nbs-rdma.txt new file mode 100755 index 00000000000..6f758faeef8 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-rdma.txt @@ -0,0 +1,15 @@ +ClientEnabled: true +Client { + QueueSize: 256 + MaxBufferSize: 4198400 + PollerThreads: 1 + AlignedDataEnabled: true +} +ServerEnabled: true +Server { + Backlog: 64 + QueueSize: 512 + MaxBufferSize: 4198400 + WaitMode: WAIT_MODE_ADAPTIVE_WAIT + MaxInflightBytes: 536870912 +} diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-server.txt b/deploy/8_node_cluster/nbs/cfg/nbs-server.txt new file mode 100644 index 00000000000..250ed53a094 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-server.txt @@ -0,0 +1,27 @@ +ServerConfig { + Host: "*" + Port: 9766 + DataHost: "localhost" + DataPort: 9767 + StrictContractValidation: true + UnixSocketPath: "/var/run/nbs/grpc.sock" + VhostEnabled: true + VhostThreadsCount: 4 + NodeType: "nbs" + EndpointStorageType: ENDPOINT_STORAGE_FILE + EndpointStorageDir: "/var/run/nbs/endpoints" + NbdEnabled: true + NbdThreadsCount: 4 + NbdSocketSuffix: "_nbd" + GrpcKeepAliveTime: 10000 + GrpcKeepAliveTimeout: 1000 + GrpcKeepAlivePermitWithoutCalls: true + GrpcHttp2MinRecvPingIntervalWithoutData: 1000 + RdmaClientEnabled: true + RdmaClientConfig { + QueueSize: 256 + MaxBufferSize: 4198400 + PollerThreads: 1 + } + VhostServerPath: "/usr/bin/blockstore-vhost-server" +} diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-shared-cache.txt b/deploy/8_node_cluster/nbs/cfg/nbs-shared-cache.txt new file mode 100755 index 00000000000..7802c7a258b --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-shared-cache.txt @@ -0,0 +1 @@ +MemoryLimit: 2000000000 diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-stats-upload.txt b/deploy/8_node_cluster/nbs/cfg/nbs-stats-upload.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-stats-upload.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-storage.txt b/deploy/8_node_cluster/nbs/cfg/nbs-storage.txt new file mode 100644 index 00000000000..4c57bddf823 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-storage.txt @@ -0,0 +1,46 @@ +SchemeShardDir: "/my_cluster/NBS" +DisableLocalService: false +CompactionGarbageThreshold: 20 +AllowVersionInModifyScheme: true +ThrottlingEnabled: true +ThrottlingEnabledSSD: true +SSDSystemChannelPoolKind: "ssdmirror" +SSDLogChannelPoolKind: "ssdmirror" +SSDIndexChannelPoolKind: "ssdmirror" +HybridMergedChannelPoolKind: "rotmirror" +AllocateSeparateMixedChannels: true +WriteRequestBatchingEnabled: true +SSDCompactionType: CT_LOAD +HDDCompactionType: CT_LOAD +V1GarbageCompactionEnabled: true +VolumePreemptionType: PREEMPTION_MOVE_MOST_HEAVY +PreemptionPushPercentage: 80 +PreemptionPullPercentage: 40 +PoolKindChangeAllowed: true +AcquireNonReplicatedDevices: true +MaxDisksInPlacementGroup: 3 +SSDMaxBlobsPerRange: 20 +IncrementalCompactionEnabled: true +SSDFreshChannelPoolKind: "ssdmirror" +FreshChannelCount: 1 +FreshChannelWriteRequestsEnabled: true +RunV2SoftGcAtStartup: true +MultipartitionVolumesEnabled: true +NonReplicatedMigrationStartAllowed: true +TabletBootInfoBackupFilePath: "/tmp/nbs/backups/nbs-tablet-boot-info-backup.txt" +PathDescriptionBackupFilePath: "/tmp/nbs/backups/nbs-path-description-backup.txt" +CpuLackThreshold: 1000 +UseNonreplicatedRdmaActor: true +BatchCompactionEnabled: true +BlobPatchingEnabled: false +UseMirrorResync: true +AddingUnconfirmedBlobsEnabled: true +BlobCompressionRate: 1000 +MirroredMigrationStartAllowed: true +AutomaticallyReplacedDevicesFreezePeriod: 172800000 +SSDMaxBlobsPerUnit: 180000 +DiskPrefixLengthWithBlockChecksumsInBlobs: 1073741824 +CheckBlockChecksumsInBlobsUponRead: true +DataScrubbingEnabled: true +CleanupDRConfigOnCMSActions: true +ResyncRangeAfterScrubbing: true diff --git a/deploy/8_node_cluster/nbs/cfg/nbs-sys.txt b/deploy/8_node_cluster/nbs/cfg/nbs-sys.txt new file mode 100644 index 00000000000..fba3e8de2ee --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs-sys.txt @@ -0,0 +1,55 @@ +Executor { + Type: BASIC + Threads: 3 + SpinThreshold: 10 + Name: "System" +} +Executor { + Type: BASIC + Threads: 3 + SpinThreshold: 10 + Name: "User" +} +Executor { + Type: BASIC + Threads: 1 + SpinThreshold: 1 + Name: "Batch" +} +Executor { + Type: IO + Threads: 1 + Name: "IO" +} +Executor { + Type: BASIC + Threads: 3 + SpinThreshold: 10 + Name: "IC" + TimePerMailboxMicroSecs: 100 +} +Scheduler { + Resolution: 64 + SpinThreshold: 0 + ProgressThreshold: 10000 +} +SysExecutor: 0 +UserExecutor: 1 +IoExecutor: 3 +BatchExecutor: 2 +ServiceExecutor { + ServiceName: "User" + ExecutorId: 1 +} +ServiceExecutor { + ServiceName: "Batch" + ExecutorId: 2 +} +ServiceExecutor { + ServiceName: "IO" + ExecutorId: 3 +} +ServiceExecutor { + ServiceName: "Interconnect" + ExecutorId: 4 +} diff --git a/deploy/8_node_cluster/nbs/cfg/nbs_disk_agent.cfg b/deploy/8_node_cluster/nbs/cfg/nbs_disk_agent.cfg new file mode 100644 index 00000000000..32582c92409 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs_disk_agent.cfg @@ -0,0 +1,92 @@ +nbs_domain="my_cluster" +nbs_ic_port="${nbs_ic_port:-"29012"}" +nbs_mon_port="${nbs_mon_port:-"8772"}" +nbs_node_broker_port="${nbs_node_broker_port:-"2135"}" +nbs_mon_address="${nbs_mon_address:-"::"}" +nbs_home="${nbs_home:-"/etc/nbs/nbs-disk-agent"}" +nbs_config="${nbs_home}/cfg" +nbs_system_file="${nbs_config}/nbs-sys.txt" +nbs_log_file="${nbs_config}/nbs-log.txt" +nbs_server_file="${nbs_config}/nbs-server.txt" +nbs_features_file="${nbs_config}/nbs-features.txt" +nbs_storage_file="${nbs_config}/nbs-storage.txt" +nbs_domain_file="${nbs_config}/nbs-domains.txt" +nbs_naming_file="${nbs_config}/nbs-names.txt" +nbs_dynamic_naming_file="${nbs_config}/dyn_ns.txt" +nbs_ic_file="${nbs_config}/nbs-ic.txt" +nbs_auth_file="${nbs_config}/nbs-auth.txt" +nbs_diag_file="${nbs_config}/nbs-diag.txt" +nbs_breakpad_file=${nbs_config}/nbs-breakpad.json +nbs_dr_proxy_file="${nbs_config}/nbs-disk-registry-proxy.txt" +nbs_disk_agent_file="${nbs_config}/nbs-disk-agent.txt" +nbs_location_file="${nbs_config}/nbs-location.txt" +nbs_rdma_file="${nbs_config}/nbs-rdma.txt" + +[ -s /etc/default/nbs_server.custom ] && . /etc/default/nbs_server.custom + +nbs_arg="${nbs_arg} --domain ${nbs_domain}" +nbs_arg="${nbs_arg} --ic-port ${nbs_ic_port}" +nbs_arg="${nbs_arg} --mon-port ${nbs_mon_port}" +nbs_arg="${nbs_arg} --node-broker-port ${nbs_node_broker_port}" +nbs_arg="${nbs_arg} --load-configs-from-cms" +if [ ! -z "${nbs_mon_address}" ]; then + nbs_arg="${nbs_arg}${nbs_mon_address:+ --mon-address ${nbs_mon_address}}" +else + echo "Monitoring address is not defined." +fi +nbs_arg="${nbs_arg}${nbs_system_file:+ --sys-file ${nbs_system_file}}" +nbs_arg="${nbs_arg}${nbs_log_file:+ --log-file ${nbs_log_file}}" +nbs_arg="${nbs_arg}${nbs_server_file:+ --server-file ${nbs_server_file}}" +if [ -f "${nbs_features_file}" ]; then + nbs_arg="${nbs_arg}${nbs_features_file:+ --features-file ${nbs_features_file}}" +fi + +nbs_arg="${nbs_arg}${nbs_storage_file:+ --storage-file ${nbs_storage_file}}" +nbs_arg="${nbs_arg}${nbs_domain_file:+ --domains-file ${nbs_domain_file}}" +nbs_arg="${nbs_arg}${nbs_naming_file:+ --naming-file ${nbs_naming_file}}" +nbs_arg="${nbs_arg}${nbs_ic_file:+ --ic-file ${nbs_ic_file}}" + +if [ -f "${nbs_auth_file}" ]; then + nbs_arg="${nbs_arg}${nbs_auth_file:+ --auth-file ${nbs_auth_file}}" +fi + +if [ -f "${nbs_diag_file}" ]; then + nbs_arg="${nbs_arg}${nbs_diag_file:+ --diag-file ${nbs_diag_file}}" +fi + +if [ -f "${nbs_dynamic_naming_file}" ]; then + nbs_arg="${nbs_arg}${nbs_dynamic_naming_file:+ --dynamic-naming-file ${nbs_dynamic_naming_file}}" +fi + +if [ -f "${nbs_restarts_count_file}" ]; then + nbs_arg="${nbs_arg}${nbs_restarts_count_file:+ --restarts-count-file ${nbs_restarts_count_file}}" +fi + +if [ -f "${nbs_profile_file}" ]; then + nbs_arg="${nbs_arg}${nbs_profile_file:+ --profile-file ${nbs_profile_file}}" +fi + +if [ -f "${nbs_dr_proxy_file}" ]; then + nbs_arg="${nbs_arg}${nbs_dr_proxy_file:+ --dr-proxy-file ${nbs_dr_proxy_file}}" +fi + +if [ -f "${nbs_disk_agent_file}" ]; then + nbs_arg="${nbs_arg}${nbs_disk_agent_file:+ --disk-agent-file ${nbs_disk_agent_file}}" +fi + +if [ -f "${nbs_location_file}" ]; then + nbs_arg="${nbs_arg}${nbs_location_file:+ --location-file ${nbs_location_file}}" +fi + +if [ -f "${nbs_rdma_file}" ]; then + nbs_arg="${nbs_arg}${nbs_rdma_file:+ --rdma-file ${nbs_rdma_file}}" +fi + +nbs_arg="${nbs_arg} --syslog-service NBS_DISK_AGENT" +nbs_arg="${nbs_arg} --lock-memory" + +if [ -f "${nbs_breakpad_file}" ] && [ $(jq .enable ${nbs_breakpad_file}) = true ]; then + nbs_breakpad_arg="${nbs_breakpad_arg} --breakpad-enable" + nbs_log="${nbs_home}/log/nbs.log" + nbs_breakpad_arg="${nbs_breakpad_arg}${nbs_log:+ --log ${nbs_log}}" +fi diff --git a/deploy/8_node_cluster/nbs/cfg/nbs_server.cfg b/deploy/8_node_cluster/nbs/cfg/nbs_server.cfg new file mode 100644 index 00000000000..25df4b53190 --- /dev/null +++ b/deploy/8_node_cluster/nbs/cfg/nbs_server.cfg @@ -0,0 +1,136 @@ +nbs_domain="my_cluster" +nbs_ic_port="${nbs_ic_port:-"29010"}" +nbs_mon_port="${nbs_mon_port:-"8766"}" +nbs_node_broker_port="${nbs_node_broker_port:-"2135"}" +nbs_mon_address="${nbs_mon_address:-"::"}" +nbs_home="${nbs_home:-"/etc/nbs/nbs-server"}" +nbs_config="${nbs_home}/cfg" +nbs_system_file="${nbs_config}/nbs-sys.txt" +nbs_log_file="${nbs_config}/nbs-log.txt" +nbs_server_file="${nbs_config}/nbs-server.txt" +nbs_client_file="${nbs_config}/nbs-client.txt" +nbs_features_file="${nbs_config}/nbs-features.txt" +nbs_storage_file="${nbs_config}/nbs-storage.txt" +nbs_domain_file="${nbs_config}/nbs-domains.txt" +nbs_naming_file="${nbs_config}/nbs-names.txt" +nbs_dynamic_naming_file="${nbs_config}/dyn_ns.txt" +nbs_ic_file="${nbs_config}/nbs-ic.txt" +nbs_auth_file="${nbs_config}/nbs-auth.txt" +nbs_diag_file="${nbs_config}/nbs-diag.txt" +nbs_discovery_file="${nbs_config}/nbs-discovery.txt" +nbs_stats_file="${nbs_config}/nbs-stats-upload.txt" +nbs_breakpad_file=${nbs_config}/nbs-breakpad.json +nbs_dr_proxy_file="${nbs_config}/nbs-disk-registry-proxy.txt" +nbs_disk_agent_file="${nbs_config}/nbs-disk-agent.txt" +nbs_location_file="${nbs_config}/nbs-location.txt" +nbs_logbroker_file="${nbs_config}/nbs-logbroker.txt" +nbs_notify_file="${nbs_config}/nbs-notify.txt" +nbs_iam_file="${nbs_config}/nbs-iam.txt" +nbs_kms_file="${nbs_config}/nbs-kms.txt" +nbs_compute_file="${nbs_config}/nbs-compute.txt" +nbs_shared_cache_file="${nbs_config}/nbs-shared-cache.txt" +nbs_rdma_file="${nbs_config}/nbs-rdma.txt" + +[ -s /etc/default/nbs_server.custom ] && . /etc/default/nbs_server.custom + +nbs_arg="${nbs_arg} --domain ${nbs_domain}" +nbs_arg="${nbs_arg} --ic-port ${nbs_ic_port}" +nbs_arg="${nbs_arg} --mon-port ${nbs_mon_port}" +nbs_arg="${nbs_arg} --node-broker-port ${nbs_node_broker_port}" +nbs_arg="${nbs_arg} --load-configs-from-cms" +if [ ! -z "${nbs_mon_address}" ]; then + nbs_arg="${nbs_arg}${nbs_mon_address:+ --mon-address ${nbs_mon_address}}" +else + echo "Monitoring address is not defined." +fi +nbs_arg="${nbs_arg}${nbs_system_file:+ --sys-file ${nbs_system_file}}" +nbs_arg="${nbs_arg}${nbs_log_file:+ --log-file ${nbs_log_file}}" +nbs_arg="${nbs_arg}${nbs_server_file:+ --server-file ${nbs_server_file}}" +if [ -f "${nbs_features_file}" ]; then + nbs_arg="${nbs_arg}${nbs_features_file:+ --features-file ${nbs_features_file}}" +fi + +nbs_arg="${nbs_arg}${nbs_storage_file:+ --storage-file ${nbs_storage_file}}" +nbs_arg="${nbs_arg}${nbs_domain_file:+ --domains-file ${nbs_domain_file}}" +nbs_arg="${nbs_arg}${nbs_naming_file:+ --naming-file ${nbs_naming_file}}" +nbs_arg="${nbs_arg}${nbs_ic_file:+ --ic-file ${nbs_ic_file}}" + +if [ -f "${nbs_auth_file}" ]; then + nbs_arg="${nbs_arg}${nbs_auth_file:+ --auth-file ${nbs_auth_file}}" +fi + +if [ -f "${nbs_diag_file}" ]; then + nbs_arg="${nbs_arg}${nbs_diag_file:+ --diag-file ${nbs_diag_file}}" +fi + +if [ -f "${nbs_dynamic_naming_file}" ]; then + nbs_arg="${nbs_arg}${nbs_dynamic_naming_file:+ --dynamic-naming-file ${nbs_dynamic_naming_file}}" +fi + +if [ -f "${nbs_discovery_file}" ]; then + nbs_arg="${nbs_arg}${nbs_discovery_file:+ --discovery-file ${nbs_discovery_file}}" +fi + +if [ -f "${nbs_stats_file}" ]; then + nbs_arg="${nbs_arg}${nbs_stats_file:+ --ydbstats-file ${nbs_stats_file}}" +fi + +if [ -f "${nbs_restarts_count_file}" ]; then + nbs_arg="${nbs_arg}${nbs_restarts_count_file:+ --restarts-count-file ${nbs_restarts_count_file}}" +fi + +if [ -f "${nbs_profile_file}" ]; then + nbs_arg="${nbs_arg}${nbs_profile_file:+ --profile-file ${nbs_profile_file}}" +fi + +if [ -f "${nbs_dr_proxy_file}" ]; then + nbs_arg="${nbs_arg}${nbs_dr_proxy_file:+ --dr-proxy-file ${nbs_dr_proxy_file}}" +fi + +if [ -f "${nbs_disk_agent_file}" ]; then + nbs_arg="${nbs_arg}${nbs_disk_agent_file:+ --disk-agent-file ${nbs_disk_agent_file}}" +fi + +if [ -f "${nbs_location_file}" ]; then + nbs_arg="${nbs_arg}${nbs_location_file:+ --location-file ${nbs_location_file}}" +fi + +if [ -f "${nbs_logbroker_file}" ]; then + nbs_arg="${nbs_arg}${nbs_logbroker_file:+ --logbroker-file ${nbs_logbroker_file}}" +fi + +if [ -f "${nbs_notify_file}" ]; then + nbs_arg="${nbs_arg}${nbs_notify_file:+ --notify-file ${nbs_notify_file}}" +fi + +if [ -f "${nbs_shared_cache_file}" ]; then + nbs_arg="${nbs_arg}${nbs_shared_cache_file:+ --shared-cache-file ${nbs_shared_cache_file}}" +fi + +if [ -f "${nbs_client_file}" ]; then + nbs_arg="${nbs_arg}${nbs_client_file:+ --client-file ${nbs_client_file}}" +fi + +if [ -f "${nbs_iam_file}" ]; then + nbs_arg="${nbs_arg}${nbs_iam_file:+ --iam-file ${nbs_iam_file}}" +fi + +if [ -f "${nbs_kms_file}" ]; then + nbs_arg="${nbs_arg}${nbs_kms_file:+ --kms-file ${nbs_kms_file}}" +fi + +if [ -f "${nbs_compute_file}" ]; then + nbs_arg="${nbs_arg}${nbs_compute_file:+ --compute-file ${nbs_compute_file}}" +fi + +if [ -f "${nbs_rdma_file}" ]; then + nbs_arg="${nbs_arg}${nbs_rdma_file:+ --rdma-file ${nbs_rdma_file}}" +fi + +nbs_arg="${nbs_arg} --lock-memory" + +if [ -f "${nbs_breakpad_file}" ] && [ $(jq .enable ${nbs_breakpad_file}) = true ]; then + nbs_breakpad_arg="${nbs_breakpad_arg} --breakpad-enable" + nbs_log="${nbs_home}/log/nbs.log" + nbs_breakpad_arg="${nbs_breakpad_arg}${nbs_log:+ --log ${nbs_log}}" +fi diff --git a/deploy/8_node_cluster/nfs-control/cfg/nfs-auth.txt b/deploy/8_node_cluster/nfs-control/cfg/nfs-auth.txt new file mode 100755 index 00000000000..88d4dc166fe --- /dev/null +++ b/deploy/8_node_cluster/nfs-control/cfg/nfs-auth.txt @@ -0,0 +1,3 @@ +UseBlackBox: false +UseAccessService: true +UseStaff: false diff --git a/deploy/8_node_cluster/nfs-control/cfg/nfs-diag.txt b/deploy/8_node_cluster/nfs-control/cfg/nfs-diag.txt new file mode 100755 index 00000000000..61163dae78a --- /dev/null +++ b/deploy/8_node_cluster/nfs-control/cfg/nfs-diag.txt @@ -0,0 +1,19 @@ +SamplingRate: 100000 +SlowRequestSamplingRate: 1000 +LWTraceShuttleCount: 50000 +RequestThresholds { + Default: 1000 +} +RequestThresholds { + Default: 1000 + MediaKind: STORAGE_MEDIA_HDD +} +RequestThresholds { + Default: 100 + ByRequestType { + key: "Compaction" + value: 1000 + } + MediaKind: STORAGE_MEDIA_SSD +} +CpuWaitServiceName: "nfs-server" diff --git a/deploy/8_node_cluster/nfs-control/cfg/nfs-domains.txt b/deploy/8_node_cluster/nfs-control/cfg/nfs-domains.txt new file mode 100644 index 00000000000..199358ecec0 --- /dev/null +++ b/deploy/8_node_cluster/nfs-control/cfg/nfs-domains.txt @@ -0,0 +1,128 @@ +Domain { + DomainId: 1 + SchemeRoot: 72057594046678944 + SSId: 1 + HiveUid: 1 + PlanResolution: 10 + Name: "my_cluster" + StoragePoolTypes { + Kind: "ssd" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssd" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "ssdmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssdmirror" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "rot" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rot" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + StoragePoolTypes { + Kind: "rotmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rotmirror" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + ExplicitMediators: 72057594046382081 + ExplicitMediators: 72057594046382082 + ExplicitMediators: 72057594046382083 + ExplicitCoordinators: 72057594046316545 + ExplicitCoordinators: 72057594046316546 + ExplicitCoordinators: 72057594046316547 + ExplicitAllocators: 72057594046447617 + ExplicitAllocators: 72057594046447618 + ExplicitAllocators: 72057594046447619 +} +StateStorage { + SSId: 1 + Ring { + NToSelect: 5 + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + } +} +HiveConfig { + HiveUid: 1 + Hive: 72057594037968897 +} + diff --git a/deploy/8_node_cluster/nfs-control/cfg/nfs-features.txt b/deploy/8_node_cluster/nfs-control/cfg/nfs-features.txt new file mode 100755 index 00000000000..a1601a86f7c --- /dev/null +++ b/deploy/8_node_cluster/nfs-control/cfg/nfs-features.txt @@ -0,0 +1,3 @@ +Features { + Name: "UseDefaultCompactionPolicy" +} diff --git a/deploy/8_node_cluster/nfs-control/cfg/nfs-ic.txt b/deploy/8_node_cluster/nfs-control/cfg/nfs-ic.txt new file mode 100755 index 00000000000..e17991f6fdc --- /dev/null +++ b/deploy/8_node_cluster/nfs-control/cfg/nfs-ic.txt @@ -0,0 +1,6 @@ +StartTcp: true +MaxInflightAmountOfDataInKB: 10240 +MergePerPeerCounters: true +HandshakeTimeoutDuration { + Seconds: 1 +} diff --git a/deploy/8_node_cluster/nfs-control/cfg/nfs-log.txt b/deploy/8_node_cluster/nfs-control/cfg/nfs-log.txt new file mode 100755 index 00000000000..9b127091d6c --- /dev/null +++ b/deploy/8_node_cluster/nfs-control/cfg/nfs-log.txt @@ -0,0 +1,39 @@ +Entry { + Component: "NFS_TABLET_WORKER" + Level: 6 +} +Entry { + Component: "NFS_SERVICE" + Level: 6 +} +Entry { + Component: "NFS_TABLET" + Level: 6 +} +Entry { + Component: "NFS_SERVICE_PROXY" + Level: 6 +} +Entry { + Component: "NFS_SERVICE_WORKER" + Level: 6 +} +Entry { + Component: "NFS_SERVER" + Level: 6 +} +Entry { + Component: "NFS_TABLET_PROXY" + Level: 6 +} +Entry { + Component: "INTERCONNECT_NETWORK" + Level: 5 +} +Entry { + Component: "NFS_TRACE" + Level: 6 +} +SysLog: true +DefaultLevel: 4 +SysLogService: "NFS_SERVER" diff --git a/deploy/8_node_cluster/nfs-control/cfg/nfs-names.txt b/deploy/8_node_cluster/nfs-control/cfg/nfs-names.txt new file mode 100644 index 00000000000..a845df4182a --- /dev/null +++ b/deploy/8_node_cluster/nfs-control/cfg/nfs-names.txt @@ -0,0 +1,52 @@ +Node { + NodeId: 1 + Port: 19001 + Host: "my.host1" + InterconnectHost: "my.host1" +} +Node { + NodeId: 2 + Port: 19001 + Host: "my.host2" + InterconnectHost: "my.host2" +} +Node { + NodeId: 3 + Port: 19001 + Host: "my.host3" + InterconnectHost: "my.host3" +} +Node { + NodeId: 4 + Port: 19001 + Host: "my.host4" + InterconnectHost: "my.host4" +} +Node { + NodeId: 5 + Port: 19001 + Host: "my.host5" + InterconnectHost: "my.host5" +} +Node { + NodeId: 6 + Port: 19001 + Host: "my.host6" + InterconnectHost: "my.host6" +} +Node { + NodeId: 7 + Port: 19001 + Host: "my.host7" + InterconnectHost: "my.host7" +} +Node { + NodeId: 8 + Port: 19001 + Host: "my.host8" + InterconnectHost: "my.host8" +} +ClusterUUID: "2452520" +AcceptUUID: "2452520" +SuppressVersionCheck: true + diff --git a/deploy/8_node_cluster/nfs-control/cfg/nfs-server.txt b/deploy/8_node_cluster/nfs-control/cfg/nfs-server.txt new file mode 100755 index 00000000000..2f1931b55db --- /dev/null +++ b/deploy/8_node_cluster/nfs-control/cfg/nfs-server.txt @@ -0,0 +1,6 @@ +ServerConfig { + Host: "*" + Port: 9021 +} +KikimrServiceConfig { +} diff --git a/deploy/8_node_cluster/nfs-control/cfg/nfs-storage.txt b/deploy/8_node_cluster/nfs-control/cfg/nfs-storage.txt new file mode 100755 index 00000000000..787ccd665ba --- /dev/null +++ b/deploy/8_node_cluster/nfs-control/cfg/nfs-storage.txt @@ -0,0 +1,17 @@ +SchemeShardDir: "/my_cluster/NFS" +CleanupThreshold: 512 +CompactionThreshold: 10 +ThrottlingEnabled: true +SSDThrottlingEnabled: true +HDDThrottlingEnabled: true +TwoStageReadEnabled: true +MaxBackpressureErrorsBeforeSuicide: 1000000000 +NewCompactionEnabled: true +NewCleanupEnabled: true +ThreeStageWriteEnabled: true +ReadAheadCacheRangeSize: 4194304 +NewLocalDBCompactionPolicyEnabled: true +NodeIndexCacheMaxNodes: 10000 +PreferredBlockSizeMultiplier: 64 +MultiTabletForwardingEnabled: true +BlobCompressionRate: 1000 diff --git a/deploy/8_node_cluster/nfs-control/cfg/nfs-sys.txt b/deploy/8_node_cluster/nfs-control/cfg/nfs-sys.txt new file mode 100755 index 00000000000..2ccc2723329 --- /dev/null +++ b/deploy/8_node_cluster/nfs-control/cfg/nfs-sys.txt @@ -0,0 +1,43 @@ +Executor { + Type: BASIC + Threads: 10 + SpinThreshold: 10 + Name: "System" +} +Executor { + Type: BASIC + Threads: 10 + SpinThreshold: 10 + Name: "User" +} +Executor { + Type: BASIC + Threads: 1 + SpinThreshold: 1 + Name: "Batch" +} +Executor { + Type: IO + Threads: 1 + Name: "IO" +} +Executor { + Type: BASIC + Threads: 10 + SpinThreshold: 10 + Name: "IC" + TimePerMailboxMicroSecs: 100 +} +Scheduler { + Resolution: 64 + SpinThreshold: 0 + ProgressThreshold: 10000 +} +SysExecutor: 0 +UserExecutor: 1 +IoExecutor: 3 +BatchExecutor: 2 +ServiceExecutor { + ServiceName: "Interconnect" + ExecutorId: 4 +} diff --git a/deploy/8_node_cluster/nfs-control/cfg/nfs_server.cfg b/deploy/8_node_cluster/nfs-control/cfg/nfs_server.cfg new file mode 100755 index 00000000000..2da4c522e0d --- /dev/null +++ b/deploy/8_node_cluster/nfs-control/cfg/nfs_server.cfg @@ -0,0 +1,71 @@ +nfs_home="${nfs_home:-"/etc/nbs/nfs-server"}" +nfs_config="${nfs_home}/cfg" +nfs_config_file="${nfs_config}/nfs-server.txt" +nfs_breakpad_file="${nfs_config}/nfs-breakpad.json" + +nfs_mon_address="::" +nfs_mon_port="8767" + +nfs_domain="my_cluster" +nfs_ic_port="29021" +nfs_node_broker_port="2135" +nfs_system_file="${nfs_config}/nfs-sys.txt" +nfs_log_file="${nfs_config}/nfs-log.txt" +nfs_domain_file="${nfs_config}/nfs-domains.txt" +nfs_naming_file="${nfs_config}/nfs-names.txt" +nfs_ic_file="${nfs_config}/nfs-ic.txt" +nfs_storage_file="${nfs_config}/nfs-storage.txt" +nfs_auth_file="${nfs_config}/nfs-auth.txt" +nfs_diag_file="${nfs_config}/nfs-diag.txt" +nfs_features_file="${nfs_config}/nfs-features.txt" +nfs_dynamic_naming_file="${nfs_config}/dyn_ns.txt" +nfs_profile_file="/var/log/nfs-server/nfs-profile.log" + +[ -s /etc/default/nfs_server.custom ] && . /etc/default/nfs_server.custom + +nfs_arg="${nfs_arg}${nfs_config_file:+ --app-config ${nfs_config_file}}" +nfs_arg="${nfs_arg} --mon-port ${nfs_mon_port}" +if [ ! -z "${nfs_mon_address}" ]; then + nfs_arg="${nfs_arg}${nfs_mon_address:+ --mon-address ${nfs_mon_address}}" +else + echo "Monitoring address is not defined." +fi + +nfs_arg="${nfs_arg} --domain ${nfs_domain}" +nfs_arg="${nfs_arg} --node-broker-port ${nfs_node_broker_port}" +nfs_arg="${nfs_arg} --ic-port ${nfs_ic_port}" + +nfs_arg="${nfs_arg}${nfs_system_file:+ --sys-file ${nfs_system_file}}" +nfs_arg="${nfs_arg}${nfs_log_file:+ --log-file ${nfs_log_file}}" +nfs_arg="${nfs_arg}${nfs_domain_file:+ --domains-file ${nfs_domain_file}}" +nfs_arg="${nfs_arg}${nfs_naming_file:+ --naming-file ${nfs_naming_file}}" +nfs_arg="${nfs_arg}${nfs_ic_file:+ --ic-file ${nfs_ic_file}}" +nfs_arg="${nfs_arg}${nfs_storage_file:+ --storage-file ${nfs_storage_file}}" +nfs_arg="${nfs_arg}${nfs_diag_file:+ --diag-file ${nfs_diag_file}}" +nfs_arg="${nfs_arg}${nfs_profile_file:+ --profile-file ${nfs_profile_file}}" + +if [ -f "${nfs_auth_file}" ]; then + nfs_arg="${nfs_arg}${nfs_auth_file:+ --auth-file ${nfs_auth_file}}" +fi + +if [ -f "${nfs_dynamic_naming_file}" ]; then + nfs_arg="${nfs_arg}${nfs_dynamic_naming_file:+ --dynamic-naming-file ${nfs_dynamic_naming_file}}" +fi + +if [ -f "${nfs_restarts_count_file}" ]; then + nfs_arg="${nfs_arg}${nfs_restarts_count_file:+ --restarts-count-file ${nfs_restarts_count_file}}" +fi + +if [ -f "${nfs_features_file}" ]; then + nfs_arg="${nfs_arg}${nfs_features_file:+ --features-file ${nfs_features_file}}" +fi + +nfs_arg="${nfs_arg} --lock-memory" +nfs_arg="${nfs_arg} --load-configs-from-cms" + +if [ -f "${nfs_breakpad_file}" ] && [ $(jq .enable ${nfs_breakpad_file}) = true ]; then + nfs_breakpad_dir="/var/tmp/breakpad-nfs" + nfs_breakpad_arg="${nfs_breakpad_arg} --breakpad-enable --datadir=$nfs_breakpad_dir --breakpad-coredir=$nfs_breakpad_dir" + nfs_log="${nfs_home}/log/nfs-server.log" + nfs_breakpad_arg="${nfs_breakpad_arg}${nfs_log:+ --log ${nfs_log}}" +fi diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-auth.txt b/deploy/8_node_cluster/nfs/cfg/nfs-auth.txt new file mode 100644 index 00000000000..88d4dc166fe --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-auth.txt @@ -0,0 +1,3 @@ +UseBlackBox: false +UseAccessService: true +UseStaff: false diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-diag.txt b/deploy/8_node_cluster/nfs/cfg/nfs-diag.txt new file mode 100644 index 00000000000..5773b72de7e --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-diag.txt @@ -0,0 +1,19 @@ +SamplingRate: 100000 +SlowRequestSamplingRate: 1000 +LWTraceShuttleCount: 50000 +RequestThresholds { + Default: 1000 +} +RequestThresholds { + Default: 1000 + MediaKind: STORAGE_MEDIA_HDD +} +RequestThresholds { + Default: 100 + ByRequestType { + key: "Compaction" + value: 1000 + } + MediaKind: STORAGE_MEDIA_SSD +} +CpuWaitServiceName: "nfs-vhost" diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-domains.txt b/deploy/8_node_cluster/nfs/cfg/nfs-domains.txt new file mode 100644 index 00000000000..199358ecec0 --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-domains.txt @@ -0,0 +1,128 @@ +Domain { + DomainId: 1 + SchemeRoot: 72057594046678944 + SSId: 1 + HiveUid: 1 + PlanResolution: 10 + Name: "my_cluster" + StoragePoolTypes { + Kind: "ssd" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssd" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "ssdmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssdmirror" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "rot" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rot" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + StoragePoolTypes { + Kind: "rotmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rotmirror" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + ExplicitMediators: 72057594046382081 + ExplicitMediators: 72057594046382082 + ExplicitMediators: 72057594046382083 + ExplicitCoordinators: 72057594046316545 + ExplicitCoordinators: 72057594046316546 + ExplicitCoordinators: 72057594046316547 + ExplicitAllocators: 72057594046447617 + ExplicitAllocators: 72057594046447618 + ExplicitAllocators: 72057594046447619 +} +StateStorage { + SSId: 1 + Ring { + NToSelect: 5 + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + } +} +HiveConfig { + HiveUid: 1 + Hive: 72057594037968897 +} + diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-http-proxy.txt b/deploy/8_node_cluster/nfs/cfg/nfs-http-proxy.txt new file mode 100644 index 00000000000..7c402d87df9 --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-http-proxy.txt @@ -0,0 +1,3 @@ +Port: 9023 +NfsVhostHost: "localhost-expanded" +NfsVhostPort: 9022 diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-ic.txt b/deploy/8_node_cluster/nfs/cfg/nfs-ic.txt new file mode 100644 index 00000000000..e17991f6fdc --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-ic.txt @@ -0,0 +1,6 @@ +StartTcp: true +MaxInflightAmountOfDataInKB: 10240 +MergePerPeerCounters: true +HandshakeTimeoutDuration { + Seconds: 1 +} diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-log.txt b/deploy/8_node_cluster/nfs/cfg/nfs-log.txt new file mode 100644 index 00000000000..9b127091d6c --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-log.txt @@ -0,0 +1,39 @@ +Entry { + Component: "NFS_TABLET_WORKER" + Level: 6 +} +Entry { + Component: "NFS_SERVICE" + Level: 6 +} +Entry { + Component: "NFS_TABLET" + Level: 6 +} +Entry { + Component: "NFS_SERVICE_PROXY" + Level: 6 +} +Entry { + Component: "NFS_SERVICE_WORKER" + Level: 6 +} +Entry { + Component: "NFS_SERVER" + Level: 6 +} +Entry { + Component: "NFS_TABLET_PROXY" + Level: 6 +} +Entry { + Component: "INTERCONNECT_NETWORK" + Level: 5 +} +Entry { + Component: "NFS_TRACE" + Level: 6 +} +SysLog: true +DefaultLevel: 4 +SysLogService: "NFS_SERVER" diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-names.txt b/deploy/8_node_cluster/nfs/cfg/nfs-names.txt new file mode 100644 index 00000000000..a845df4182a --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-names.txt @@ -0,0 +1,52 @@ +Node { + NodeId: 1 + Port: 19001 + Host: "my.host1" + InterconnectHost: "my.host1" +} +Node { + NodeId: 2 + Port: 19001 + Host: "my.host2" + InterconnectHost: "my.host2" +} +Node { + NodeId: 3 + Port: 19001 + Host: "my.host3" + InterconnectHost: "my.host3" +} +Node { + NodeId: 4 + Port: 19001 + Host: "my.host4" + InterconnectHost: "my.host4" +} +Node { + NodeId: 5 + Port: 19001 + Host: "my.host5" + InterconnectHost: "my.host5" +} +Node { + NodeId: 6 + Port: 19001 + Host: "my.host6" + InterconnectHost: "my.host6" +} +Node { + NodeId: 7 + Port: 19001 + Host: "my.host7" + InterconnectHost: "my.host7" +} +Node { + NodeId: 8 + Port: 19001 + Host: "my.host8" + InterconnectHost: "my.host8" +} +ClusterUUID: "2452520" +AcceptUUID: "2452520" +SuppressVersionCheck: true + diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-options.json b/deploy/8_node_cluster/nfs/cfg/nfs-options.json new file mode 100644 index 00000000000..6a03f510d11 --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-options.json @@ -0,0 +1 @@ +{"vhost_service": "kikimr"} diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-server.txt b/deploy/8_node_cluster/nfs/cfg/nfs-server.txt new file mode 100644 index 00000000000..2f1931b55db --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-server.txt @@ -0,0 +1,6 @@ +ServerConfig { + Host: "*" + Port: 9021 +} +KikimrServiceConfig { +} diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-storage-vhost.txt b/deploy/8_node_cluster/nfs/cfg/nfs-storage-vhost.txt new file mode 100644 index 00000000000..1dde02b7a3c --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-storage-vhost.txt @@ -0,0 +1,4 @@ +SchemeShardDir: "/my_cluster/NFS" +CleanupThreshold: 512 +CompactionThreshold: 10 +TwoStageReadEnabled: true diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-storage.txt b/deploy/8_node_cluster/nfs/cfg/nfs-storage.txt new file mode 100644 index 00000000000..76288eae948 --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-storage.txt @@ -0,0 +1,5 @@ +SchemeShardDir: "/my_cluster/NFS" +DisableLocalService: true +MultiTabletForwardingEnabled: true +GetNodeAttrBatchEnabled: true +UnalignedThreeStageWriteEnabled: true diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-sys.txt b/deploy/8_node_cluster/nfs/cfg/nfs-sys.txt new file mode 100644 index 00000000000..fba3e8de2ee --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-sys.txt @@ -0,0 +1,55 @@ +Executor { + Type: BASIC + Threads: 3 + SpinThreshold: 10 + Name: "System" +} +Executor { + Type: BASIC + Threads: 3 + SpinThreshold: 10 + Name: "User" +} +Executor { + Type: BASIC + Threads: 1 + SpinThreshold: 1 + Name: "Batch" +} +Executor { + Type: IO + Threads: 1 + Name: "IO" +} +Executor { + Type: BASIC + Threads: 3 + SpinThreshold: 10 + Name: "IC" + TimePerMailboxMicroSecs: 100 +} +Scheduler { + Resolution: 64 + SpinThreshold: 0 + ProgressThreshold: 10000 +} +SysExecutor: 0 +UserExecutor: 1 +IoExecutor: 3 +BatchExecutor: 2 +ServiceExecutor { + ServiceName: "User" + ExecutorId: 1 +} +ServiceExecutor { + ServiceName: "Batch" + ExecutorId: 2 +} +ServiceExecutor { + ServiceName: "IO" + ExecutorId: 3 +} +ServiceExecutor { + ServiceName: "Interconnect" + ExecutorId: 4 +} diff --git a/deploy/8_node_cluster/nfs/cfg/nfs-vhost.txt b/deploy/8_node_cluster/nfs/cfg/nfs-vhost.txt new file mode 100644 index 00000000000..77a263df3ba --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs-vhost.txt @@ -0,0 +1,16 @@ +ServerConfig { + Host: "*" + Port: 9022 +} +VhostServiceConfig { + ServiceEndpoints { + Name: "" + ClientConfig { + Host: "localhost" + Port: 9021 + RetryTimeout: 86400000 + } + } + EndpointStorageType: ENDPOINT_STORAGE_FILE + EndpointStorageDir: "/var/run/nfs-vhost/endpoints" +} diff --git a/deploy/8_node_cluster/nfs/cfg/nfs_server.cfg b/deploy/8_node_cluster/nfs/cfg/nfs_server.cfg new file mode 100644 index 00000000000..b0e76c8d1ea --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs_server.cfg @@ -0,0 +1,71 @@ +nfs_home="${nfs_home:-"/etc/nbs/nfs-server"}" +nfs_config="${nfs_home}/cfg" +nfs_config_file="${nfs_config}/nfs-server.txt" +nfs_breakpad_file="${nfs_config}/nfs-breakpad.json" + +nfs_mon_address="::" +nfs_mon_port="8767" + +nfs_domain="my_cluster" +nfs_ic_port="29021" +nfs_node_broker_port="2135" +nfs_system_file="${nfs_config}/nfs-sys.txt" +nfs_log_file="${nfs_config}/nfs-log.txt" +nfs_domain_file="${nfs_config}/nfs-domains.txt" +nfs_naming_file="${nfs_config}/nfs-names.txt" +nfs_ic_file="${nfs_config}/nfs-ic.txt" +nfs_storage_file="${nfs_config}/nfs-storage.txt" +nfs_auth_file="${nfs_config}/nfs-auth.txt" +nfs_diag_file="${nfs_config}/nfs-diag.txt" +nfs_features_file="${nfs_config}/nfs-features.txt" +nfs_dynamic_naming_file="${nfs_config}/dyn_ns.txt" +nfs_profile_file="/var/log/nfs/nfs-profile.log" + +[ -s /etc/default/nfs_server.custom ] && . /etc/default/nfs_server.custom + +nfs_arg="${nfs_arg}${nfs_config_file:+ --app-config ${nfs_config_file}}" +nfs_arg="${nfs_arg} --mon-port ${nfs_mon_port}" +if [ ! -z "${nfs_mon_address}" ]; then + nfs_arg="${nfs_arg}${nfs_mon_address:+ --mon-address ${nfs_mon_address}}" +else + echo "Monitoring address is not defined." +fi + +nfs_arg="${nfs_arg} --domain ${nfs_domain}" +nfs_arg="${nfs_arg} --node-broker-port ${nfs_node_broker_port}" +nfs_arg="${nfs_arg} --ic-port ${nfs_ic_port}" + +nfs_arg="${nfs_arg}${nfs_system_file:+ --sys-file ${nfs_system_file}}" +nfs_arg="${nfs_arg}${nfs_log_file:+ --log-file ${nfs_log_file}}" +nfs_arg="${nfs_arg}${nfs_domain_file:+ --domains-file ${nfs_domain_file}}" +nfs_arg="${nfs_arg}${nfs_naming_file:+ --naming-file ${nfs_naming_file}}" +nfs_arg="${nfs_arg}${nfs_ic_file:+ --ic-file ${nfs_ic_file}}" +nfs_arg="${nfs_arg}${nfs_storage_file:+ --storage-file ${nfs_storage_file}}" +nfs_arg="${nfs_arg}${nfs_diag_file:+ --diag-file ${nfs_diag_file}}" +nfs_arg="${nfs_arg}${nfs_profile_file:+ --profile-file ${nfs_profile_file}}" + +if [ -f "${nfs_auth_file}" ]; then + nfs_arg="${nfs_arg}${nfs_auth_file:+ --auth-file ${nfs_auth_file}}" +fi + +if [ -f "${nfs_dynamic_naming_file}" ]; then + nfs_arg="${nfs_arg}${nfs_dynamic_naming_file:+ --dynamic-naming-file ${nfs_dynamic_naming_file}}" +fi + +if [ -f "${nfs_restarts_count_file}" ]; then + nfs_arg="${nfs_arg}${nfs_restarts_count_file:+ --restarts-count-file ${nfs_restarts_count_file}}" +fi + +if [ -f "${nfs_features_file}" ]; then + nfs_arg="${nfs_arg}${nfs_features_file:+ --features-file ${nfs_features_file}}" +fi + +nfs_arg="${nfs_arg} --lock-memory" +nfs_arg="${nfs_arg} --load-configs-from-cms" + +if [ -f "${nfs_breakpad_file}" ] && [ $(jq .enable ${nfs_breakpad_file}) = true ]; then + nfs_breakpad_dir="/var/tmp/breakpad-nfs" + nfs_breakpad_arg="${nfs_breakpad_arg} --breakpad-enable --datadir=$nfs_breakpad_dir --breakpad-coredir=$nfs_breakpad_dir" + nfs_log="${nfs_home}/log/nfs-server.log" + nfs_breakpad_arg="${nfs_breakpad_arg}${nfs_log:+ --log ${nfs_log}}" +fi diff --git a/deploy/8_node_cluster/nfs/cfg/nfs_vhost.cfg b/deploy/8_node_cluster/nfs/cfg/nfs_vhost.cfg new file mode 100644 index 00000000000..b3c48c17a73 --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/nfs_vhost.cfg @@ -0,0 +1,72 @@ +vhost_home="${vhost_home:-"/etc/nbs/nfs-vhost"}" +vhost_config="${vhost_home}/cfg" +vhost_config_file="${vhost_config}/nfs-vhost.txt" +vhost_breakpad_file="${vhost_config}/nfs-breakpad.json" +vhost_options_file="${vhost_config}/nfs-options.json" + +vhost_mon_address="::" +vhost_mon_port="8768" + +vhost_domain="my_cluster" +vhost_ic_port="29022" +vhost_node_broker_port="2135" +vhost_system_file="${vhost_config}/nfs-sys.txt" +vhost_log_file="${vhost_config}/vhost-log.txt" +vhost_domain_file="${vhost_config}/nfs-domains.txt" +vhost_naming_file="${vhost_config}/nfs-names.txt" +vhost_ic_file="${vhost_config}/nfs-ic.txt" +vhost_storage_file="${vhost_config}/nfs-storage.txt" +vhost_auth_file="${vhost_config}/nfs-auth.txt" +vhost_diag_file="${vhost_config}/nfs-diag.txt" +vhost_dynamic_naming_file="${vhost_config}/dyn_ns.txt" +vhost_profile_file="/var/log/nfs/vhost-profile.log" + +[ -s /etc/default/nfs_vhost.custom ] && . /etc/default/nfs_vhost.custom + +vhost_arg="${vhost_arg}${vhost_config_file:+ --app-config ${vhost_config_file}}" +vhost_arg="${vhost_arg} --mon-port ${vhost_mon_port}" +if [ ! -z "${vhost_mon_address}" ]; then + vhost_arg="${vhost_arg}${vhost_mon_address:+ --mon-address ${vhost_mon_address}}" +else + echo "Monitoring address is not defined." +fi + +if [ -f "${vhost_restarts_count_file}" ]; then + vhost_arg="${vhost_arg}${vhost_restarts_count_file:+ --restarts-count-file ${vhost_restarts_count_file}}" +fi + +if [ -f "${vhost_options_file}" ] && [ $(jq .vhost_service ${vhost_options_file}) = \"kikimr\" ]; then + vhost_arg="${vhost_arg} --service kikimr" + + vhost_arg="${vhost_arg} --domain ${vhost_domain}" + vhost_arg="${vhost_arg} --node-broker-port ${vhost_node_broker_port}" + vhost_arg="${vhost_arg} --ic-port ${vhost_ic_port}" + + vhost_arg="${vhost_arg}${vhost_system_file:+ --sys-file ${vhost_system_file}}" + vhost_arg="${vhost_arg}${vhost_log_file:+ --log-file ${vhost_log_file}}" + vhost_arg="${vhost_arg}${vhost_domain_file:+ --domains-file ${vhost_domain_file}}" + vhost_arg="${vhost_arg}${vhost_naming_file:+ --naming-file ${vhost_naming_file}}" + vhost_arg="${vhost_arg}${vhost_ic_file:+ --ic-file ${vhost_ic_file}}" + vhost_arg="${vhost_arg}${vhost_storage_file:+ --storage-file ${vhost_storage_file}}" + vhost_arg="${vhost_arg}${vhost_diag_file:+ --diag-file ${vhost_diag_file}}" + vhost_arg="${vhost_arg}${vhost_profile_file:+ --profile-file ${vhost_profile_file}}" + + if [ -f "${vhost_auth_file}" ]; then + vhost_arg="${vhost_arg}${vhost_auth_file:+ --auth-file ${vhost_auth_file}}" + fi + + if [ -f "${vhost_dynamic_naming_file}" ]; then + vhost_arg="${vhost_arg}${vhost_dynamic_naming_file:+ --dynamic-naming-file ${vhost_dynamic_naming_file}}" + fi + + vhost_arg="${vhost_arg} --load-configs-from-cms" +fi + +vhost_arg="${vhost_arg} --lock-memory" + +if [ -f "${vhost_breakpad_file}" ] && [ $(jq .enable ${vhost_breakpad_file}) = true ]; then + nfs_breakpad_dir="/var/tmp/breakpad-nfs" + nfs_breakpad_arg="${nfs_breakpad_arg} --breakpad-enable --datadir=$nfs_breakpad_dir --breakpad-coredir=$nfs_breakpad_dir" + vhost_log="${vhost_home}/log/nfs-vhost.log" + nfs_breakpad_arg="${nfs_breakpad_arg}${vhost_log:+ --log ${vhost_log}}" +fi diff --git a/deploy/8_node_cluster/nfs/cfg/vhost-log.txt b/deploy/8_node_cluster/nfs/cfg/vhost-log.txt new file mode 100755 index 00000000000..b0770873dfb --- /dev/null +++ b/deploy/8_node_cluster/nfs/cfg/vhost-log.txt @@ -0,0 +1,27 @@ +Entry { + Component: "NFS_SERVICE" + Level: 6 +} +Entry { + Component: "NFS_SERVICE_PROXY" + Level: 6 +} +Entry { + Component: "NFS_SERVICE_WORKER" + Level: 6 +} +Entry { + Component: "NFS_VHOST" + Level: 6 +} +Entry { + Component: "NFS_FUSE" + Level: 6 +} +Entry { + Component: "NFS_TRACE" + Level: 6 +} +SysLog: true +DefaultLevel: 3 +SysLogService: "NFS_VHOST" diff --git a/deploy/8_node_cluster/storage/cfg/Configure-my_cluster-init-1.txt b/deploy/8_node_cluster/storage/cfg/Configure-my_cluster-init-1.txt new file mode 100644 index 00000000000..d22b211afbe --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/Configure-my_cluster-init-1.txt @@ -0,0 +1,71 @@ +ConfigureRequest { + +Actions { + RemoveConfigItems { + CookieFilter { + Cookies: "cloud-ssdencrypted-base-table-profile" + } + } +} # Actions + +Actions { +AddConfigItem { +ConfigItem { +Cookie: "cloud-ssdencrypted-base-table-profile" +Config { +TableProfilesConfig { + + CompactionPolicies { + Name: "default" + } + ExecutionPolicies { + Name: "default" + } + PartitioningPolicies { + Name: "default" + AutoSplit: true + AutoMerge: false + SizeToSplit: 2147483648 + } + StoragePolicies { + Name: "default" + ColumnFamilies { + StorageConfig { + SysLog { + PreferredPoolKind: "ssdencrypted" + } + Log { + PreferredPoolKind: "ssdencrypted" + } + Data { + PreferredPoolKind: "ssdencrypted" + } + } + } + } + ReplicationPolicies { + Name: "default" + } + CachingPolicies { + Name: "default" + # Note: code-level default is 32MB + # ExecutorCacheSize: 4194304 + } + TableProfiles { + Name: "default" + CompactionPolicy: "default" + ExecutionPolicy: "default" + PartitioningPolicy: "default" + StoragePolicy: "default" + ReplicationPolicy: "default" + CachingPolicy: "default" + } + +} +} +} +} +} # Actions + +} # ConfigureRequest + diff --git a/deploy/8_node_cluster/storage/cfg/Configure-my_cluster.txt b/deploy/8_node_cluster/storage/cfg/Configure-my_cluster.txt new file mode 100644 index 00000000000..565229c81c5 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/Configure-my_cluster.txt @@ -0,0 +1,1248 @@ +ConfigureRequest { + Actions { + AddConfigItem { + ConfigItem { + Config { + ActorSystemConfig { + Executor { + Type: BASIC + Threads: 8 + SpinThreshold: 10 + Name: "System" + } + Executor { + Type: BASIC + Threads: 8 + SpinThreshold: 10 + Name: "User" + } + Executor { + Type: BASIC + Threads: 8 + SpinThreshold: 1 + Name: "Batch" + } + Executor { + Type: IO + Threads: 1 + Name: "IO" + } + Executor { + Type: BASIC + Threads: 3 + SpinThreshold: 10 + Name: "IC" + TimePerMailboxMicroSecs: 100 + } + Scheduler { + Resolution: 256 + SpinThreshold: 0 + ProgressThreshold: 10000 + } + SysExecutor: 0 + UserExecutor: 1 + IoExecutor: 3 + BatchExecutor: 2 + ServiceExecutor { + ServiceName: "Interconnect" + ExecutorId: 4 + } + } + LogConfig { + DefaultLevel: 3 + } + NameserviceConfig { + Node { + NodeId: 1 + Port: 19001 + Host: "my.host1" + InterconnectHost: "my.host1" + Location { + DataCenter: "my_dc" + Rack: "lab-1:1" + Body: 1 + } + } + Node { + NodeId: 2 + Port: 19001 + Host: "my.host2" + InterconnectHost: "my.host2" + Location { + DataCenter: "my_dc" + Rack: "lab-2:1" + Body: 2 + } + } + Node { + NodeId: 3 + Port: 19001 + Host: "my.host3" + InterconnectHost: "my.host3" + Location { + DataCenter: "my_dc" + Rack: "lab-3:1" + Body: 3 + } + } + Node { + NodeId: 4 + Port: 19001 + Host: "my.host4" + InterconnectHost: "my.host4" + Location { + DataCenter: "my_dc" + Rack: "lab-4:1" + Body: 4 + } + } + Node { + NodeId: 5 + Port: 19001 + Host: "my.host5" + InterconnectHost: "my.host5" + Location { + DataCenter: "my_dc" + Rack: "lab-5:1" + Body: 5 + } + } + Node { + NodeId: 6 + Port: 19001 + Host: "my.host6" + InterconnectHost: "my.host6" + Location { + DataCenter: "my_dc" + Rack: "lab-6:1" + Body: 6 + } + } + Node { + NodeId: 7 + Port: 19001 + Host: "my.host7" + InterconnectHost: "my.host7" + Location { + DataCenter: "my_dc" + Rack: "lab-7:1" + Body: 7 + } + } + Node { + NodeId: 8 + Port: 19001 + Host: "my.host8" + InterconnectHost: "my.host8" + Location { + DataCenter: "my_dc" + Rack: "lab-8:1" + Body: 8 + } + } + ClusterUUID: "2452520" + AcceptUUID: "2452520" + } + InterconnectConfig { + StartTcp: true + MaxInflightAmountOfDataInKB: 10240 + HandshakeTimeoutDuration { + Seconds: 1 + } + } + DomainsConfig { + Domain { + DomainId: 1 + SchemeRoot: 72057594046678944 + SSId: 1 + HiveUid: 1 + PlanResolution: 10 + Name: "my_cluster" + StoragePoolTypes { + Kind: "ssd" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssd" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "ssdmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssdmirror" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "rot" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rot" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + StoragePoolTypes { + Kind: "rotmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rotmirror" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + ExplicitMediators: 72057594046382081 + ExplicitMediators: 72057594046382082 + ExplicitMediators: 72057594046382083 + ExplicitCoordinators: 72057594046316545 + ExplicitCoordinators: 72057594046316546 + ExplicitCoordinators: 72057594046316547 + ExplicitAllocators: 72057594046447617 + ExplicitAllocators: 72057594046447618 + ExplicitAllocators: 72057594046447619 + } + StateStorage { + SSId: 1 + Ring { + NToSelect: 5 + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + } + } + HiveConfig { + HiveUid: 1 + Hive: 72057594037968897 + } + } + BlobStorageConfig { + ServiceSet { + PDisks { + NodeID: 1 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 10738122723267084431 + PDiskCategory: 1 + } + PDisks { + NodeID: 2 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 16106689533420805170 + PDiskCategory: 1 + } + PDisks { + NodeID: 3 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 15479661826898200215 + PDiskCategory: 1 + } + PDisks { + NodeID: 4 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 5795881613215917266 + PDiskCategory: 1 + } + PDisks { + NodeID: 5 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 18089278276041197701 + PDiskCategory: 1 + } + PDisks { + NodeID: 6 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 7117908886488453734 + PDiskCategory: 1 + } + PDisks { + NodeID: 7 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 12400000310110999597 + PDiskCategory: 1 + } + PDisks { + NodeID: 8 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 16321399264440587887 + PDiskCategory: 1 + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 0 + VDisk: 0 + } + VDiskLocation { + NodeID: 1 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 10738122723267084431 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 1 + VDisk: 0 + } + VDiskLocation { + NodeID: 2 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16106689533420805170 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 2 + VDisk: 0 + } + VDiskLocation { + NodeID: 3 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 15479661826898200215 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 3 + VDisk: 0 + } + VDiskLocation { + NodeID: 4 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 5795881613215917266 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 4 + VDisk: 0 + } + VDiskLocation { + NodeID: 5 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 18089278276041197701 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 5 + VDisk: 0 + } + VDiskLocation { + NodeID: 6 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 7117908886488453734 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 6 + VDisk: 0 + } + VDiskLocation { + NodeID: 7 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 12400000310110999597 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 7 + VDisk: 0 + } + VDiskLocation { + NodeID: 8 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16321399264440587887 + } + VDiskKind: Default + } + Groups { + GroupID: 0 + GroupGeneration: 1 + ErasureSpecies: 4 + Rings { + FailDomains { + VDiskLocations { + NodeID: 1 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 10738122723267084431 + } + } + FailDomains { + VDiskLocations { + NodeID: 2 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16106689533420805170 + } + } + FailDomains { + VDiskLocations { + NodeID: 3 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 15479661826898200215 + } + } + FailDomains { + VDiskLocations { + NodeID: 4 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 5795881613215917266 + } + } + FailDomains { + VDiskLocations { + NodeID: 5 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 18089278276041197701 + } + } + FailDomains { + VDiskLocations { + NodeID: 6 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 7117908886488453734 + } + } + FailDomains { + VDiskLocations { + NodeID: 7 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 12400000310110999597 + } + } + FailDomains { + VDiskLocations { + NodeID: 8 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16321399264440587887 + } + } + } + } + AvailabilityDomains: 1 + } + } + ChannelProfileConfig { + Profile { + ProfileId: 0 + Channel { + ErasureSpecies: "block-4-2" + PDiskCategory: 1 + VDiskCategory: Default + } + Channel { + ErasureSpecies: "block-4-2" + PDiskCategory: 1 + VDiskCategory: Default + } + Channel { + ErasureSpecies: "block-4-2" + PDiskCategory: 1 + VDiskCategory: Default + } + } + } + BootstrapConfig { + Tablet { + Type: FLAT_HIVE + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037968897 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: FLAT_BS_CONTROLLER + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037932033 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: FLAT_SCHEMESHARD + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046678944 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: FLAT_TX_COORDINATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046316545 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: FLAT_TX_COORDINATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046316546 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: FLAT_TX_COORDINATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046316547 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_MEDIATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046382081 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_MEDIATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046382082 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_MEDIATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046382083 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_ALLOCATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046447617 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_ALLOCATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046447618 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_ALLOCATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046447619 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: CMS + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936128 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: NODE_BROKER + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936129 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TENANT_SLOT_BROKER + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936130 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: CONSOLE + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936131 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + } + VDiskConfig { + VDiskKinds { + Kind: Log + BaseKind: Default + Config { + FreshUseDreg: true + AllowKeepFlags: false + } + } + VDiskKinds { + Kind: LocalMode + BaseKind: Default + Config { + HullCompLevel0MaxSstsAtOnce: 2 + HullCompSortedPartsNum: 1 + } + } + } + KQPConfig { + Enable: true + Settings { + Name: "_ResultRowsLimit" + Value: "10000" + } + } + GRpcConfig { + StartGRpcProxy: true + Host: "[::]" + Port: 2135 + GRpcMemoryQuotaBytes: 1073741824 + StreamingConfig { + EnableOutputStreams: true + } + KeepAliveEnable: true + KeepAliveIdleTimeoutTriggerSec: 90 + KeepAliveMaxProbeCount: 3 + KeepAliveProbeIntervalSec: 10 + } + DynamicNameserviceConfig { + MaxStaticNodeId: 50000 + } + FeatureFlags { + EnableSeparateSolomonShardForPDisk: true + EnableVPatch: true + } + PQConfig { + Enabled: false + } + } + } + EnableAutoSplit: true + } + } + Actions { + AddConfigItem { + ConfigItem { + Config { + TenantPoolConfig { + IsEnabled: true + Slots { + Id: "dynamic-slot-1" + DomainName: "my_cluster" + IsDynamic: true + Type: "default" + } + } + } + UsageScope { + TenantAndNodeTypeFilter { + Tenant: "dynamic" + } + } + } + } + } +} + diff --git a/deploy/8_node_cluster/storage/cfg/Console-Config-my_cluster.txt b/deploy/8_node_cluster/storage/cfg/Console-Config-my_cluster.txt new file mode 100644 index 00000000000..e08654f983d --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/Console-Config-my_cluster.txt @@ -0,0 +1,55 @@ +SetConfigRequest { + Config { + TenantsConfig { + AvailabilityZoneKinds { + Kind: "any" + } + AvailabilityZoneKinds { + Kind: "single" + SlotLocation { + CollocationGroup: 1 + ForceCollocation: true + } + } + AvailabilityZoneKinds { + Kind: "my_dc" + DataCenterName: "my_dc" + } + AvailabilityZoneSets { + Name: "all" + ZoneKinds: "any" + ZoneKinds: "single" + ZoneKinds: "my_dc" + } + ComputationalUnitKinds { + Kind: "slot" + TenantSlotType: "default" + AvailabilityZoneSet: "all" + } + } + ConfigsConfig { + UsageScopeRestrictions { + AllowedNodeIdUsageScopeKinds: 2 + AllowedHostUsageScopeKinds: 2 + AllowedTenantUsageScopeKinds: 1 + AllowedTenantUsageScopeKinds: 2 + AllowedTenantUsageScopeKinds: 5 + AllowedTenantUsageScopeKinds: 20 + AllowedTenantUsageScopeKinds: 26 + AllowedTenantUsageScopeKinds: 29 + AllowedTenantUsageScopeKinds: 35 + AllowedTenantUsageScopeKinds: 38 + AllowedTenantUsageScopeKinds: 37 + AllowedTenantUsageScopeKinds: 28 + AllowedTenantUsageScopeKinds: 17 + AllowedTenantUsageScopeKinds: 4 + AllowedTenantUsageScopeKinds: 45 + AllowedTenantUsageScopeKinds: 34 + AllowedTenantUsageScopeKinds: 30 + AllowedNodeTypeUsageScopeKinds: 2 + AllowedNodeTypeUsageScopeKinds: 45 + } + } + } +} + diff --git a/deploy/8_node_cluster/storage/cfg/CreateTenant-1.txt b/deploy/8_node_cluster/storage/cfg/CreateTenant-1.txt new file mode 100644 index 00000000000..6fe36ab24aa --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/CreateTenant-1.txt @@ -0,0 +1,27 @@ +CreateTenantRequest { + Request { + path: "/my_cluster/NBS" + resources { + storage_units { + unit_kind: "ssd" + count: 10 + } + storage_units { + unit_kind: "rot" + count: 5 + } + storage_units { + unit_kind: "rotmirror" + count: 5 + } + storage_units { + unit_kind: "ssdmirror" + count: 10 + } + } + options { + disable_tx_service: true + } + } +} + diff --git a/deploy/8_node_cluster/storage/cfg/CreateTenant-2.txt b/deploy/8_node_cluster/storage/cfg/CreateTenant-2.txt new file mode 100644 index 00000000000..d40293a946d --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/CreateTenant-2.txt @@ -0,0 +1,19 @@ +CreateTenantRequest { + Request { + path: "/my_cluster/NFS" + resources { + storage_units { + unit_kind: "ssd" + count: 20 + } + storage_units { + unit_kind: "rot" + count: 1 + } + } + options { + disable_tx_service: true + } + } +} + diff --git a/deploy/8_node_cluster/storage/cfg/CreateTenant-3.txt b/deploy/8_node_cluster/storage/cfg/CreateTenant-3.txt new file mode 100644 index 00000000000..48ae68a7a5b --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/CreateTenant-3.txt @@ -0,0 +1,12 @@ +CreateTenantRequest { + Request { + path: "/my_cluster/disk_manager" + resources { + storage_units { + unit_kind: "ssd" + count: 1 + } + } + } +} + diff --git a/deploy/8_node_cluster/storage/cfg/CreateTenant-4.txt b/deploy/8_node_cluster/storage/cfg/CreateTenant-4.txt new file mode 100644 index 00000000000..6bc06d91214 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/CreateTenant-4.txt @@ -0,0 +1,12 @@ +CreateTenantRequest { + Request { + path: "/my_cluster/snapshot" + resources { + storage_units { + unit_kind: "ssd" + count: 1 + } + } + } +} + diff --git a/deploy/8_node_cluster/storage/cfg/DefineBox.txt b/deploy/8_node_cluster/storage/cfg/DefineBox.txt new file mode 100644 index 00000000000..03795f5e409 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/DefineBox.txt @@ -0,0 +1,82 @@ +Command { + DefineHostConfig { + HostConfigId: 1 + Drive { + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + Type: SSD + } + Drive { + Path: "/dev/disk/by-partlabel/NVMEKIKIMR02" + Type: SSD + } + Drive { + Path: "/dev/disk/by-partlabel/NVMEKIKIMR03" + Type: SSD + } + Drive { + Path: "/dev/disk/by-partlabel/ROTKIKIMR01" + } + } +} +Command { + DefineBox { + BoxId: 1 + Host { + Key { + Fqdn: "my.host1" + IcPort: 19001 + } + HostConfigId: 1 + } + Host { + Key { + Fqdn: "my.host2" + IcPort: 19001 + } + HostConfigId: 1 + } + Host { + Key { + Fqdn: "my.host3" + IcPort: 19001 + } + HostConfigId: 1 + } + Host { + Key { + Fqdn: "my.host4" + IcPort: 19001 + } + HostConfigId: 1 + } + Host { + Key { + Fqdn: "my.host5" + IcPort: 19001 + } + HostConfigId: 1 + } + Host { + Key { + Fqdn: "my.host6" + IcPort: 19001 + } + HostConfigId: 1 + } + Host { + Key { + Fqdn: "my.host7" + IcPort: 19001 + } + HostConfigId: 1 + } + Host { + Key { + Fqdn: "my.host8" + IcPort: 19001 + } + HostConfigId: 1 + } + } +} + diff --git a/deploy/8_node_cluster/storage/cfg/DefineStoragePools.txt b/deploy/8_node_cluster/storage/cfg/DefineStoragePools.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/DefineStoragePools.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/storage/cfg/app_config.proto b/deploy/8_node_cluster/storage/cfg/app_config.proto new file mode 100644 index 00000000000..c599007554b --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/app_config.proto @@ -0,0 +1,1212 @@ +ActorSystemConfig { + Executor { + Type: BASIC + Threads: 8 + SpinThreshold: 10 + Name: "System" + } + Executor { + Type: BASIC + Threads: 8 + SpinThreshold: 10 + Name: "User" + } + Executor { + Type: BASIC + Threads: 8 + SpinThreshold: 1 + Name: "Batch" + } + Executor { + Type: IO + Threads: 1 + Name: "IO" + } + Executor { + Type: BASIC + Threads: 3 + SpinThreshold: 10 + Name: "IC" + TimePerMailboxMicroSecs: 100 + } + Scheduler { + Resolution: 256 + SpinThreshold: 0 + ProgressThreshold: 10000 + } + SysExecutor: 0 + UserExecutor: 1 + IoExecutor: 3 + BatchExecutor: 2 + ServiceExecutor { + ServiceName: "Interconnect" + ExecutorId: 4 + } +} +LogConfig { + DefaultLevel: 3 +} +NameserviceConfig { + Node { + NodeId: 1 + Port: 19001 + Host: "my.host1" + InterconnectHost: "my.host1" + Location { + DataCenter: "my_dc" + Rack: "lab-1:1" + Body: 1 + } + } + Node { + NodeId: 2 + Port: 19001 + Host: "my.host2" + InterconnectHost: "my.host2" + Location { + DataCenter: "my_dc" + Rack: "lab-2:1" + Body: 2 + } + } + Node { + NodeId: 3 + Port: 19001 + Host: "my.host3" + InterconnectHost: "my.host3" + Location { + DataCenter: "my_dc" + Rack: "lab-3:1" + Body: 3 + } + } + Node { + NodeId: 4 + Port: 19001 + Host: "my.host4" + InterconnectHost: "my.host4" + Location { + DataCenter: "my_dc" + Rack: "lab-4:1" + Body: 4 + } + } + Node { + NodeId: 5 + Port: 19001 + Host: "my.host5" + InterconnectHost: "my.host5" + Location { + DataCenter: "my_dc" + Rack: "lab-5:1" + Body: 5 + } + } + Node { + NodeId: 6 + Port: 19001 + Host: "my.host6" + InterconnectHost: "my.host6" + Location { + DataCenter: "my_dc" + Rack: "lab-6:1" + Body: 6 + } + } + Node { + NodeId: 7 + Port: 19001 + Host: "my.host7" + InterconnectHost: "my.host7" + Location { + DataCenter: "my_dc" + Rack: "lab-7:1" + Body: 7 + } + } + Node { + NodeId: 8 + Port: 19001 + Host: "my.host8" + InterconnectHost: "my.host8" + Location { + DataCenter: "my_dc" + Rack: "lab-8:1" + Body: 8 + } + } + ClusterUUID: "2452520" + AcceptUUID: "2452520" +} +InterconnectConfig { + StartTcp: true + MaxInflightAmountOfDataInKB: 10240 + HandshakeTimeoutDuration { + Seconds: 1 + } +} +DomainsConfig { + Domain { + DomainId: 1 + SchemeRoot: 72057594046678944 + SSId: 1 + HiveUid: 1 + PlanResolution: 10 + Name: "my_cluster" + StoragePoolTypes { + Kind: "ssd" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssd" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "ssdmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssdmirror" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "rot" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rot" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + StoragePoolTypes { + Kind: "rotmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rotmirror" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + ExplicitMediators: 72057594046382081 + ExplicitMediators: 72057594046382082 + ExplicitMediators: 72057594046382083 + ExplicitCoordinators: 72057594046316545 + ExplicitCoordinators: 72057594046316546 + ExplicitCoordinators: 72057594046316547 + ExplicitAllocators: 72057594046447617 + ExplicitAllocators: 72057594046447618 + ExplicitAllocators: 72057594046447619 + } + StateStorage { + SSId: 1 + Ring { + NToSelect: 5 + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + } + } + HiveConfig { + HiveUid: 1 + Hive: 72057594037968897 + } +} +BlobStorageConfig { + ServiceSet { + PDisks { + NodeID: 1 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 10738122723267084431 + PDiskCategory: 1 + } + PDisks { + NodeID: 2 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 16106689533420805170 + PDiskCategory: 1 + } + PDisks { + NodeID: 3 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 15479661826898200215 + PDiskCategory: 1 + } + PDisks { + NodeID: 4 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 5795881613215917266 + PDiskCategory: 1 + } + PDisks { + NodeID: 5 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 18089278276041197701 + PDiskCategory: 1 + } + PDisks { + NodeID: 6 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 7117908886488453734 + PDiskCategory: 1 + } + PDisks { + NodeID: 7 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 12400000310110999597 + PDiskCategory: 1 + } + PDisks { + NodeID: 8 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 16321399264440587887 + PDiskCategory: 1 + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 0 + VDisk: 0 + } + VDiskLocation { + NodeID: 1 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 10738122723267084431 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 1 + VDisk: 0 + } + VDiskLocation { + NodeID: 2 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16106689533420805170 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 2 + VDisk: 0 + } + VDiskLocation { + NodeID: 3 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 15479661826898200215 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 3 + VDisk: 0 + } + VDiskLocation { + NodeID: 4 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 5795881613215917266 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 4 + VDisk: 0 + } + VDiskLocation { + NodeID: 5 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 18089278276041197701 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 5 + VDisk: 0 + } + VDiskLocation { + NodeID: 6 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 7117908886488453734 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 6 + VDisk: 0 + } + VDiskLocation { + NodeID: 7 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 12400000310110999597 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 7 + VDisk: 0 + } + VDiskLocation { + NodeID: 8 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16321399264440587887 + } + VDiskKind: Default + } + Groups { + GroupID: 0 + GroupGeneration: 1 + ErasureSpecies: 4 + Rings { + FailDomains { + VDiskLocations { + NodeID: 1 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 10738122723267084431 + } + } + FailDomains { + VDiskLocations { + NodeID: 2 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16106689533420805170 + } + } + FailDomains { + VDiskLocations { + NodeID: 3 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 15479661826898200215 + } + } + FailDomains { + VDiskLocations { + NodeID: 4 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 5795881613215917266 + } + } + FailDomains { + VDiskLocations { + NodeID: 5 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 18089278276041197701 + } + } + FailDomains { + VDiskLocations { + NodeID: 6 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 7117908886488453734 + } + } + FailDomains { + VDiskLocations { + NodeID: 7 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 12400000310110999597 + } + } + FailDomains { + VDiskLocations { + NodeID: 8 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16321399264440587887 + } + } + } + } + AvailabilityDomains: 1 + } +} +ChannelProfileConfig { + Profile { + ProfileId: 0 + Channel { + ErasureSpecies: "block-4-2" + PDiskCategory: 1 + VDiskCategory: Default + } + Channel { + ErasureSpecies: "block-4-2" + PDiskCategory: 1 + VDiskCategory: Default + } + Channel { + ErasureSpecies: "block-4-2" + PDiskCategory: 1 + VDiskCategory: Default + } + } +} +BootstrapConfig { + Tablet { + Type: FLAT_HIVE + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037968897 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: FLAT_BS_CONTROLLER + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037932033 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: FLAT_SCHEMESHARD + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046678944 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: FLAT_TX_COORDINATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046316545 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: FLAT_TX_COORDINATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046316546 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: FLAT_TX_COORDINATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046316547 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_MEDIATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046382081 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_MEDIATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046382082 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_MEDIATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046382083 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_ALLOCATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046447617 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_ALLOCATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046447618 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TX_ALLOCATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046447619 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: CMS + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936128 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: NODE_BROKER + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936129 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: TENANT_SLOT_BROKER + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936130 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } + Tablet { + Type: CONSOLE + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936131 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } + } +} +VDiskConfig { + VDiskKinds { + Kind: Log + BaseKind: Default + Config { + FreshUseDreg: true + AllowKeepFlags: false + } + } + VDiskKinds { + Kind: LocalMode + BaseKind: Default + Config { + HullCompLevel0MaxSstsAtOnce: 2 + HullCompSortedPartsNum: 1 + } + } +} +KQPConfig { + Enable: true + Settings { + Name: "_ResultRowsLimit" + Value: "10000" + } +} +GRpcConfig { + StartGRpcProxy: true + Host: "[::]" + Port: 2135 + GRpcMemoryQuotaBytes: 1073741824 + StreamingConfig { + EnableOutputStreams: true + } + KeepAliveEnable: true + KeepAliveIdleTimeoutTriggerSec: 90 + KeepAliveMaxProbeCount: 3 + KeepAliveProbeIntervalSec: 10 +} +FeatureFlags { + EnableSeparateSolomonShardForPDisk: true + EnableVPatch: true +} +PQConfig { + Enabled: false +} + diff --git a/deploy/8_node_cluster/storage/cfg/auth.txt b/deploy/8_node_cluster/storage/cfg/auth.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/auth.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/storage/cfg/boot.txt b/deploy/8_node_cluster/storage/cfg/boot.txt new file mode 100644 index 00000000000..da64c0a4e6d --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/boot.txt @@ -0,0 +1,609 @@ +Tablet { + Type: FLAT_HIVE + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037968897 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: FLAT_BS_CONTROLLER + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037932033 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: FLAT_SCHEMESHARD + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046678944 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: FLAT_TX_COORDINATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046316545 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: FLAT_TX_COORDINATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046316546 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: FLAT_TX_COORDINATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046316547 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: TX_MEDIATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046382081 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: TX_MEDIATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046382082 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: TX_MEDIATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046382083 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: TX_ALLOCATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046447617 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: TX_ALLOCATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046447618 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: TX_ALLOCATOR + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594046447619 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: CMS + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936128 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: NODE_BROKER + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936129 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: TENANT_SLOT_BROKER + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936130 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} +Tablet { + Type: CONSOLE + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + Info { + TabletID: 72057594037936131 + Channels { + Channel: 0 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 1 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + Channels { + Channel: 2 + History { + FromGeneration: 0 + GroupID: 0 + } + ChannelErasureName: "block-4-2" + } + } +} + diff --git a/deploy/8_node_cluster/storage/cfg/bs.txt b/deploy/8_node_cluster/storage/cfg/bs.txt new file mode 100644 index 00000000000..1348744c780 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/bs.txt @@ -0,0 +1,259 @@ +ServiceSet { + PDisks { + NodeID: 1 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 10738122723267084431 + PDiskCategory: 1 + } + PDisks { + NodeID: 2 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 16106689533420805170 + PDiskCategory: 1 + } + PDisks { + NodeID: 3 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 15479661826898200215 + PDiskCategory: 1 + } + PDisks { + NodeID: 4 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 5795881613215917266 + PDiskCategory: 1 + } + PDisks { + NodeID: 5 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 18089278276041197701 + PDiskCategory: 1 + } + PDisks { + NodeID: 6 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 7117908886488453734 + PDiskCategory: 1 + } + PDisks { + NodeID: 7 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 12400000310110999597 + PDiskCategory: 1 + } + PDisks { + NodeID: 8 + PDiskID: 1 + Path: "/dev/disk/by-partlabel/NVMEKIKIMR01" + PDiskGuid: 16321399264440587887 + PDiskCategory: 1 + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 0 + VDisk: 0 + } + VDiskLocation { + NodeID: 1 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 10738122723267084431 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 1 + VDisk: 0 + } + VDiskLocation { + NodeID: 2 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16106689533420805170 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 2 + VDisk: 0 + } + VDiskLocation { + NodeID: 3 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 15479661826898200215 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 3 + VDisk: 0 + } + VDiskLocation { + NodeID: 4 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 5795881613215917266 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 4 + VDisk: 0 + } + VDiskLocation { + NodeID: 5 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 18089278276041197701 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 5 + VDisk: 0 + } + VDiskLocation { + NodeID: 6 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 7117908886488453734 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 6 + VDisk: 0 + } + VDiskLocation { + NodeID: 7 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 12400000310110999597 + } + VDiskKind: Default + } + VDisks { + VDiskID { + GroupID: 0 + GroupGeneration: 1 + Ring: 0 + Domain: 7 + VDisk: 0 + } + VDiskLocation { + NodeID: 8 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16321399264440587887 + } + VDiskKind: Default + } + Groups { + GroupID: 0 + GroupGeneration: 1 + ErasureSpecies: 4 + Rings { + FailDomains { + VDiskLocations { + NodeID: 1 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 10738122723267084431 + } + } + FailDomains { + VDiskLocations { + NodeID: 2 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16106689533420805170 + } + } + FailDomains { + VDiskLocations { + NodeID: 3 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 15479661826898200215 + } + } + FailDomains { + VDiskLocations { + NodeID: 4 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 5795881613215917266 + } + } + FailDomains { + VDiskLocations { + NodeID: 5 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 18089278276041197701 + } + } + FailDomains { + VDiskLocations { + NodeID: 6 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 7117908886488453734 + } + } + FailDomains { + VDiskLocations { + NodeID: 7 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 12400000310110999597 + } + } + FailDomains { + VDiskLocations { + NodeID: 8 + PDiskID: 1 + VDiskSlotID: 0 + PDiskGuid: 16321399264440587887 + } + } + } + } + AvailabilityDomains: 1 +} + diff --git a/deploy/8_node_cluster/storage/cfg/channels.txt b/deploy/8_node_cluster/storage/cfg/channels.txt new file mode 100644 index 00000000000..73751dc5fbf --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/channels.txt @@ -0,0 +1,19 @@ +Profile { + ProfileId: 0 + Channel { + ErasureSpecies: "block-4-2" + PDiskCategory: 1 + VDiskCategory: Default + } + Channel { + ErasureSpecies: "block-4-2" + PDiskCategory: 1 + VDiskCategory: Default + } + Channel { + ErasureSpecies: "block-4-2" + PDiskCategory: 1 + VDiskCategory: Default + } +} + diff --git a/deploy/8_node_cluster/storage/cfg/cms.txt b/deploy/8_node_cluster/storage/cfg/cms.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/cms.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/storage/cfg/config.yaml b/deploy/8_node_cluster/storage/cfg/config.yaml new file mode 100644 index 00000000000..c5df10f9d82 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/config.yaml @@ -0,0 +1,221 @@ +actor_system_config: + batch_executor: 2 + executor: + - {name: System, spin_threshold: '10', threads: 8, type: BASIC} + - {name: User, spin_threshold: '10', threads: 8, type: BASIC} + - {name: Batch, spin_threshold: '1', threads: 8, type: BASIC} + - {name: IO, threads: 1, type: IO} + - {name: IC, spin_threshold: '10', threads: 3, time_per_mailbox_micro_secs: 100, + type: BASIC} + io_executor: 3 + scheduler: {progress_threshold: '10000', resolution: '256', spin_threshold: '0'} + service_executor: + - {executor_id: 4, service_name: Interconnect} + sys_executor: 0 + user_executor: 1 +blob_storage_config: + service_set: + availability_domains: ['1'] + groups: + - erasure_species: 4 + group_generation: 1 + group_id: 0 + rings: + - fail_domains: + - vdisk_locations: + - {node_id: 1, path: /dev/disk/by-partlabel/NVMEKIKIMR01, pdisk_category: '1', + pdisk_guid: '10738122723267084431', pdisk_id: 1} + - vdisk_locations: + - {node_id: 2, path: /dev/disk/by-partlabel/NVMEKIKIMR01, pdisk_category: '1', + pdisk_guid: '16106689533420805170', pdisk_id: 1} + - vdisk_locations: + - {node_id: 3, path: /dev/disk/by-partlabel/NVMEKIKIMR01, pdisk_category: '1', + pdisk_guid: '15479661826898200215', pdisk_id: 1} + - vdisk_locations: + - {node_id: 4, path: /dev/disk/by-partlabel/NVMEKIKIMR01, pdisk_category: '1', + pdisk_guid: '5795881613215917266', pdisk_id: 1} + - vdisk_locations: + - {node_id: 5, path: /dev/disk/by-partlabel/NVMEKIKIMR01, pdisk_category: '1', + pdisk_guid: '18089278276041197701', pdisk_id: 1} + - vdisk_locations: + - {node_id: 6, path: /dev/disk/by-partlabel/NVMEKIKIMR01, pdisk_category: '1', + pdisk_guid: '7117908886488453734', pdisk_id: 1} + - vdisk_locations: + - {node_id: 7, path: /dev/disk/by-partlabel/NVMEKIKIMR01, pdisk_category: '1', + pdisk_guid: '12400000310110999597', pdisk_id: 1} + - vdisk_locations: + - {node_id: 8, path: /dev/disk/by-partlabel/NVMEKIKIMR01, pdisk_category: '1', + pdisk_guid: '16321399264440587887', pdisk_id: 1} +bootstrap_config: {} +channel_profile_config: + profile: + - channel: + - {erasure_species: block-4-2, pdisk_category: '1', vdisk_category: Default} + - {erasure_species: block-4-2, pdisk_category: '1', vdisk_category: Default} + - {erasure_species: block-4-2, pdisk_category: '1', vdisk_category: Default} + profile_id: 0 +domains_config: + disable_builtin_security: true + domain: + - domain_id: 1 + name: my_cluster + plan_resolution: '10' + scheme_root: '72057594046678944' + ssid: [1] + storage_pool_types: + - kind: ssd + pool_config: + box_id: '1' + erasure_species: block-4-2 + geometry: {domain_level_begin: 10, domain_level_end: 50, num_fail_domains_per_fail_realm: 8, + num_fail_realms: 1, num_vdisks_per_fail_domain: 1, realm_level_begin: 10, + realm_level_end: 20} + kind: ssd + pdisk_filter: + - property: + - {type: SSD} + vdisk_kind: Default + - kind: ssdmirror + pool_config: + box_id: '1' + erasure_species: mirror-3of4 + geometry: {domain_level_begin: 10, domain_level_end: 50, num_fail_domains_per_fail_realm: 8, + num_fail_realms: 1, num_vdisks_per_fail_domain: 1, realm_level_begin: 10, + realm_level_end: 20} + kind: ssdmirror + pdisk_filter: + - property: + - {type: SSD} + vdisk_kind: Default + - kind: rot + pool_config: + box_id: '1' + erasure_species: block-4-2 + geometry: {domain_level_begin: 10, domain_level_end: 50, num_fail_domains_per_fail_realm: 8, + num_fail_realms: 1, num_vdisks_per_fail_domain: 1, realm_level_begin: 10, + realm_level_end: 20} + kind: rot + pdisk_filter: + - property: + - {type: ROT} + vdisk_kind: Default + - kind: rotmirror + pool_config: + box_id: '1' + erasure_species: mirror-3of4 + geometry: {domain_level_begin: 10, domain_level_end: 50, num_fail_domains_per_fail_realm: 8, + num_fail_realms: 1, num_vdisks_per_fail_domain: 1, realm_level_begin: 10, + realm_level_end: 20} + kind: rotmirror + pdisk_filter: + - property: + - {type: ROT} + vdisk_kind: Default + state_storage: + - ring: + node: [1, 2, 3, 4, 5, 6, 7, 8] + nto_select: 5 + ssid: 1 +dynamic_nameservice_config: {max_static_node_id: 50000} +feature_flags: {enable_separate_solomon_shard_for_pdisk: true, enable_vpatch: true} +grpc_config: + grpc_memory_quota_bytes: '1073741824' + host: '[::]' + keep_alive_enable: true + keep_alive_idle_timeout_trigger_sec: 90 + keep_alive_max_probe_count: 3 + keep_alive_probe_interval_sec: 10 + port: 2135 + start_grpc_proxy: true + streaming_config: {enable_output_streams: true} +hosts: +- host: my.host1 + location: {body: 1, data_center: my_dc, rack: 'lab-1:1'} + node_id: 1 +- host: my.host2 + location: {body: 2, data_center: my_dc, rack: 'lab-2:1'} + node_id: 2 +- host: my.host3 + location: {body: 3, data_center: my_dc, rack: 'lab-3:1'} + node_id: 3 +- host: my.host4 + location: {body: 4, data_center: my_dc, rack: 'lab-4:1'} + node_id: 4 +- host: my.host5 + location: {body: 5, data_center: my_dc, rack: 'lab-5:1'} + node_id: 5 +- host: my.host6 + location: {body: 6, data_center: my_dc, rack: 'lab-6:1'} + node_id: 6 +- host: my.host7 + location: {body: 7, data_center: my_dc, rack: 'lab-7:1'} + node_id: 7 +- host: my.host8 + location: {body: 8, data_center: my_dc, rack: 'lab-8:1'} + node_id: 8 +interconnect_config: + handshake_timeout_duration: {seconds: '1'} + max_inflight_amount_of_data_in_kb: 10240 + start_tcp: true +kqpconfig: + enable: true + settings: + - {name: _ResultRowsLimit, value: '10000'} +log_config: {default_level: 3} +nameservice_config: + accept_uuid: ['2452520'] + cluster_uuid: '2452520' +pqconfig: {enabled: false} +static_erasure: block-4-2 +system_tablets: + cms: + - info: {tablet_id: '72057594037936128'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + console: + - info: {tablet_id: '72057594037936131'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + flat_bs_controller: + - info: {tablet_id: '72057594037932033'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + flat_hive: + - info: {tablet_id: '72057594037968897'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + flat_schemeshard: + - info: {tablet_id: '72057594046678944'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + flat_tx_coordinator: + - info: {tablet_id: '72057594046316545'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + - info: {tablet_id: '72057594046316546'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + - info: {tablet_id: '72057594046316547'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + node_broker: + - info: {tablet_id: '72057594037936129'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + tenant_slot_broker: + - info: {tablet_id: '72057594037936130'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + tx_allocator: + - info: {tablet_id: '72057594046447617'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + - info: {tablet_id: '72057594046447618'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + - info: {tablet_id: '72057594046447619'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + tx_mediator: + - info: {tablet_id: '72057594046382081'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + - info: {tablet_id: '72057594046382082'} + node: [1, 2, 3, 4, 5, 6, 7, 8] + - info: {tablet_id: '72057594046382083'} + node: [1, 2, 3, 4, 5, 6, 7, 8] +vdisk_config: + vdisk_kinds: + - base_kind: Default + config: {allow_keep_flags: false, fresh_use_dreg: true} + kind: Log + - base_kind: Default + config: {hull_comp_level0_max_ssts_at_once: 2, hull_comp_sorted_parts_num: 1} + kind: LocalMode + diff --git a/deploy/8_node_cluster/storage/cfg/domains.txt b/deploy/8_node_cluster/storage/cfg/domains.txt new file mode 100644 index 00000000000..199358ecec0 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/domains.txt @@ -0,0 +1,128 @@ +Domain { + DomainId: 1 + SchemeRoot: 72057594046678944 + SSId: 1 + HiveUid: 1 + PlanResolution: 10 + Name: "my_cluster" + StoragePoolTypes { + Kind: "ssd" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssd" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "ssdmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "ssdmirror" + PDiskFilter { + Property { + Type: SSD + } + } + } + } + StoragePoolTypes { + Kind: "rot" + PoolConfig { + BoxId: 1 + ErasureSpecies: "block-4-2" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rot" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + StoragePoolTypes { + Kind: "rotmirror" + PoolConfig { + BoxId: 1 + ErasureSpecies: "mirror-3of4" + Geometry { + RealmLevelBegin: 10 + RealmLevelEnd: 20 + DomainLevelBegin: 10 + DomainLevelEnd: 50 + NumFailRealms: 1 + NumFailDomainsPerFailRealm: 8 + NumVDisksPerFailDomain: 1 + } + VDiskKind: "Default" + Kind: "rotmirror" + PDiskFilter { + Property { + Type: ROT + } + } + } + } + ExplicitMediators: 72057594046382081 + ExplicitMediators: 72057594046382082 + ExplicitMediators: 72057594046382083 + ExplicitCoordinators: 72057594046316545 + ExplicitCoordinators: 72057594046316546 + ExplicitCoordinators: 72057594046316547 + ExplicitAllocators: 72057594046447617 + ExplicitAllocators: 72057594046447618 + ExplicitAllocators: 72057594046447619 +} +StateStorage { + SSId: 1 + Ring { + NToSelect: 5 + Node: 1 + Node: 2 + Node: 3 + Node: 4 + Node: 5 + Node: 6 + Node: 7 + Node: 8 + } +} +HiveConfig { + HiveUid: 1 + Hive: 72057594037968897 +} + diff --git a/deploy/8_node_cluster/storage/cfg/dyn_ns.txt b/deploy/8_node_cluster/storage/cfg/dyn_ns.txt new file mode 100644 index 00000000000..97d533220ad --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/dyn_ns.txt @@ -0,0 +1,2 @@ +MaxStaticNodeId: 50000 + diff --git a/deploy/8_node_cluster/storage/cfg/dynamic_server.cfg b/deploy/8_node_cluster/storage/cfg/dynamic_server.cfg new file mode 100644 index 00000000000..63d4876fa28 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/dynamic_server.cfg @@ -0,0 +1,100 @@ +kikimr_grpc_port="${kikimr_grpc_port:? expected not empty var}" +kikimr_home="${kikimr_home:? expected not empty var}" +kikimr_ic_port="${kikimr_ic_port:? expected not empty var}" +kikimr_binaries_base_path="/usr/bin/kikimr" +kikimr_mbus_port="${kikimr_mbus_port:? expected not empty var}" +kikimr_mon_address="" +kikimr_mon_port="${kikimr_mon_port:? expected not empty var}" +kikimr_node_broker_port="2135" +kikimr_syslog_service_tag="${kikimr_syslog_service_tag:? expected not empty var}" +kikimr_tenant="${kikimr_tenant:? expected not empty var}" +kikimr_config="${kikimr_home}/cfg" +kikimr_domain_file="${kikimr_config}/domains.txt" +kikimr_naming_file="${kikimr_config}/names.txt" +kikimr_blobstorage_file="${kikimr_config}/bs.txt" +kikimr_cms_file="${kikimr_config}/cms.txt" +kikimr_ic_file="${kikimr_config}/ic.txt" +kikimr_grpc_file="${kikimr_config}/grpc.txt" +kikimr_bootstrap_file="${kikimr_config}/boot.txt" +kikimr_channels_file="${kikimr_config}/channels.txt" +kikimr_logfile="${kikimr_config}/log.txt" +kikimr_vdisksfile="${kikimr_config}/vdisks.txt" +kikimr_kqp_file="${kikimr_config}/kqp.txt" +kikimr_feature_flags_file="${kikimr_config}/feature_flags.txt" +kikimr_netmode="--tcp" +kikimr_mon_threads="10" +kikimr_udfs_dir="${kikimr_binaries_base_path}/libs" +kikimr_key_file="${kikimr_config}/key.txt" +kikimr_auth_file="${kikimr_config}/auth.txt" +kikimr_auth_token_file="${kikimr_home}/token/kikimr.token" +kikimr_dyn_ns_file="${kikimr_config}/dyn_ns.txt" +kikimr_tracing_file="${kikimr_config}/tracing.txt" +kikimr_pdisk_key_file="${kikimr_config}/pdisk_key.txt" + +#Custom config +[ -s /etc/default/kikimr.custom ] && . /etc/default/kikimr.custom + + +if [ -z "$kikimr_system_file" ]; then + kikimr_system_file="${kikimr_config}/sys.txt" +fi + + +kikimr_arg="${kikimr_arg} server" +kikimr_arg="${kikimr_arg}${kikimr_loglevel:+ --log-level ${kikimr_loglevel}} --syslog" +kikimr_arg="${kikimr_arg}${kikimr_netmode:+ ${kikimr_netmode}}" +kikimr_arg="${kikimr_arg}${kikimr_udfs_dir:+ --udfs-dir ${kikimr_udfs_dir}}" +kikimr_arg="${kikimr_arg}${kikimr_logfile:+ --log-file ${kikimr_logfile}}" +kikimr_arg="${kikimr_arg}${kikimr_vdisksfile:+ --vdisk-file ${kikimr_vdisksfile}}" +kikimr_arg="${kikimr_arg}${kikimr_system_file:+ --sys-file ${kikimr_system_file}}" +kikimr_arg="${kikimr_arg}${kikimr_domain_file:+ --domains-file ${kikimr_domain_file}}" +kikimr_arg="${kikimr_arg}${kikimr_naming_file:+ --naming-file ${kikimr_naming_file}}" +kikimr_arg="${kikimr_arg}${kikimr_blobstorage_file:+ --bs-file ${kikimr_blobstorage_file}}" +kikimr_arg="${kikimr_arg}${kikimr_cms_file:+ --cms-file ${kikimr_cms_file}}" +kikimr_arg="${kikimr_arg}${kikimr_ic_file:+ --ic-file ${kikimr_ic_file}}" +kikimr_arg="${kikimr_arg}${kikimr_grpc_file:+ --grpc-file ${kikimr_grpc_file}}" +kikimr_arg="${kikimr_arg}${kikimr_channels_file:+ --channels-file ${kikimr_channels_file}}" +kikimr_arg="${kikimr_arg}${kikimr_mon_port:+ --mon-port ${kikimr_mon_port}}" +kikimr_arg="${kikimr_arg}${kikimr_mon_threads:+ --mon-threads ${kikimr_mon_threads}}" +kikimr_arg="${kikimr_arg}${kikimr_bootstrap_file:+ --bootstrap-file ${kikimr_bootstrap_file}}" +kikimr_arg="${kikimr_arg}${kikimr_kqp_file:+ --kqp-file ${kikimr_kqp_file}}" +kikimr_arg="${kikimr_arg}${kikimr_feature_flags_file:+ --feature-flags-file ${kikimr_feature_flags_file}}" +kikimr_arg="${kikimr_arg}${kikimr_grpc_port:+ --grpc-port ${kikimr_grpc_port}}" +kikimr_arg="${kikimr_arg}${kikimr_ic_port:+ --ic-port ${kikimr_ic_port}}" +if [ ! -z "${kikimr_mon_address}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_mon_address:+ --mon-address ${kikimr_mon_address}}" +else + echo "Monitoring address is not defined." +fi + +if [ -f "${kikimr_key_file}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_key_file:+ --key-file ${kikimr_key_file}}" +else + echo "Key file not found!" +fi + +if [ -f "${kikimr_auth_file}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_auth_file:+ --auth-file ${kikimr_auth_file}}" +fi +if [ -f "${kikimr_auth_token_file}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_auth_token_file:+ --auth-token-file ${kikimr_auth_token_file}}" +fi + +if [ -f "${kikimr_dyn_ns_file}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_dyn_ns_file:+ --dyn-nodes-file ${kikimr_dyn_ns_file}}" +fi + +if [ -f "${kikimr_pdisk_key_file}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_pdisk_key_file:+ --pdisk-key-file ${kikimr_pdisk_key_file}}" +else + echo "PDisk Key file not found!" +fi + +kikimr_arg="${kikimr_arg}${kikimr_ca:+ --ca=${kikimr_ca}}${kikimr_cert:+ --cert=${kikimr_cert}}${kikimr_key:+ --key=${kikimr_key}}" +kikimr_arg="${kikimr_arg} ${kikimr_tracing_file}" + +kikimr_arg="${kikimr_arg}${kikimr_node_broker_port:+ --node-broker-port ${kikimr_node_broker_port}}" + +kikimr_arg="${kikimr_arg}${kikimr_syslog_service_tag:+ --syslog-service-tag ${kikimr_syslog_service_tag}}" + +kikimr_arg="${kikimr_arg}${kikimr_tenant:+ --tenant ${kikimr_tenant}}" diff --git a/deploy/8_node_cluster/storage/cfg/feature_flags.txt b/deploy/8_node_cluster/storage/cfg/feature_flags.txt new file mode 100644 index 00000000000..ec43bbc7d7b --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/feature_flags.txt @@ -0,0 +1,3 @@ +EnableSeparateSolomonShardForPDisk: true +EnableVPatch: true + diff --git a/deploy/8_node_cluster/storage/cfg/grpc.txt b/deploy/8_node_cluster/storage/cfg/grpc.txt new file mode 100644 index 00000000000..122e120bfff --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/grpc.txt @@ -0,0 +1,12 @@ +StartGRpcProxy: true +Host: "[::]" +Port: 2135 +GRpcMemoryQuotaBytes: 1073741824 +StreamingConfig { + EnableOutputStreams: true +} +KeepAliveEnable: true +KeepAliveIdleTimeoutTriggerSec: 90 +KeepAliveMaxProbeCount: 3 +KeepAliveProbeIntervalSec: 10 + diff --git a/deploy/8_node_cluster/storage/cfg/ic.txt b/deploy/8_node_cluster/storage/cfg/ic.txt new file mode 100644 index 00000000000..f3b5859bab1 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/ic.txt @@ -0,0 +1,6 @@ +StartTcp: true +MaxInflightAmountOfDataInKB: 10240 +HandshakeTimeoutDuration { + Seconds: 1 +} + diff --git a/deploy/8_node_cluster/storage/cfg/init_cms.bash b/deploy/8_node_cluster/storage/cfg/init_cms.bash new file mode 100644 index 00000000000..cf3b9295be1 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/init_cms.bash @@ -0,0 +1,2 @@ +set -eu +./kikimr -s grpc://my.host1:2135 admin console execute --domain=my_cluster --retry=10 cfg/Configure-my_cluster.txt diff --git a/deploy/8_node_cluster/storage/cfg/init_compute.bash b/deploy/8_node_cluster/storage/cfg/init_compute.bash new file mode 100644 index 00000000000..41c682c7cbc --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/init_compute.bash @@ -0,0 +1 @@ +set -eu diff --git a/deploy/8_node_cluster/storage/cfg/init_databases.bash b/deploy/8_node_cluster/storage/cfg/init_databases.bash new file mode 100644 index 00000000000..b8d02dd8877 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/init_databases.bash @@ -0,0 +1,3 @@ +./kikimr -s grpc://my.host1:2135 admin console execute --domain=my_cluster --retry=10 cfg/CreateTenant-1.txt +./kikimr -s grpc://my.host1:2135 admin console execute --domain=my_cluster --retry=10 cfg/CreateTenant-2.txt +exit 0 diff --git a/deploy/8_node_cluster/storage/cfg/init_root_storage.bash b/deploy/8_node_cluster/storage/cfg/init_root_storage.bash new file mode 100644 index 00000000000..41c682c7cbc --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/init_root_storage.bash @@ -0,0 +1 @@ +set -eu diff --git a/deploy/8_node_cluster/storage/cfg/init_storage.bash b/deploy/8_node_cluster/storage/cfg/init_storage.bash new file mode 100644 index 00000000000..4ca577a2bf0 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/init_storage.bash @@ -0,0 +1,3 @@ +set -eu +./kikimr -s grpc://my.host1:2135 admin bs config invoke --proto-file cfg/DefineBox.txt +./kikimr -s grpc://my.host1:2135 admin bs config invoke --proto-file cfg/DefineStoragePools.txt diff --git a/deploy/8_node_cluster/storage/cfg/kikimr.cfg b/deploy/8_node_cluster/storage/cfg/kikimr.cfg new file mode 100644 index 00000000000..e0419254113 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/kikimr.cfg @@ -0,0 +1,94 @@ +kikimr_mon_address="::" +kikimr_loglevel="3" +kikimr_ic_port="19001" +kikimr_mon_port="8765" +kikimr_home="/etc/kikimr" +kikimr_binaries_base_path="/usr/bin" +kikimr_config="${kikimr_home}/cfg" +kikimr_system_file="${kikimr_config}/sys.txt" +kikimr_domain_file="${kikimr_config}/domains.txt" +kikimr_naming_file="${kikimr_config}/names.txt" +kikimr_blobstorage_file="${kikimr_config}/bs.txt" +kikimr_cms_file="${kikimr_config}/cms.txt" +kikimr_ic_file="${kikimr_config}/ic.txt" +kikimr_grpc_file="${kikimr_config}/grpc.txt" +kikimr_bootstrap_file="${kikimr_config}/boot.txt" +kikimr_channels_file="${kikimr_config}/channels.txt" +kikimr_logfile="${kikimr_config}/log.txt" +kikimr_vdisksfile="${kikimr_config}/vdisks.txt" +kikimr_kqp_file="${kikimr_config}/kqp.txt" +kikimr_feature_flags_file="${kikimr_config}/feature_flags.txt" +kikimr_netmode="--tcp" +kikimr_mon_threads="10" +kikimr_udfs_dir="${kikimr_binaries_base_path}/libs" +kikimr_key_file="${kikimr_config}/key.txt" +kikimr_auth_file="${kikimr_config}/auth.txt" +kikimr_auth_token_file="${kikimr_home}/token/kikimr.token" +kikimr_dyn_ns_file="${kikimr_config}/dyn_ns.txt" +kikimr_tracing_file="${kikimr_config}/tracing.txt" +kikimr_pdisk_key_file="${kikimr_config}/pdisk_key.txt" + +kikimr_node_id="static" + + +#Custom config +[ -s /etc/default/kikimr.custom ] && . /etc/default/kikimr.custom + + +kikimr_arg="${kikimr_arg} server" +kikimr_arg="${kikimr_arg}${kikimr_loglevel:+ --log-level ${kikimr_loglevel}} --syslog" +kikimr_arg="${kikimr_arg}${kikimr_netmode:+ ${kikimr_netmode}}" +kikimr_arg="${kikimr_arg}${kikimr_udfs_dir:+ --udfs-dir ${kikimr_udfs_dir}}" +kikimr_arg="${kikimr_arg}${kikimr_logfile:+ --log-file ${kikimr_logfile}}" +kikimr_arg="${kikimr_arg}${kikimr_vdisksfile:+ --vdisk-file ${kikimr_vdisksfile}}" +kikimr_arg="${kikimr_arg}${kikimr_system_file:+ --sys-file ${kikimr_system_file}}" +kikimr_arg="${kikimr_arg}${kikimr_domain_file:+ --domains-file ${kikimr_domain_file}}" +kikimr_arg="${kikimr_arg}${kikimr_naming_file:+ --naming-file ${kikimr_naming_file}}" +kikimr_arg="${kikimr_arg}${kikimr_blobstorage_file:+ --bs-file ${kikimr_blobstorage_file}}" +kikimr_arg="${kikimr_arg}${kikimr_cms_file:+ --cms-file ${kikimr_cms_file}}" +kikimr_arg="${kikimr_arg}${kikimr_ic_file:+ --ic-file ${kikimr_ic_file}}" +kikimr_arg="${kikimr_arg}${kikimr_grpc_file:+ --grpc-file ${kikimr_grpc_file}}" +kikimr_arg="${kikimr_arg}${kikimr_channels_file:+ --channels-file ${kikimr_channels_file}}" +kikimr_arg="${kikimr_arg}${kikimr_mon_port:+ --mon-port ${kikimr_mon_port}}" +kikimr_arg="${kikimr_arg}${kikimr_mon_threads:+ --mon-threads ${kikimr_mon_threads}}" +kikimr_arg="${kikimr_arg}${kikimr_bootstrap_file:+ --bootstrap-file ${kikimr_bootstrap_file}}" +kikimr_arg="${kikimr_arg}${kikimr_kqp_file:+ --kqp-file ${kikimr_kqp_file}}" +kikimr_arg="${kikimr_arg}${kikimr_feature_flags_file:+ --feature-flags-file ${kikimr_feature_flags_file}}" +kikimr_arg="${kikimr_arg}${kikimr_grpc_port:+ --grpc-port ${kikimr_grpc_port}}" +kikimr_arg="${kikimr_arg}${kikimr_ic_port:+ --ic-port ${kikimr_ic_port}}" +if [ ! -z "${kikimr_mon_address}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_mon_address:+ --mon-address ${kikimr_mon_address}}" +else + echo "Monitoring address is not defined." +fi + +if [ -f "${kikimr_key_file}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_key_file:+ --key-file ${kikimr_key_file}}" +else + echo "Key file not found!" +fi + +if [ -f "${kikimr_auth_file}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_auth_file:+ --auth-file ${kikimr_auth_file}}" +fi +if [ -f "${kikimr_auth_token_file}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_auth_token_file:+ --auth-token-file ${kikimr_auth_token_file}}" +fi + +if [ -f "${kikimr_dyn_ns_file}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_dyn_ns_file:+ --dyn-nodes-file ${kikimr_dyn_ns_file}}" +fi + +if [ -f "${kikimr_pdisk_key_file}" ]; then + kikimr_arg="${kikimr_arg}${kikimr_pdisk_key_file:+ --pdisk-key-file ${kikimr_pdisk_key_file}}" +else + echo "PDisk Key file not found!" +fi + +kikimr_arg="${kikimr_arg}${kikimr_ca:+ --ca=${kikimr_ca}}${kikimr_cert:+ --cert=${kikimr_cert}}${kikimr_key:+ --key=${kikimr_key}}" +kikimr_arg="${kikimr_arg} ${kikimr_tracing_file}" + +kikimr_arg="${kikimr_arg}${kikimr_node_id:+ --node ${kikimr_node_id}}" + + + diff --git a/deploy/8_node_cluster/storage/cfg/kqp.txt b/deploy/8_node_cluster/storage/cfg/kqp.txt new file mode 100644 index 00000000000..a59b2e424c2 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/kqp.txt @@ -0,0 +1,6 @@ +Enable: true +Settings { + Name: "_ResultRowsLimit" + Value: "10000" +} + diff --git a/deploy/8_node_cluster/storage/cfg/log.txt b/deploy/8_node_cluster/storage/cfg/log.txt new file mode 100644 index 00000000000..3630274c5fd --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/log.txt @@ -0,0 +1,2 @@ +DefaultLevel: 3 + diff --git a/deploy/8_node_cluster/storage/cfg/names.txt b/deploy/8_node_cluster/storage/cfg/names.txt new file mode 100644 index 00000000000..b745ef84503 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/names.txt @@ -0,0 +1,91 @@ +Node { + NodeId: 1 + Port: 19001 + Host: "my.host1" + InterconnectHost: "my.host1" + Location { + DataCenter: "my_dc" + Rack: "lab-1:1" + Body: 1 + } +} +Node { + NodeId: 2 + Port: 19001 + Host: "my.host2" + InterconnectHost: "my.host2" + Location { + DataCenter: "my_dc" + Rack: "lab-2:1" + Body: 2 + } +} +Node { + NodeId: 3 + Port: 19001 + Host: "my.host3" + InterconnectHost: "my.host3" + Location { + DataCenter: "my_dc" + Rack: "lab-3:1" + Body: 3 + } +} +Node { + NodeId: 4 + Port: 19001 + Host: "my.host4" + InterconnectHost: "my.host4" + Location { + DataCenter: "my_dc" + Rack: "lab-4:1" + Body: 4 + } +} +Node { + NodeId: 5 + Port: 19001 + Host: "my.host5" + InterconnectHost: "my.host5" + Location { + DataCenter: "my_dc" + Rack: "lab-5:1" + Body: 5 + } +} +Node { + NodeId: 6 + Port: 19001 + Host: "my.host6" + InterconnectHost: "my.host6" + Location { + DataCenter: "my_dc" + Rack: "lab-6:1" + Body: 6 + } +} +Node { + NodeId: 7 + Port: 19001 + Host: "my.host7" + InterconnectHost: "my.host7" + Location { + DataCenter: "my_dc" + Rack: "lab-7:1" + Body: 7 + } +} +Node { + NodeId: 8 + Port: 19001 + Host: "my.host8" + InterconnectHost: "my.host8" + Location { + DataCenter: "my_dc" + Rack: "lab-8:1" + Body: 8 + } +} +ClusterUUID: "2452520" +AcceptUUID: "2452520" + diff --git a/deploy/8_node_cluster/storage/cfg/netclassifier.txt b/deploy/8_node_cluster/storage/cfg/netclassifier.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/netclassifier.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/storage/cfg/pq.txt b/deploy/8_node_cluster/storage/cfg/pq.txt new file mode 100644 index 00000000000..5af886dc334 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/pq.txt @@ -0,0 +1,2 @@ +Enabled: false + diff --git a/deploy/8_node_cluster/storage/cfg/pqcd.txt b/deploy/8_node_cluster/storage/cfg/pqcd.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/pqcd.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/storage/cfg/sqs.txt b/deploy/8_node_cluster/storage/cfg/sqs.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/sqs.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/storage/cfg/sys.txt b/deploy/8_node_cluster/storage/cfg/sys.txt new file mode 100644 index 00000000000..84e64278551 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/sys.txt @@ -0,0 +1,44 @@ +Executor { + Type: BASIC + Threads: 8 + SpinThreshold: 10 + Name: "System" +} +Executor { + Type: BASIC + Threads: 8 + SpinThreshold: 10 + Name: "User" +} +Executor { + Type: BASIC + Threads: 8 + SpinThreshold: 1 + Name: "Batch" +} +Executor { + Type: IO + Threads: 1 + Name: "IO" +} +Executor { + Type: BASIC + Threads: 3 + SpinThreshold: 10 + Name: "IC" + TimePerMailboxMicroSecs: 100 +} +Scheduler { + Resolution: 256 + SpinThreshold: 0 + ProgressThreshold: 10000 +} +SysExecutor: 0 +UserExecutor: 1 +IoExecutor: 3 +BatchExecutor: 2 +ServiceExecutor { + ServiceName: "Interconnect" + ExecutorId: 4 +} + diff --git a/deploy/8_node_cluster/storage/cfg/tracing.txt b/deploy/8_node_cluster/storage/cfg/tracing.txt new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/tracing.txt @@ -0,0 +1 @@ + diff --git a/deploy/8_node_cluster/storage/cfg/vdisks.txt b/deploy/8_node_cluster/storage/cfg/vdisks.txt new file mode 100644 index 00000000000..f90e81f3992 --- /dev/null +++ b/deploy/8_node_cluster/storage/cfg/vdisks.txt @@ -0,0 +1,17 @@ +VDiskKinds { + Kind: Log + BaseKind: Default + Config { + FreshUseDreg: true + AllowKeepFlags: false + } +} +VDiskKinds { + Kind: LocalMode + BaseKind: Default + Config { + HullCompLevel0MaxSstsAtOnce: 2 + HullCompSortedPartsNum: 1 + } +} + diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 00000000000..6f33cb37acf --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,27 @@ +# Sample configs for cluster deployment + +## Configs +There is a sample 8-node cluster configuration in the 8_node_cluster folder. It supposes that you have 8 hosts with fqdns: +* my.host1 +* my.host2 +* my.host3 +* my.host4 +* my.host5 +* my.host6 +* my.host7 +* my.host8 + +Those hosts should have a bunch of disks which should be available at the following paths: +* /dev/disk/by-partlabel/NVMEKIKIMR01 - required +* /dev/disk/by-partlabel/NVMEKIKIMR0[2-3] - optional +* /dev/disk/by-partlabel/ROTKIKIMR0[2-3] - optional +* /dev/disk/by-partlabel/ROTNBS[01-99] - optional +* /dev/disk/by-partlabel/NVMENBS[01-99] - optional + +The cluster is called "my_cluster". + +The configs include both NBS and Filestore. + +## Build and run +See the [instructions](/example/README.md) to find out how to build and run NBS binaries. +See the [instructions](/cloud/filestore/README.md) to find out how to build and run Filestore binaries.