From 3e3896fe51703f82b2f1460259580844d2811025 Mon Sep 17 00:00:00 2001 From: Andrei Strelkovskii Date: Thu, 1 Feb 2024 15:57:04 +0300 Subject: [PATCH] merge to stable-23-3 (#314) * loadtest env tear_down should happen inside finally blocks because otherwise we may get an exception because blockstore-loadtest run failed and will never know that kikimr daemon crashed (#290) * issue #95 NBSNEBIUS-70: if Flush/Compaction/Cleanup/FlushBytes ops are enqueued before CompactionMap is completely loaded, their EOperationStates should not get stuck in the Enqueued state (#303) --- .../tests/loadtest/local-auth/test.py | 25 ++-- .../tests/loadtest/local-checkpoint/test.py | 21 +-- .../tests/loadtest/local-discovery/test.py | 39 +++--- .../tests/loadtest/local-edgecase/test.py | 19 +-- .../tests/loadtest/local-emergency/test.py | 19 +-- .../tests/loadtest/local-encryption/test.py | 23 ++-- .../tests/loadtest/local-endpoints/test.py | 23 ++-- .../tests/loadtest/local-mirror/test.py | 31 ++--- .../tests/loadtest/local-nemesis/test.py | 25 ++-- .../tests/loadtest/local-newfeatures/test.py | 23 ++-- .../tests/loadtest/local-nonrepl/test.py | 29 +++-- .../tests/loadtest/local-overflow/test.py | 21 +-- .../tests/loadtest/local-overlay/test.py | 21 +-- .../tests/loadtest/local-throttling/test.py | 23 ++-- .../tests/loadtest/local-v2/test.py | 21 +-- cloud/blockstore/tests/loadtest/local/test.py | 25 ++-- .../storage/tablet/tablet_actor_cleanup.cpp | 4 + .../tablet/tablet_actor_compaction.cpp | 4 + .../storage/tablet/tablet_actor_counters.cpp | 6 + .../storage/tablet/tablet_actor_flush.cpp | 4 + .../tablet/tablet_actor_flush_bytes.cpp | 4 + .../libs/storage/tablet/tablet_ut_data.cpp | 121 ++++++++++++++++++ .../filestore/private/api/protos/tablet.proto | 5 + 23 files changed, 350 insertions(+), 186 deletions(-) diff --git a/cloud/blockstore/tests/loadtest/local-auth/test.py b/cloud/blockstore/tests/loadtest/local-auth/test.py index a09cbfe014b..755f7e81b77 100644 --- a/cloud/blockstore/tests/loadtest/local-auth/test.py +++ b/cloud/blockstore/tests/loadtest/local-auth/test.py @@ -43,17 +43,18 @@ def test_load(): "cloud/blockstore/tests/certs/server.crt") client.AuthToken = "test_auth_token" - ret = run_test( - "load", - common.source_path( - "cloud/blockstore/tests/loadtest/local-auth/local.txt"), - env.nbs_secure_port, - env.mon_port, - client_config=client, - enable_tls=True, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + "load", + common.source_path( + "cloud/blockstore/tests/loadtest/local-auth/local.txt"), + env.nbs_secure_port, + env.mon_port, + client_config=client, + enable_tls=True, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-checkpoint/test.py b/cloud/blockstore/tests/loadtest/local-checkpoint/test.py index f77fa631848..304e7a919f8 100644 --- a/cloud/blockstore/tests/loadtest/local-checkpoint/test.py +++ b/cloud/blockstore/tests/loadtest/local-checkpoint/test.py @@ -123,16 +123,17 @@ def __run_test(test_case): use_in_memory_pdisks=True, ) - ret = run_test( - test_case.name, - test_case.config_path, - env.nbs_port, - env.mon_port, - nbs_log_path=env.nbs_log_path, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + test_case.name, + test_case.config_path, + env.nbs_port, + env.mon_port, + nbs_log_path=env.nbs_log_path, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-discovery/test.py b/cloud/blockstore/tests/loadtest/local-discovery/test.py index 78632fb7f05..b495955f0bb 100644 --- a/cloud/blockstore/tests/loadtest/local-discovery/test.py +++ b/cloud/blockstore/tests/loadtest/local-discovery/test.py @@ -115,27 +115,28 @@ def test_load(): use_in_memory_pdisks=True, ) - ret = run_test( - "load", - common.source_path( - "cloud/blockstore/tests/loadtest/local-discovery/local.txt"), - env.nbs_port, - env.mon_port, - env_processes=[env.nbs], - ) - - results_path = os.path.join(common.output_path(), "results.txt") + try: + ret = run_test( + "load", + common.source_path( + "cloud/blockstore/tests/loadtest/local-discovery/local.txt"), + env.nbs_port, + env.mon_port, + env_processes=[env.nbs], + ) - def is_good(port): - return port in good_ports + results_path = os.path.join(common.output_path(), "results.txt") - with open(results_path, "r") as f: - for line in f: - data = json.loads(line) - ports = [x['Port'] for x in data['Instances']] - for port in ports: - assert is_good(port) + def is_good(port): + return port in good_ports - env.tear_down() + with open(results_path, "r") as f: + for line in f: + data = json.loads(line) + ports = [x['Port'] for x in data['Instances']] + for port in ports: + assert is_good(port) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-edgecase/test.py b/cloud/blockstore/tests/loadtest/local-edgecase/test.py index 0e3716bab5c..5c3fb12906d 100644 --- a/cloud/blockstore/tests/loadtest/local-edgecase/test.py +++ b/cloud/blockstore/tests/loadtest/local-edgecase/test.py @@ -131,15 +131,16 @@ def __run_test(test_case): use_in_memory_pdisks=True, ) - ret = run_test( - test_case.name, - test_case.config_path, - env.nbs_port, - env.mon_port, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + test_case.name, + test_case.config_path, + env.nbs_port, + env.mon_port, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-emergency/test.py b/cloud/blockstore/tests/loadtest/local-emergency/test.py index e0f25169979..8c4969cea1f 100644 --- a/cloud/blockstore/tests/loadtest/local-emergency/test.py +++ b/cloud/blockstore/tests/loadtest/local-emergency/test.py @@ -99,15 +99,16 @@ def __run_test(test_case): env.kikimr_cluster.restart_nodes() env.nbs.restart() - ret = run_test( - "emergency-%s" % test_case.name, - common.source_path(test_case.config_path), - env.nbs_port, - env.mon_port, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + "emergency-%s" % test_case.name, + common.source_path(test_case.config_path), + env.nbs_port, + env.mon_port, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-encryption/test.py b/cloud/blockstore/tests/loadtest/local-encryption/test.py index cf19979c7ed..c2a77b0d775 100644 --- a/cloud/blockstore/tests/loadtest/local-encryption/test.py +++ b/cloud/blockstore/tests/loadtest/local-encryption/test.py @@ -121,17 +121,18 @@ def __run_test(test_case): client = TClientConfig() client.NbdSocketSuffix = nbd_socket_suffix - ret = run_test( - test_case.name, - config_path, - env.nbs_port, - env.mon_port, - nbs_log_path=env.nbs_log_path, - client_config=client, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + test_case.name, + config_path, + env.nbs_port, + env.mon_port, + nbs_log_path=env.nbs_log_path, + client_config=client, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-endpoints/test.py b/cloud/blockstore/tests/loadtest/local-endpoints/test.py index 09f55da672d..6a31afb7e41 100644 --- a/cloud/blockstore/tests/loadtest/local-endpoints/test.py +++ b/cloud/blockstore/tests/loadtest/local-endpoints/test.py @@ -118,17 +118,18 @@ def __run_test(test_case): client_config.NbdSocketSuffix = nbd_socket_suffix client_config.NbdStructuredReply = test_case.nbd_structured_reply - ret = run_test( - test_case.name, - test_case.config_path, - env.nbs_port, - env.mon_port, - nbs_log_path=env.nbs_log_path, - client_config=client_config, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + test_case.name, + test_case.config_path, + env.nbs_port, + env.mon_port, + nbs_log_path=env.nbs_log_path, + client_config=client_config, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-mirror/test.py b/cloud/blockstore/tests/loadtest/local-mirror/test.py index a27400ffad2..d0a4a077040 100644 --- a/cloud/blockstore/tests/loadtest/local-mirror/test.py +++ b/cloud/blockstore/tests/loadtest/local-mirror/test.py @@ -259,21 +259,22 @@ def __run_test(test_case): config_path = __process_config(test_case.config_path, devices_per_agent) - ret = run_test( - test_case.name, - config_path, - nbs.nbs_port, - nbs.mon_port, - nbs_log_path=nbs.stderr_file_name, - client_config=client, - env_processes=disk_agents + [nbs], - ) - - for disk_agent in disk_agents: - disk_agent.stop() - - nbs.stop() - kikimr_cluster.stop() + try: + ret = run_test( + test_case.name, + config_path, + nbs.nbs_port, + nbs.mon_port, + nbs_log_path=nbs.stderr_file_name, + client_config=client, + env_processes=disk_agents + [nbs], + ) + finally: + for disk_agent in disk_agents: + disk_agent.stop() + + nbs.stop() + kikimr_cluster.stop() finally: __cleanup_file_devices(devices) diff --git a/cloud/blockstore/tests/loadtest/local-nemesis/test.py b/cloud/blockstore/tests/loadtest/local-nemesis/test.py index 52d7a019f7d..ab1c026a71c 100644 --- a/cloud/blockstore/tests/loadtest/local-nemesis/test.py +++ b/cloud/blockstore/tests/loadtest/local-nemesis/test.py @@ -107,18 +107,19 @@ def __run_test(test_case): client.NbdSocketSuffix = nbd_socket_suffix client.NbdStructuredReply = test_case.nbd_structured_reply - ret = run_test( - test_case.name, - test_case.config_path, - env.nbs_port, - env.mon_port, - nbs_log_path=env.nbs_log_path, - client_config=client, - endpoint_storage_dir=endpoint_storage_dir, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + test_case.name, + test_case.config_path, + env.nbs_port, + env.mon_port, + nbs_log_path=env.nbs_log_path, + client_config=client, + endpoint_storage_dir=endpoint_storage_dir, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-newfeatures/test.py b/cloud/blockstore/tests/loadtest/local-newfeatures/test.py index f5ee3084cff..55639c3ad17 100644 --- a/cloud/blockstore/tests/loadtest/local-newfeatures/test.py +++ b/cloud/blockstore/tests/loadtest/local-newfeatures/test.py @@ -280,17 +280,18 @@ def __run_test(test_case): client = TClientConfig() client.NbdSocketSuffix = nbd_socket_suffix - ret = run_test( - test_case.name, - test_case.config_path, - env.nbs_port, - env.mon_port, - client_config=client, - endpoint_storage_dir=endpoint_storage_dir, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + test_case.name, + test_case.config_path, + env.nbs_port, + env.mon_port, + client_config=client, + endpoint_storage_dir=endpoint_storage_dir, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-nonrepl/test.py b/cloud/blockstore/tests/loadtest/local-nonrepl/test.py index 351dd9711bb..a923af80953 100644 --- a/cloud/blockstore/tests/loadtest/local-nonrepl/test.py +++ b/cloud/blockstore/tests/loadtest/local-nonrepl/test.py @@ -318,20 +318,21 @@ def __run_test(test_case): client = TClientConfig() client.NbdSocketSuffix = nbd_socket_suffix - ret = run_test( - test_case.name, - config_path, - nbs.nbs_port, - nbs.mon_port, - nbs_log_path=nbs.stderr_file_name, - client_config=client, - env_processes=disk_agents + [nbs]) - - for disk_agent in disk_agents: - disk_agent.stop() - - nbs.stop() - kikimr_cluster.stop() + try: + ret = run_test( + test_case.name, + config_path, + nbs.nbs_port, + nbs.mon_port, + nbs_log_path=nbs.stderr_file_name, + client_config=client, + env_processes=disk_agents + [nbs]) + finally: + for disk_agent in disk_agents: + disk_agent.stop() + + nbs.stop() + kikimr_cluster.stop() finally: if not test_case.use_memory_devices: cleanup_file_devices(devices) diff --git a/cloud/blockstore/tests/loadtest/local-overflow/test.py b/cloud/blockstore/tests/loadtest/local-overflow/test.py index 9a07129c08b..4b56fb69368 100644 --- a/cloud/blockstore/tests/loadtest/local-overflow/test.py +++ b/cloud/blockstore/tests/loadtest/local-overflow/test.py @@ -72,16 +72,17 @@ def __run_test(test_case): ], ) - ret = run_test( - "overflow-%s" % test_case.name, - common.source_path(test_case.config_path), - env.nbs_port, - env.mon_port, - stat_filter=test_case.stat_filter, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + "overflow-%s" % test_case.name, + common.source_path(test_case.config_path), + env.nbs_port, + env.mon_port, + stat_filter=test_case.stat_filter, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-overlay/test.py b/cloud/blockstore/tests/loadtest/local-overlay/test.py index 615092cc69f..b4f3174c807 100644 --- a/cloud/blockstore/tests/loadtest/local-overlay/test.py +++ b/cloud/blockstore/tests/loadtest/local-overlay/test.py @@ -140,16 +140,17 @@ def __run_test(test_case): use_in_memory_pdisks=True, ) - ret = run_test( - test_case.name, - test_case.config_path, - env.nbs_port, - env.mon_port, - nbs_log_path=env.nbs_log_path, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + test_case.name, + test_case.config_path, + env.nbs_port, + env.mon_port, + nbs_log_path=env.nbs_log_path, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-throttling/test.py b/cloud/blockstore/tests/loadtest/local-throttling/test.py index c881f183817..a896afba9f2 100644 --- a/cloud/blockstore/tests/loadtest/local-throttling/test.py +++ b/cloud/blockstore/tests/loadtest/local-throttling/test.py @@ -73,17 +73,18 @@ def __run_test(test_case): use_in_memory_pdisks=True, ) - ret = run_test( - "throttling-%s" % test_case.name, - common.source_path(test_case.config_path), - env.nbs_port, - env.mon_port, - nbs_log_path=env.nbs_log_path, - track_filter=test_case.track_filter, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + "throttling-%s" % test_case.name, + common.source_path(test_case.config_path), + env.nbs_port, + env.mon_port, + nbs_log_path=env.nbs_log_path, + track_filter=test_case.track_filter, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local-v2/test.py b/cloud/blockstore/tests/loadtest/local-v2/test.py index 15e0549d442..e2e9cc0b9e2 100644 --- a/cloud/blockstore/tests/loadtest/local-v2/test.py +++ b/cloud/blockstore/tests/loadtest/local-v2/test.py @@ -84,16 +84,17 @@ def __run_test(test_case): use_in_memory_pdisks=True, ) - ret = run_test( - test_case.name, - test_case.config_path, - env.nbs_port, - env.mon_port, - nbs_log_path=env.nbs_log_path, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + test_case.name, + test_case.config_path, + env.nbs_port, + env.mon_port, + nbs_log_path=env.nbs_log_path, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/blockstore/tests/loadtest/local/test.py b/cloud/blockstore/tests/loadtest/local/test.py index d2cae081733..70910bcdd66 100644 --- a/cloud/blockstore/tests/loadtest/local/test.py +++ b/cloud/blockstore/tests/loadtest/local/test.py @@ -182,18 +182,19 @@ def __run_test(test_case): bs_cache_file_path=cache_folder + "/bs_cache.txt", ) - ret = run_test( - test_case.name, - test_case.config_path, - env.nbs_port, - env.mon_port, - stat_filter=test_case.stat_filter, - nbs_log_path=env.nbs_log_path, - track_filter=test_case.track_filter, - env_processes=[env.nbs], - ) - - env.tear_down() + try: + ret = run_test( + test_case.name, + test_case.config_path, + env.nbs_port, + env.mon_port, + stat_filter=test_case.stat_filter, + nbs_log_path=env.nbs_log_path, + track_filter=test_case.track_filter, + env_processes=[env.nbs], + ) + finally: + env.tear_down() return ret diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp index 40958a9cbbe..30dad1a6304 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp @@ -41,6 +41,10 @@ void TIndexTabletActor::HandleCleanup( }; if (!CompactionStateLoadStatus.Finished) { + if (BlobIndexOpState.GetOperationState() == EOperationState::Enqueued) { + BlobIndexOpState.Complete(); + } + replyError(MakeError(E_TRY_AGAIN, "compaction state not loaded yet")); return; } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp index 87d950b5ac8..f6c8e4e6554 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp @@ -411,6 +411,10 @@ void TIndexTabletActor::HandleCompaction( }; if (!CompactionStateLoadStatus.Finished) { + if (BlobIndexOpState.GetOperationState() == EOperationState::Enqueued) { + BlobIndexOpState.Complete(); + } + replyError(MakeError(E_TRY_AGAIN, "compaction state not loaded yet")); return; } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp index 1bb30424a65..bd2198997b7 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp @@ -352,6 +352,12 @@ void TIndexTabletActor::HandleGetStorageStats( } } + stats->SetFlushState(static_cast(FlushState.GetOperationState())); + stats->SetBlobIndexOpState(static_cast( + BlobIndexOpState.GetOperationState())); + stats->SetCollectGarbageState(static_cast( + CollectGarbageState.GetOperationState())); + NCloud::Reply(ctx, *ev, std::move(response)); } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_flush.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_flush.cpp index baf5ff40950..913f71c205b 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_flush.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_flush.cpp @@ -271,6 +271,10 @@ void TIndexTabletActor::HandleFlush( }; if (!CompactionStateLoadStatus.Finished) { + if (FlushState.GetOperationState() == EOperationState::Enqueued) { + FlushState.Complete(); + } + replyError( ctx, *ev, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_flush_bytes.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_flush_bytes.cpp index 59a3bdd106b..6740e521cbc 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_flush_bytes.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_flush_bytes.cpp @@ -490,6 +490,10 @@ void TIndexTabletActor::HandleFlushBytes( }; if (!CompactionStateLoadStatus.Finished) { + if (BlobIndexOpState.GetOperationState() == EOperationState::Enqueued) { + BlobIndexOpState.Complete(); + } + replyError( ctx, *ev, diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp index f7c3133ec3a..85214e8961f 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp @@ -2,6 +2,7 @@ #include "tablet_schema.h" #include +#include #include #include @@ -2882,6 +2883,126 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) tablet.DestroyHandle(handle); } + TABLET_TEST(BackgroundOperationsShouldNotGetStuckForeverDuringCompactionMapLoading) + { + const auto block = tabletConfig.BlockSize; + + NProto::TStorageConfig storageConfig; + // hard to test anything apart from Compaction - it shares + // EOperationState with Cleanup and FlushBytes + storageConfig.SetCompactionThreshold(2); + // Flush has a separate EOperationState + storageConfig.SetFlushThreshold(1); + storageConfig.SetLoadedCompactionRangesPerTx(2); + storageConfig.SetWriteBlobThreshold(2 * block); + + TTestEnv env({}, std::move(storageConfig)); + + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + tabletConfig); + tablet.InitSession("client", "session"); + + auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); + auto handle = CreateHandle(tablet, id); + + // generating at least one compaction range + tablet.WriteData(handle, 0, block, 'a'); + + TAutoPtr loadChunk; + ui32 loadChunkCount = 0; + ui32 flushCount = 0; + ui32 compactionCount = 0; + env.GetRuntime().SetEventFilter([&] (auto& runtime, auto& event) { + Y_UNUSED(runtime); + + switch (event->GetTypeRewrite()) { + case TEvIndexTabletPrivate::EvFlushRequest: { + ++flushCount; + break; + } + + case TEvIndexTabletPrivate::EvCompactionRequest: { + ++compactionCount; + break; + } + + case TEvIndexTabletPrivate::EvLoadCompactionMapChunkRequest: { + ++loadChunkCount; + + // catching the second chunk - first one should be loaded + // so that we are able to write (and thus trigger our + // background ops) + if (loadChunkCount == 2) { + loadChunk = event.Release(); + return true; + } + } + } + + return false; + }); + + // rebooting to trigger compaction map reloading + tablet.RebootTablet(); + tablet.RecoverSession(); + + handle = CreateHandle(tablet, id); + + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(1)); + UNIT_ASSERT(loadChunk); + UNIT_ASSERT_VALUES_EQUAL(2, loadChunkCount); + + // this write should succeed - it targets the range that should be + // loaded at this point of time + tablet.SendWriteDataRequest(handle, 0, block, 'a'); + { + auto response = tablet.RecvWriteDataResponse(); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + } + + // Flush should've been triggered and its operation state should've + // been reset to Idle + UNIT_ASSERT_VALUES_EQUAL(1, flushCount); + + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL( + static_cast(EOperationState::Idle), + static_cast(stats.GetFlushState())); + } + + // this write should succeed - it targets the range that should be + // loaded at this point of time + tablet.SendWriteDataRequest(handle, 0, 2 * block, 'a'); + { + auto response = tablet.RecvWriteDataResponse(); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + } + + // Compaction should've been triggered and its operation state should've + // been reset to Idle + UNIT_ASSERT_VALUES_EQUAL(1, compactionCount); + + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL( + static_cast(EOperationState::Idle), + static_cast(stats.GetBlobIndexOpState())); + } + + tablet.DestroyHandle(handle); + } + #undef TABLET_TEST } diff --git a/cloud/filestore/private/api/protos/tablet.proto b/cloud/filestore/private/api/protos/tablet.proto index 94e196e571e..1cb002e7150 100644 --- a/cloud/filestore/private/api/protos/tablet.proto +++ b/cloud/filestore/private/api/protos/tablet.proto @@ -125,6 +125,11 @@ message TStorageStats // compaction map range stats repeated TCompactionRangeStats CompactionRangeStats = 3000; + + // background operation states + uint32 FlushState = 4001; + uint32 BlobIndexOpState = 4002; + uint32 CollectGarbageState = 4003; } message TGetStorageStatsRequest