From ad5892fa9e62ef1a41412f952d354f3890dd5851 Mon Sep 17 00:00:00 2001 From: drbasic Date: Sun, 21 Jan 2024 19:27:04 +0700 Subject: [PATCH] NBS-4826 send message about long running blob operations to TPartitionActor and TVolumeActor (#151) * NBS-4826 send message about long running blob operations to TPartitionActor and TVolumeActor * NBS-4826 Fix review issues * Remove unnecessary comments --- .../storage/partition/part_actor_readblob.cpp | 1 + .../partition/part_actor_writeblob.cpp | 14 +- .../libs/storage/partition/part_ut.cpp | 133 +++++++++++++----- .../partition_common/actor_read_blob.cpp | 25 ++-- .../partition_common/actor_read_blob.h | 9 +- .../partition_common/actor_read_blob_ut.cpp | 3 + .../storage/partition_common/events_private.h | 6 +- .../long_running_operation_companion.cpp | 53 +++++-- .../long_running_operation_companion.h | 7 +- .../long_running_operation_companion_ut.cpp | 7 + .../read_disk_registry_based_overlay.cpp | 6 +- .../storage/volume/volume_actor_stats.cpp | 4 +- 12 files changed, 194 insertions(+), 74 deletions(-) diff --git a/cloud/blockstore/libs/storage/partition/part_actor_readblob.cpp b/cloud/blockstore/libs/storage/partition/part_actor_readblob.cpp index 7ebb207ce16..6db1ca300f6 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_readblob.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_readblob.cpp @@ -47,6 +47,7 @@ void TPartitionActor::HandleReadBlob( auto readBlobActor = std::make_unique( requestInfo, SelfId(), + VolumeActorId, TabletID(), State->GetBlockSize(), StorageAccessMode, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_writeblob.cpp b/cloud/blockstore/libs/storage/partition/part_actor_writeblob.cpp index 58824908212..cd43782118e 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_writeblob.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_writeblob.cpp @@ -46,6 +46,7 @@ class TWriteBlobActor final public: TWriteBlobActor( const TActorId& tabletActorId, + const TActorId& volumeActorId, TRequestInfoPtr requestInfo, ui64 tabletId, std::unique_ptr request, @@ -84,6 +85,7 @@ class TWriteBlobActor final TWriteBlobActor::TWriteBlobActor( const TActorId& tabletActorId, + const TActorId& volumeActorId, TRequestInfoPtr requestInfo, ui64 tabletId, std::unique_ptr request, @@ -91,6 +93,7 @@ TWriteBlobActor::TWriteBlobActor( ui32 groupId) : TLongRunningOperationCompanion( tabletActorId, + volumeActorId, longRunningThreshold, TLongRunningOperationCompanion::EOperation::WriteBlob, groupId) @@ -188,8 +191,13 @@ void TWriteBlobActor::ReplyAndDie( RequestInfo->CallContext->RequestId); } + if (HasError(response->GetError())) { + TLongRunningOperationCompanion::RequestCancelled(ctx); + } else { + TLongRunningOperationCompanion::RequestFinished(ctx); + } + NCloud::Reply(ctx, *RequestInfo, std::move(response)); - TLongRunningOperationCompanion::RequestFinished(ctx); Die(ctx); } @@ -340,6 +348,7 @@ void TPartitionActor::HandleWriteBlob( channel, std::make_unique( SelfId(), + VolumeActorId, requestInfo, TabletID(), std::unique_ptr( @@ -450,9 +459,6 @@ void TPartitionActor::HandleLongRunningBlobOperation( State->RegisterDowntime(ctx.Now(), msg.GroupId); Actors.MarkLongRunning(ev->Sender, msg.Operation); } - - // Forward request to TVolumeActor - ctx.Send(ev->Forward(VolumeActorId)); } } // namespace NCloud::NBlockStore::NStorage::NPartition diff --git a/cloud/blockstore/libs/storage/partition/part_ut.cpp b/cloud/blockstore/libs/storage/partition/part_ut.cpp index ed148f5e5a6..fe0dac1d96d 100644 --- a/cloud/blockstore/libs/storage/partition/part_ut.cpp +++ b/cloud/blockstore/libs/storage/partition/part_ut.cpp @@ -10252,7 +10252,8 @@ Y_UNIT_TEST_SUITE(TPartitionTest) FSend sendRequest, FReceive receiveResponse, TEvPartitionCommonPrivate::TEvLongRunningOperation::EOperation - expectedOperation) + expectedOperation, + bool killPartition) { auto config = DefaultConfig(); @@ -10296,34 +10297,39 @@ Y_UNIT_TEST_SUITE(TPartitionTest) // Make handler for intercepting EvLongRunningOperation message. // Attention! counters will be doubled, because we will intercept the - // initial request, and forwarded to TVolumeActor. + // requests sent to TPartitionActor and TVolumeActor. ui32 longRunningBeginCount = 0; ui32 longRunningFinishCount = 0; ui32 longRunningPingCount = 0; + ui32 longRunningCanceledCount = 0; auto takeCounters = [&](TTestActorRuntimeBase& runtime, TAutoPtr& event) { using TEvLongRunningOperation = TEvPartitionCommonPrivate::TEvLongRunningOperation; - switch (event->GetTypeRewrite()) { - case TEvPartitionCommonPrivate::EvLongRunningOperation: { - auto* msg = event->Get(); - UNIT_ASSERT_VALUES_EQUAL( - expectedOperation, - msg->Operation); + using EReason = TEvLongRunningOperation::EReason; + + if (event->GetTypeRewrite() == + TEvPartitionCommonPrivate::EvLongRunningOperation) + { + auto* msg = event->Get(); - if (msg->Reason == - TEvLongRunningOperation::EReason::LongRunningDetected) - { + UNIT_ASSERT_VALUES_EQUAL(expectedOperation, msg->Operation); + + switch (msg->Reason) { + case EReason::LongRunningDetected: if (msg->FirstNotify) { longRunningBeginCount++; } else { longRunningPingCount++; } - } else { + break; + case EReason::Finished: longRunningFinishCount++; - } - break; + break; + case EReason::Cancelled: + longRunningCanceledCount++; + break; } } return TTestActorRuntime::DefaultObserverFunc(runtime, event); @@ -10368,9 +10374,23 @@ Y_UNIT_TEST_SUITE(TPartitionTest) runtime->DispatchEvents(options, TDuration::Seconds(1)); } - // Returning stolen requests to complete the execution of the request. - for (auto& request: stolenRequests) { - runtime->Send(request.release()); + if (killPartition) { + partition.KillTablet(); + } else { + // Returning stolen requests to complete the execution of the + // request. + for (auto& request: stolenRequests) { + runtime->Send(request.release()); + } + } + + // Wait for EvLongRunningOperation (finish or cancel) arrival. + { + TDispatchOptions options; + options.FinalEvents.emplace_back( + TEvPartitionCommonPrivate::EvLongRunningOperation); + runtime->AdvanceCurrentTime(TDuration::Seconds(60)); + runtime->DispatchEvents(options, TDuration::Seconds(1)); } // Wait for background operations completion. @@ -10380,11 +10400,10 @@ Y_UNIT_TEST_SUITE(TPartitionTest) { auto response = receiveResponse(partition); UNIT_ASSERT_VALUES_EQUAL_C( - S_OK, + killPartition ? E_REJECTED : S_OK, response->GetStatus(), response->GetErrorReason()); } - // Wait for EvVolumePartCounters arrived. { TDispatchOptions options; @@ -10392,41 +10411,43 @@ Y_UNIT_TEST_SUITE(TPartitionTest) TEvStatsService::EvVolumePartCounters); runtime->DispatchEvents(options); + UNIT_ASSERT_VALUES_EQUAL(2, longRunningBeginCount); + UNIT_ASSERT_VALUES_EQUAL(4, longRunningPingCount); UNIT_ASSERT_VALUES_EQUAL( - 2, - longRunningBeginCount); - UNIT_ASSERT_VALUES_EQUAL( - 2, + killPartition ? 0 : 2, longRunningFinishCount); UNIT_ASSERT_VALUES_EQUAL( - 4, - longRunningPingCount); + killPartition ? 2 : 0, + longRunningCanceledCount); } - // smoke test for monpage - auto channelsTab = partition.RemoteHttpInfo( - BuildRemoteHttpQuery(TestTabletId, {}, "Channels"), - HTTP_METHOD::HTTP_METHOD_GET); + if (!killPartition) { + // smoke test for monpage + auto channelsTab = partition.RemoteHttpInfo( + BuildRemoteHttpQuery(TestTabletId, {}, "Channels"), + HTTP_METHOD::HTTP_METHOD_GET); - UNIT_ASSERT_C(channelsTab->Html.Contains("svg"), channelsTab->Html); + UNIT_ASSERT_C(channelsTab->Html.Contains("svg"), channelsTab->Html); + } } Y_UNIT_TEST(ShouldReportLongRunningReadBlobOperations) { auto sendRequest = [](TPartitionClient& partition) { - partition.SendReadBlocksRequest(0); // + partition.SendReadBlocksRequest(0); }; auto receiveResponse = [](TPartitionClient& partition) { - return partition.RecvReadBlocksResponse(); // + return partition.RecvReadBlocksResponse(); }; DoShouldReportLongRunningBlobOperations( sendRequest, receiveResponse, TEvPartitionCommonPrivate::TEvLongRunningOperation::EOperation:: - ReadBlob); + ReadBlob, + false); } Y_UNIT_TEST(ShouldReportLongRunningWriteBlobOperations) @@ -10434,18 +10455,58 @@ Y_UNIT_TEST_SUITE(TPartitionTest) auto sendRequest = [](TPartitionClient& partition) { partition.SendWriteBlocksRequest( - TBlockRange32::WithLength(0, 255)); // + TBlockRange32::WithLength(0, 255)); + }; + auto receiveResponse = [](TPartitionClient& partition) + { + return partition.RecvWriteBlocksResponse(); + }; + + DoShouldReportLongRunningBlobOperations( + sendRequest, + receiveResponse, + TEvPartitionCommonPrivate::TEvLongRunningOperation::EOperation:: + WriteBlob, + false); + } + + Y_UNIT_TEST(ShouldReportLongRunningReadBlobOperationCancel) + { + auto sendRequest = [](TPartitionClient& partition) + { + partition.SendReadBlocksRequest(0); + }; + auto receiveResponse = [](TPartitionClient& partition) + { + return partition.RecvReadBlocksResponse(); + }; + + DoShouldReportLongRunningBlobOperations( + sendRequest, + receiveResponse, + TEvPartitionCommonPrivate::TEvLongRunningOperation::EOperation:: + ReadBlob, + true); + } + + Y_UNIT_TEST(ShouldReportLongRunningWriteBlobOperationCancel) + { + auto sendRequest = [](TPartitionClient& partition) + { + partition.SendWriteBlocksRequest( + TBlockRange32::WithLength(0, 255)); }; auto receiveResponse = [](TPartitionClient& partition) { - return partition.RecvWriteBlocksResponse(); // + return partition.RecvWriteBlocksResponse(); }; DoShouldReportLongRunningBlobOperations( sendRequest, receiveResponse, TEvPartitionCommonPrivate::TEvLongRunningOperation::EOperation:: - WriteBlob); + WriteBlob, + true); } } diff --git a/cloud/blockstore/libs/storage/partition_common/actor_read_blob.cpp b/cloud/blockstore/libs/storage/partition_common/actor_read_blob.cpp index 54ae6b2cc2c..1a0f97504f2 100644 --- a/cloud/blockstore/libs/storage/partition_common/actor_read_blob.cpp +++ b/cloud/blockstore/libs/storage/partition_common/actor_read_blob.cpp @@ -15,20 +15,22 @@ LWTRACE_USING(BLOCKSTORE_STORAGE_PROVIDER); TReadBlobActor::TReadBlobActor( TRequestInfoPtr requestInfo, - const TActorId& tablet, - ui64 tabletId, + const TActorId& partitionActorId, + const TActorId& volumeActorId, + ui64 partitionTabletId, ui32 blockSize, const EStorageAccessMode storageAccessMode, std::unique_ptr request, TDuration longRunningThreshold) : TLongRunningOperationCompanion( - tablet, + partitionActorId, + volumeActorId, longRunningThreshold, TLongRunningOperationCompanion::EOperation::ReadBlob, request->GroupId) , RequestInfo(std::move(requestInfo)) - , Tablet(tablet) - , TabletId(tabletId) + , PartitionActorId(partitionActorId) + , PartitionTabletId(partitionTabletId) , BlockSize(blockSize) , StorageAccessMode(storageAccessMode) , Request(std::move(request)) @@ -104,7 +106,7 @@ void TReadBlobActor::NotifyCompleted( request->RequestTime = ResponseReceived - RequestSent; request->GroupId = Request->GroupId; - NCloud::Send(ctx, Tablet, std::move(request)); + NCloud::Send(ctx, PartitionActorId, std::move(request)); } void TReadBlobActor::ReplyAndDie( @@ -123,8 +125,13 @@ void TReadBlobActor::ReplyAndDie( RequestInfo->CallContext->RequestId); } + if (HasError(response->GetError())) { + TLongRunningOperationCompanion::RequestCancelled(ctx); + } else { + TLongRunningOperationCompanion::RequestFinished(ctx); + } + NCloud::Reply(ctx, *RequestInfo, std::move(response)); - TLongRunningOperationCompanion::RequestFinished(ctx); Die(ctx); } @@ -135,7 +142,7 @@ void TReadBlobActor::ReplyError( { LOG_ERROR(ctx, TBlockStoreComponents::PARTITION_COMMON, "[%lu] TEvBlobStorage::TEvGet failed: %s\n%s", - TabletId, + PartitionTabletId, description.data(), response.Print(false).data()); @@ -176,7 +183,7 @@ void TReadBlobActor::HandleGetResult( { LOG_WARN(ctx, TBlockStoreComponents::PARTITION_COMMON, "[%lu] Repairing TEvBlobStorage::TEvGet %s error (%s)", - TabletId, + PartitionTabletId, NKikimrProto::EReplyStatus_Name(response.Status).data(), msg->Print(false).data()); diff --git a/cloud/blockstore/libs/storage/partition_common/actor_read_blob.h b/cloud/blockstore/libs/storage/partition_common/actor_read_blob.h index 01ab24b81a3..9b5b9f5e2cc 100644 --- a/cloud/blockstore/libs/storage/partition_common/actor_read_blob.h +++ b/cloud/blockstore/libs/storage/partition_common/actor_read_blob.h @@ -25,8 +25,8 @@ class TReadBlobActor final private: const TRequestInfoPtr RequestInfo; - const NActors::TActorId Tablet; - const ui64 TabletId; + const NActors::TActorId PartitionActorId; + const ui64 PartitionTabletId; const ui32 BlockSize; const EStorageAccessMode StorageAccessMode; const std::unique_ptr Request; @@ -37,8 +37,9 @@ class TReadBlobActor final public: TReadBlobActor( TRequestInfoPtr requestInfo, - const NActors::TActorId& tablet, - ui64 tabletId, + const NActors::TActorId& partitionActorId, + const NActors::TActorId& volumeActorId, + ui64 partitionTabletId, ui32 blockSize, const EStorageAccessMode storageAccessMode, std::unique_ptr request, diff --git a/cloud/blockstore/libs/storage/partition_common/actor_read_blob_ut.cpp b/cloud/blockstore/libs/storage/partition_common/actor_read_blob_ut.cpp index 583b62ae54a..44d3f7a34d0 100644 --- a/cloud/blockstore/libs/storage/partition_common/actor_read_blob_ut.cpp +++ b/cloud/blockstore/libs/storage/partition_common/actor_read_blob_ut.cpp @@ -68,6 +68,7 @@ Y_UNIT_TEST_SUITE(TReadBlobTests) 0ull, MakeIntrusive()), EdgeActor, + NActors::TActorId(), 0, blockSize, EStorageAccessMode::Default, @@ -153,6 +154,7 @@ Y_UNIT_TEST_SUITE(TReadBlobTests) 0ull, MakeIntrusive()), EdgeActor, + NActors::TActorId(), 0, blockSize, EStorageAccessMode::Default, @@ -222,6 +224,7 @@ Y_UNIT_TEST_SUITE(TReadBlobTests) 0ull, MakeIntrusive()), EdgeActor, + NActors::TActorId(), 0, blockSize, EStorageAccessMode::Default, diff --git a/cloud/blockstore/libs/storage/partition_common/events_private.h b/cloud/blockstore/libs/storage/partition_common/events_private.h index bcb60e70bfb..5ae8cc32120 100644 --- a/cloud/blockstore/libs/storage/partition_common/events_private.h +++ b/cloud/blockstore/libs/storage/partition_common/events_private.h @@ -154,6 +154,7 @@ struct TEvPartitionCommonPrivate { LongRunningDetected, Finished, + Cancelled, }; const EOperation Operation; @@ -167,11 +168,10 @@ struct TEvPartitionCommonPrivate bool firstNotify, TDuration duration, ui32 groupId, - bool finished) + EReason reason) : Operation(operation) , FirstNotify(firstNotify) - , Reason( - finished ? EReason::Finished : EReason::LongRunningDetected) + , Reason(reason) , Duration(duration) , GroupId(groupId) {} diff --git a/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion.cpp b/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion.cpp index c0e227641b6..8d794d71bd3 100644 --- a/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion.cpp +++ b/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion.cpp @@ -68,11 +68,13 @@ TRunningActors::TTimeoutsStat TRunningActors::ExtractLongRunningStat() //////////////////////////////////////////////////////////////////////////////// TLongRunningOperationCompanion::TLongRunningOperationCompanion( - NActors::TActorId parentActor, + NActors::TActorId partitionActor, + NActors::TActorId volumeActor, TDuration longRunningThreshold, TLongRunningOperationCompanion::EOperation operation, ui32 groupId) - : ParentActor(parentActor) + : PartitionActor(partitionActor) + , VolumeActor(volumeActor) , Operation(operation) , GroupId(groupId) , PingDelayProvider( @@ -92,44 +94,71 @@ void TLongRunningOperationCompanion::RequestStarted(const TActorContext& ctx) void TLongRunningOperationCompanion::RequestFinished(const TActorContext& ctx) { + using TEvLongRunningOperation = + TEvPartitionCommonPrivate::TEvLongRunningOperation; + if (!LongRunningDetected) { return; } - NCloud::Send( - ctx, - ParentActor, - std::make_unique( + auto makeMessage = [&]() + { + return std::make_unique( Operation, true, ctx.Now() - StartAt, GroupId, - true)); + TEvLongRunningOperation::EReason::Finished); + }; + NCloud::Send(ctx, PartitionActor, makeMessage()); + NCloud::Send(ctx, VolumeActor, makeMessage()); } void TLongRunningOperationCompanion::HandleTimeout( const TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx) { + using TEvLongRunningOperation = + TEvPartitionCommonPrivate::TEvLongRunningOperation; Y_UNUSED(ev); LongRunningDetected = true; PingCount++; - NCloud::Send( - ctx, - ParentActor, - std::make_unique( + auto makeMessage = [&]() + { + return std::make_unique( Operation, PingCount == 1, ctx.Now() - StartAt, GroupId, - false)); + TEvLongRunningOperation::EReason::LongRunningDetected); + }; + NCloud::Send(ctx, PartitionActor, makeMessage()); + NCloud::Send(ctx, VolumeActor, makeMessage()); PingDelayProvider.IncreaseDelay(); ctx.Schedule(PingDelayProvider.GetDelay(), new TEvents::TEvWakeup()); } +void TLongRunningOperationCompanion::RequestCancelled( + const NActors::TActorContext& ctx) +{ + using TEvLongRunningOperation = + TEvPartitionCommonPrivate::TEvLongRunningOperation; + auto makeMessage = [&]() + { + return std::make_unique( + Operation, + true, + ctx.Now() - StartAt, + GroupId, + TEvLongRunningOperation::EReason::Cancelled); + }; + NCloud::Send(ctx, PartitionActor, makeMessage()); + NCloud::Send(ctx, VolumeActor, makeMessage()); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion.h b/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion.h index 05b5bb52f9b..d974fb7bf71 100644 --- a/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion.h +++ b/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion.h @@ -45,7 +45,8 @@ class TLongRunningOperationCompanion TEvPartitionCommonPrivate::TEvLongRunningOperation::EOperation; private: - const NActors::TActorId ParentActor; + const NActors::TActorId PartitionActor; + const NActors::TActorId VolumeActor; const EOperation Operation; const ui32 GroupId; @@ -56,13 +57,15 @@ class TLongRunningOperationCompanion public: TLongRunningOperationCompanion( - NActors::TActorId parentActor, + NActors::TActorId partitionActor, + NActors::TActorId volumeActor, TDuration longRunningThreshold, EOperation operation, ui32 groupId); void RequestStarted(const NActors::TActorContext& ctx); void RequestFinished(const NActors::TActorContext& ctx); + void RequestCancelled(const NActors::TActorContext& ctx); void HandleTimeout( const NActors::TEvents::TEvWakeup::TPtr& ev, diff --git a/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion_ut.cpp b/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion_ut.cpp index 94187150b0a..962b0746c3a 100644 --- a/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion_ut.cpp +++ b/cloud/blockstore/libs/storage/partition_common/long_running_operation_companion_ut.cpp @@ -68,6 +68,7 @@ class TActorWithCompanion TActorWithCompanion(TActorId parentActor) : TActor(&TThis::Main) , TLongRunningOperationCompanion( + parentActor, parentActor, TDuration::MilliSeconds(500), TLongRunningOperationCompanion::EOperation::DontCare, @@ -117,10 +118,16 @@ Y_UNIT_TEST_SUITE(TLongRunningOperationCompanionTest) testEnv.DispatchEvents(TDuration::Seconds(1)); { + // Get two LongRunningEvent (for TPartitionActor and TVolumeActor) auto longRunningEvent = testEnv.GrabLongRunningEvent(); UNIT_ASSERT_VALUES_EQUAL( TLongRunningOperationCompanion::EOperation::DontCare, longRunningEvent.Get()->Operation); + + longRunningEvent = testEnv.GrabLongRunningEvent(); + UNIT_ASSERT_VALUES_EQUAL( + TLongRunningOperationCompanion::EOperation::DontCare, + longRunningEvent.Get()->Operation); } } } diff --git a/cloud/blockstore/libs/storage/volume/actors/read_disk_registry_based_overlay.cpp b/cloud/blockstore/libs/storage/volume/actors/read_disk_registry_based_overlay.cpp index 2431de029b2..7707170d62c 100644 --- a/cloud/blockstore/libs/storage/volume/actors/read_disk_registry_based_overlay.cpp +++ b/cloud/blockstore/libs/storage/volume/actors/read_disk_registry_based_overlay.cpp @@ -370,6 +370,7 @@ void TReadDiskRegistryBasedOverlayActor::HandleDescribeBlocksCompleted( ctx, requestInfo, TBase::SelfId(), + VolumeActorId, VolumeTabletId, BlockSize, Mode, @@ -395,6 +396,7 @@ void TReadDiskRegistryBasedOverlayActor::HandleDescribeBlocksCompleted( ctx, requestInfo, TBase::SelfId(), + VolumeActorId, VolumeTabletId, BlockSize, EStorageAccessMode::Default, @@ -430,8 +432,8 @@ void TReadDiskRegistryBasedOverlayActor::HandleLongRunningBlobOperation const TEvPartitionCommonPrivate::TEvLongRunningOperation::TPtr& ev, const TActorContext& ctx) { - // Forward request to TVolumeActor - ctx.Send(ev->Forward(VolumeActorId)); + Y_UNUSED(ev); + Y_UNUSED(ctx); } template diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_stats.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_stats.cpp index edea4c48dd7..58c997807e7 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_stats.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_stats.cpp @@ -418,10 +418,10 @@ void TVolumeActor::HandleLongRunningBlobOperation( LOG_WARN( ctx, TBlockStoreComponents::VOLUME, - "[%lu] For volume %s completion %s (actor %s, group %u) " - "detected after %s", + "[%lu] For volume %s %s %s (actor %s, group %u) detected after %s", TabletID(), State->GetDiskId().Quote().c_str(), + ToString(ev->Get()->Reason).c_str(), ToString(msg.Operation).c_str(), ev->Sender.ToString().c_str(), msg.GroupId,