From e9b87658b3f01d675d6764746034057222a53eb7 Mon Sep 17 00:00:00 2001 From: Kirill Pleshivtsev Date: Mon, 21 Oct 2024 15:47:19 +0700 Subject: [PATCH] Always respond with E_REJECT when killing a volume tablet actor (#1988) * Save all partition requests info in VolumeRequests * Test * Fix review issues * Small improvements --- .../storage/volume/multi_partition_requests.h | 34 -- .../libs/storage/volume/volume_actor.cpp | 3 - .../libs/storage/volume/volume_actor.h | 41 ++- .../volume/volume_actor_checkpoint.cpp | 11 - .../storage/volume/volume_actor_forward.cpp | 317 ++++++++++-------- .../volume/volume_actor_forward_trackused.cpp | 1 - .../storage/volume/volume_events_private.h | 8 +- .../libs/storage/volume/volume_ut.cpp | 96 ++++++ 8 files changed, 308 insertions(+), 203 deletions(-) diff --git a/cloud/blockstore/libs/storage/volume/multi_partition_requests.h b/cloud/blockstore/libs/storage/volume/multi_partition_requests.h index d51bedb092b..55e272184c5 100644 --- a/cloud/blockstore/libs/storage/volume/multi_partition_requests.h +++ b/cloud/blockstore/libs/storage/volume/multi_partition_requests.h @@ -54,8 +54,6 @@ class TMultiPartitionRequestActor final { private: const TRequestInfoPtr RequestInfo; - const NActors::TActorId VolumeActorId; - const ui64 VolumeRequestId; const TBlockRange64 OriginalRange; const ui32 BlocksPerStripe; const ui32 BlockSize; @@ -72,8 +70,6 @@ class TMultiPartitionRequestActor final public: TMultiPartitionRequestActor( TRequestInfoPtr requestInfo, - NActors::TActorId volumeActorId, - ui64 volumeRequestId, TBlockRange64 originalRange, ui32 blocksPerStripe, ui32 blockSize, @@ -276,8 +272,6 @@ class TMultiPartitionRequestActor final MergeCommonFields(src, dst); } - void NotifyCompleted(const NActors::TActorContext& ctx); - void ForkTraces(TCallContextPtr callContext) { auto& cc = RequestInfo->CallContext; @@ -305,8 +299,6 @@ class TMultiPartitionRequestActor final template TMultiPartitionRequestActor::TMultiPartitionRequestActor( TRequestInfoPtr requestInfo, - NActors::TActorId volumeActorId, - ui64 volumeRequestId, TBlockRange64 originalRange, ui32 blocksPerStripe, ui32 blockSize, @@ -314,8 +306,6 @@ TMultiPartitionRequestActor::TMultiPartitionRequestActor( TVector> partitionRequests, TRequestTraceInfo traceInfo) : RequestInfo(std::move(requestInfo)) - , VolumeActorId(volumeActorId) - , VolumeRequestId(volumeRequestId) , OriginalRange(originalRange) , BlocksPerStripe(blocksPerStripe) , BlockSize(blockSize) @@ -368,26 +358,6 @@ void TMultiPartitionRequestActor::Prepare( //////////////////////////////////////////////////////////////////////////////// -template -void TMultiPartitionRequestActor::NotifyCompleted( - const NActors::TActorContext& ctx) -{ - if constexpr (RequiresReadWriteAccess) { - using TEvent = TEvVolumePrivate::TEvMultipartitionWriteOrZeroCompleted; - auto ev = std::make_unique( - VolumeRequestId, - Record.GetError().GetCode()); - - NCloud::Send( - ctx, - VolumeActorId, - std::move(ev) - ); - } -} - -//////////////////////////////////////////////////////////////////////////////// - template void TMultiPartitionRequestActor::HandlePartitionResponse( const typename TMethod::TResponse::TPtr& ev, @@ -426,8 +396,6 @@ void TMultiPartitionRequestActor::HandlePartitionResponse( NCloud::Reply(ctx, *RequestInfo, std::move(response)); - NotifyCompleted(ctx); - TBase::Die(ctx); } } @@ -456,8 +424,6 @@ void TMultiPartitionRequestActor::HandleUndelivery( NCloud::Reply(ctx, *RequestInfo, std::move(response)); - NotifyCompleted(ctx); - TBase::Die(ctx); } diff --git a/cloud/blockstore/libs/storage/volume/volume_actor.cpp b/cloud/blockstore/libs/storage/volume/volume_actor.cpp index 914d72ef6bc..2b91595800b 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor.cpp @@ -1016,9 +1016,6 @@ STFUNC(TVolumeActor::StateWork) HandleDiskRegistryBasedPartCounters); HFunc(TEvStatsService::TEvVolumePartCounters, HandlePartCounters); HFunc(TEvVolumePrivate::TEvPartStatsSaved, HandlePartStatsSaved); - HFunc( - TEvVolumePrivate::TEvMultipartitionWriteOrZeroCompleted, - HandleMultipartitionWriteOrZeroCompleted); HFunc( TEvVolumePrivate::TEvWriteOrZeroCompleted, HandleWriteOrZeroCompleted); diff --git a/cloud/blockstore/libs/storage/volume/volume_actor.h b/cloud/blockstore/libs/storage/volume/volume_actor.h index 63a894edd85..4433deef009 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor.h +++ b/cloud/blockstore/libs/storage/volume/volume_actor.h @@ -140,12 +140,13 @@ class TVolumeActor final TCallContext& callContext, NProto::TError error); - NActors::TActorId Caller; - ui64 CallerCookie; - TCallContextPtr CallContext; - TCallContextPtr ForkedContext; - ui64 ReceiveTime; - TCancelRoutine* CancelRoutine; + const NActors::TActorId Caller; + const ui64 CallerCookie; + const TCallContextPtr CallContext; + const TCallContextPtr ForkedContext; + const ui64 ReceiveTime; + TCancelRoutine* const CancelRoutine; + const bool IsMultipartitionWriteOrZero; TVolumeRequest( const NActors::TActorId& caller, @@ -153,13 +154,15 @@ class TVolumeActor final TCallContextPtr callContext, TCallContextPtr forkedContext, ui64 receiveTime, - TCancelRoutine cancelRoutine) + TCancelRoutine cancelRoutine, + bool isMultipartitionWriteOrZero) : Caller(caller) , CallerCookie(callerCookie) , CallContext(std::move(callContext)) , ForkedContext(std::move(forkedContext)) , ReceiveTime(receiveTime) , CancelRoutine(cancelRoutine) + , IsMultipartitionWriteOrZero(isMultipartitionWriteOrZero) {} void CancelRequest( @@ -645,14 +648,18 @@ class TVolumeActor final const TEvVolumePrivate::TEvPartStatsSaved::TPtr& ev, const NActors::TActorContext& ctx); - void HandleMultipartitionWriteOrZeroCompleted( - const TEvVolumePrivate::TEvMultipartitionWriteOrZeroCompleted::TPtr& ev, - const NActors::TActorContext& ctx); - void HandleWriteOrZeroCompleted( const TEvVolumePrivate::TEvWriteOrZeroCompleted::TPtr& ev, const NActors::TActorContext& ctx); + template + bool ReplyToOriginalRequest( + const NActors::TActorContext& ctx, + NActors::TActorId sender, + NActors::IEventHandle::TEventFlags flags, + ui64 volumeRequestId, + std::unique_ptr response); + void ReplyToDuplicateRequests( const NActors::TActorContext& ctx, ui64 key, @@ -818,6 +825,15 @@ class TVolumeActor final TCallContext& callContext, ui64 startTime); + template + typename TMethod::TRequest::TPtr WrapRequest( + const typename TMethod::TRequest::TPtr& ev, + NActors::TActorId newRecipient, + ui64 volumeRequestId, + ui64 traceTime, + bool forkTraces, + bool isMultipartition); + template void SendRequestToPartition( const NActors::TActorContext& ctx, @@ -849,7 +865,6 @@ class TVolumeActor final void HandleCheckpointRequest( const NActors::TActorContext& ctx, const typename TMethod::TRequest::TPtr& ev, - ui64 volumeRequestId, bool isTraced, ui64 traceTs); @@ -865,7 +880,7 @@ class TVolumeActor final bool throttlingDisabled); template - void HandleResponse( + void ForwardResponse( const NActors::TActorContext& ctx, const typename TMethod::TResponse::TPtr& ev); diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_checkpoint.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_checkpoint.cpp index 2c16d0bbec2..15f1eccf66b 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_checkpoint.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_checkpoint.cpp @@ -926,12 +926,9 @@ template <> void TVolumeActor::HandleCheckpointRequest( const TActorContext& ctx, const TCreateCheckpointMethod::TRequest::TPtr& ev, - ui64 volumeRequestId, bool isTraced, ui64 traceTs) { - Y_UNUSED(volumeRequestId); - const auto& msg = *ev->Get(); auto requestInfo = CreateRequestInfo( @@ -962,12 +959,9 @@ template <> void TVolumeActor::HandleCheckpointRequest( const TActorContext& ctx, const TDeleteCheckpointMethod::TRequest::TPtr& ev, - ui64 volumeRequestId, bool isTraced, ui64 traceTs) { - Y_UNUSED(volumeRequestId); - const auto& msg = *ev->Get(); auto requestInfo = CreateRequestInfo( @@ -991,12 +985,9 @@ template <> void TVolumeActor::HandleCheckpointRequest( const TActorContext& ctx, const TDeleteCheckpointDataMethod::TRequest::TPtr& ev, - ui64 volumeRequestId, bool isTraced, ui64 traceTs) { - Y_UNUSED(volumeRequestId); - const auto& msg = *ev->Get(); auto requestInfo = CreateRequestInfo( @@ -1020,11 +1011,9 @@ template <> void TVolumeActor::HandleCheckpointRequest( const TActorContext& ctx, const TGetCheckpointStatusMethod::TRequest::TPtr& ev, - ui64 volumeRequestId, bool isTraced, ui64 traceTs) { - Y_UNUSED(volumeRequestId); Y_UNUSED(isTraced); Y_UNUSED(traceTs); diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp index 2ab5a3c3ccc..0bc00c4079e 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp @@ -88,8 +88,7 @@ bool TVolumeActor::HandleMultipartitionVolumeRequest( blocksPerStripe, ev, &partitionRequests, - &blockRange - ); + &blockRange); if (!ok) { return false; @@ -119,15 +118,20 @@ bool TVolumeActor::HandleMultipartitionVolumeRequest( ); } - if constexpr (RequiresReadWriteAccess) { - ++MultipartitionWriteAndZeroRequestsInProgress; - } + auto wrappedRequest = WrapRequest( + ev, + TActorId{}, + volumeRequestId, + traceTs, + false, + IsWriteMethod); NCloud::Register>( ctx, - CreateRequestInfo(ev->Sender, ev->Cookie, ev->Get()->CallContext), - SelfId(), - volumeRequestId, + CreateRequestInfo( + wrappedRequest->Sender, + wrappedRequest->Cookie, + wrappedRequest->Get()->CallContext), blockRange, blocksPerStripe, State->GetBlockSize(), @@ -140,6 +144,64 @@ bool TVolumeActor::HandleMultipartitionVolumeRequest( //////////////////////////////////////////////////////////////////////////////// +template +typename TMethod::TRequest::TPtr TVolumeActor::WrapRequest( + const typename TMethod::TRequest::TPtr& ev, + NActors::TActorId newRecipient, + ui64 volumeRequestId, + ui64 traceTime, + bool forkTraces, + bool isMultipartitionWriteOrZero) +{ + auto* msg = ev->Get(); + + auto originalContext = msg->CallContext; + if (forkTraces) { + msg->CallContext = + MakeIntrusive(originalContext->RequestId); + + if (!originalContext->LWOrbit.Fork(msg->CallContext->LWOrbit)) { + LWTRACK( + ForkFailed, + originalContext->LWOrbit, + TMethod::Name, + originalContext->RequestId); + } + } + + // We wrap the original message so that the response goes through + // TVolumeActor + auto selfId = SelfId(); + auto newEvent = typename TMethod::TRequest::TPtr( + static_cast(new IEventHandle( + newRecipient, + selfId, + ev->ReleaseBase().Release(), + IEventHandle::FlagForwardOnNondelivery, // flags + volumeRequestId, // cookie + &selfId // forwardOnNondelivery + ))); + + // We save the original sender to reply to him when we receive a response + // from the partition. + VolumeRequests.emplace( + volumeRequestId, + TVolumeRequest( + ev->Sender, + ev->Cookie, + std::move(originalContext), + forkTraces ? msg->CallContext : nullptr, + traceTime, + &RejectVolumeRequest, + isMultipartitionWriteOrZero)); + + if (isMultipartitionWriteOrZero) { + ++MultipartitionWriteAndZeroRequestsInProgress; + } + + return newEvent; +} + template void TVolumeActor::SendRequestToPartition( const TActorContext& ctx, @@ -167,46 +229,26 @@ void TVolumeActor::SendRequestToPartition( ToString(partActorId).data()); } - const bool processed = SendRequestToPartitionWithUsedBlockTracking( - ctx, + auto wrappedRequest = WrapRequest( ev, partActorId, - volumeRequestId); + volumeRequestId, + traceTime, + true, + false); - if (processed) { + if (SendRequestToPartitionWithUsedBlockTracking( + ctx, + wrappedRequest, + partActorId, + TabletID())) + { + // The request was sent to the partition with tracking of used blocks. return; } - auto* msg = ev->Get(); - - auto callContext = msg->CallContext; - msg->CallContext = MakeIntrusive(callContext->RequestId); - - if (!callContext->LWOrbit.Fork(msg->CallContext->LWOrbit)) { - LWTRACK(ForkFailed, callContext->LWOrbit, TMethod::Name, callContext->RequestId); - } - - auto selfId = SelfId(); - auto event = std::make_unique( - partActorId, - selfId, - ev->ReleaseBase().Release(), - IEventHandle::FlagForwardOnNondelivery, // flags - volumeRequestId, // cookie - &selfId // forwardOnNondelivery - ); - - VolumeRequests.emplace( - volumeRequestId, - TVolumeRequest( - ev->Sender, - ev->Cookie, - std::move(callContext), - msg->CallContext, - traceTime, - &RejectVolumeRequest)); - - ctx.Send(std::move(event)); + // Send request to the partition. + ctx.Send(wrappedRequest.Release()); } //////////////////////////////////////////////////////////////////////////////// @@ -297,75 +339,91 @@ NProto::TError TVolumeActor::ProcessAndValidateReadFromCheckpoint( //////////////////////////////////////////////////////////////////////////////// template -void TVolumeActor::HandleResponse( +void TVolumeActor::ForwardResponse( const NActors::TActorContext& ctx, const typename TMethod::TResponse::TPtr& ev) { - Y_UNUSED(ctx); - - auto* msg = ev->Get(); const ui64 volumeRequestId = ev->Cookie; - WriteAndZeroRequestsInFlight.RemoveRequest(volumeRequestId); - ReplyToDuplicateRequests( + + auto response = std::unique_ptr( + static_cast(ev->ReleaseBase().Release())); + + ReplyToOriginalRequest( ctx, + ev->Sender, + ev->Flags, volumeRequestId, - msg->Record.GetError().GetCode()); + std::move(response)); +} - if (auto it = VolumeRequests.find(volumeRequestId); - it != VolumeRequests.end()) - { - const TVolumeRequest& volumeRequest = it->second; - auto& cc = *volumeRequest.CallContext; - cc.LWOrbit.Join(it->second.ForkedContext->LWOrbit); +void TVolumeActor::HandleWriteOrZeroCompleted( + const TEvVolumePrivate::TEvWriteOrZeroCompleted::TPtr& ev, + const TActorContext& ctx) +{ + Y_UNUSED(ev); + Y_UNUSED(ctx); +} - FillResponse(*msg, cc, volumeRequest.ReceiveTime); +template +bool TVolumeActor::ReplyToOriginalRequest( + const NActors::TActorContext& ctx, + NActors::TActorId sender, + IEventHandle::TEventFlags flags, + ui64 volumeRequestId, + std::unique_ptr response) +{ + if constexpr (IsWriteMethod) { + ReplyToDuplicateRequests( + ctx, + volumeRequestId, + response->Record.GetError().GetCode()); + } - // forward response to the caller - auto event = std::make_unique( - volumeRequest.Caller, - ev->Sender, - ev->ReleaseBase().Release(), - ev->Flags, - volumeRequest.CallerCookie); + auto it = VolumeRequests.find(volumeRequestId); + if (it == VolumeRequests.end()) { + return false; + } - ctx.Send(event.release()); + const TVolumeRequest& volumeRequest = it->second; - VolumeRequests.erase(it); + if (volumeRequest.ForkedContext) { + volumeRequest.CallContext->LWOrbit.Join( + volumeRequest.ForkedContext->LWOrbit); } -} -void TVolumeActor::HandleMultipartitionWriteOrZeroCompleted( - const TEvVolumePrivate::TEvMultipartitionWriteOrZeroCompleted::TPtr& ev, - const TActorContext& ctx) -{ - auto* msg = ev->Get(); - - WriteAndZeroRequestsInFlight.RemoveRequest(msg->VolumeRequestId); - ReplyToDuplicateRequests(ctx, msg->VolumeRequestId, msg->ResultCode); + FillResponse( + *response, + *volumeRequest.CallContext, + volumeRequest.ReceiveTime); - Y_DEBUG_ABORT_UNLESS(MultipartitionWriteAndZeroRequestsInProgress > 0); - --MultipartitionWriteAndZeroRequestsInProgress; - ProcessCheckpointRequests(ctx); -} + // forward response to the caller + auto event = std::make_unique( + volumeRequest.Caller, + sender, + response.release(), + flags, + volumeRequest.CallerCookie); + ctx.Send(std::move(event)); -void TVolumeActor::HandleWriteOrZeroCompleted( - const TEvVolumePrivate::TEvWriteOrZeroCompleted::TPtr& ev, - const TActorContext& ctx) -{ - Y_UNUSED(ctx); + if (volumeRequest.IsMultipartitionWriteOrZero) { + Y_DEBUG_ABORT_UNLESS(MultipartitionWriteAndZeroRequestsInProgress > 0); + --MultipartitionWriteAndZeroRequestsInProgress; + ProcessCheckpointRequests(ctx); + } - auto* msg = ev->Get(); + VolumeRequests.erase(it); - WriteAndZeroRequestsInFlight.RemoveRequest(msg->VolumeRequestId); - ReplyToDuplicateRequests(ctx, msg->VolumeRequestId, msg->ResultCode); + return true; } void TVolumeActor::ReplyToDuplicateRequests( const TActorContext& ctx, - ui64 key, + ui64 volumeRequestId, ui32 resultCode) { - auto it = DuplicateWriteAndZeroRequests.find(key); + WriteAndZeroRequestsInFlight.RemoveRequest(volumeRequestId); + + auto it = DuplicateWriteAndZeroRequests.find(volumeRequestId); if (it == DuplicateWriteAndZeroRequests.end()) { return; } @@ -437,30 +495,18 @@ void TVolumeActor::ForwardRequest( // PartitionRequests undelivery handing if (ev->Sender == SelfId()) { const ui64 volumeRequestId = ev->Cookie; - if (auto it = VolumeRequests.find(volumeRequestId); - it != VolumeRequests.end()) - { - const TVolumeRequest& volumeRequest = it->second; - auto response = - std::make_unique(MakeError( - E_REJECTED, - TStringBuilder() - << "Volume not ready: " << State->GetDiskId().Quote())); - - FillResponse( - *response, - *volumeRequest.CallContext, - volumeRequest.ReceiveTime); - - NCloud::Send( - ctx, - volumeRequest.Caller, - std::move(response), - volumeRequest.CallerCookie); + auto response = std::make_unique(MakeError( + E_REJECTED, + TStringBuilder() + << "Volume not ready: " << State->GetDiskId().Quote())); - WriteAndZeroRequestsInFlight.RemoveRequest(volumeRequestId); - ReplyToDuplicateRequests(ctx, volumeRequestId, E_REJECTED); - VolumeRequests.erase(it); + if (ReplyToOriginalRequest( + ctx, + SelfId(), + 0, // flags + volumeRequestId, + std::move(response))) + { return; } } @@ -672,7 +718,7 @@ void TVolumeActor::ForwardRequest( * Processing overlapping writes. Overlapping writes should not be sent * to the underlying (storage) layer. */ - if constexpr (RequiresReadWriteAccess) { + if constexpr (IsWriteMethod) { const auto range = BuildRequestBlockRange( *msg, State->GetBlockSize()); @@ -698,12 +744,8 @@ void TVolumeActor::ForwardRequest( msg->CallContext->RequestId, addResult.DuplicateRequestId); - auto& q = - DuplicateWriteAndZeroRequests[addResult.DuplicateRequestId]; - - auto callContext = ev->Get()->CallContext; - q.push_back({ - std::move(callContext), + DuplicateWriteAndZeroRequests[addResult.DuplicateRequestId].push_back({ + ev->Get()->CallContext, static_cast(TMethod::TRequest::EventType), NActors::IEventHandlePtr(ev.Release()), now @@ -719,26 +761,33 @@ void TVolumeActor::ForwardRequest( } } - - /* - * Passing the request to the underlying (storage) layer. - */ + // Now we are ready to send the request to the underlying layer for + // processing. We need to save information about all sent requests in + // VolumeRequests to respond with E_REJECTED if the partition is stopped. + // When the underlying layer responds, we should remove the request from + // VolumeRequests. + // To do this, all requests before sending to the underlying actor are + // prepared by the WrapRequest() method, which replaces the sender + // and receiver. const bool isSinglePartitionVolume = State->GetPartitions().size() <= 1; if constexpr (IsCheckpointMethod) { - HandleCheckpointRequest(ctx, ev, volumeRequestId, isTraced, now); + HandleCheckpointRequest(ctx, ev, isTraced, now); } else if (isSinglePartitionVolume) { SendRequestToPartition(ctx, ev, volumeRequestId, 0, now); - } else if (!HandleMultipartitionVolumeRequest( - ctx, - ev, - volumeRequestId, - isTraced, - now)) - { - WriteAndZeroRequestsInFlight.RemoveRequest(volumeRequestId); - - replyError(MakeError(E_REJECTED, "Sglist destroyed")); + } else { + if (!HandleMultipartitionVolumeRequest( + ctx, + ev, + volumeRequestId, + isTraced, + now)) + { + if constexpr (IsWriteMethod) { + WriteAndZeroRequestsInFlight.RemoveRequest(volumeRequestId); + } + replyError(MakeError(E_REJECTED, "Sglist destroyed")); + } } } @@ -755,7 +804,7 @@ void TVolumeActor::ForwardRequest( const ns::TEv##name##Response::TPtr& ev, \ const NActors::TActorContext& ctx) \ { \ - HandleResponse(ctx, ev); \ + ForwardResponse(ctx, ev); \ } \ // BLOCKSTORE_FORWARD_REQUEST diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_forward_trackused.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_forward_trackused.cpp index b2688cfeba5..ce9cee8eff9 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_forward_trackused.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_forward_trackused.cpp @@ -36,7 +36,6 @@ bool TVolumeActor::SendRequestToPartitionWithUsedBlockTracking( { auto requestInfo = CreateRequestInfo(ev->Sender, ev->Cookie, msg->CallContext); - // TODO(drbasic) // For encrypted disk-registry based disks, we will continue to // write a map of encrypted blocks for a while. diff --git a/cloud/blockstore/libs/storage/volume/volume_events_private.h b/cloud/blockstore/libs/storage/volume/volume_events_private.h index 3af2f1612a7..f54026eee45 100644 --- a/cloud/blockstore/libs/storage/volume/volume_events_private.h +++ b/cloud/blockstore/libs/storage/volume/volume_events_private.h @@ -206,7 +206,7 @@ struct TEvVolumePrivate }; // - // MultipartitionWriteOrZeroCompleted & WriteOrZeroCompleted + // WriteOrZeroCompleted // struct TWriteOrZeroCompleted @@ -320,7 +320,6 @@ struct TEvVolumePrivate EvRetryStartPartition, EvAcquireDiskIfNeeded, EvPartStatsSaved, - EvMultipartitionWriteOrZeroCompleted, EvWriteOrZeroCompleted, EvUpdateReadWriteClientInfo, EvRemoveExpiredVolumeParams, @@ -363,11 +362,6 @@ struct TEvVolumePrivate EvPartStatsSaved >; - using TEvMultipartitionWriteOrZeroCompleted = TRequestEvent< - TMultipartitionWriteOrZeroCompleted, - EvMultipartitionWriteOrZeroCompleted - >; - using TEvWriteOrZeroCompleted = TRequestEvent< TWriteOrZeroCompleted, EvWriteOrZeroCompleted diff --git a/cloud/blockstore/libs/storage/volume/volume_ut.cpp b/cloud/blockstore/libs/storage/volume/volume_ut.cpp index 4403037462b..3aa34335d92 100644 --- a/cloud/blockstore/libs/storage/volume/volume_ut.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_ut.cpp @@ -8944,6 +8944,102 @@ Y_UNIT_TEST_SUITE(TVolumeTest) UNIT_ASSERT_VALUES_EQUAL(1u, updateUsedBlocksRequestCount); } + + void DoShouldRejectRequestsWhenVolumeIsKilled( + bool multipartition, + bool trackUsed) + { + auto runtime = PrepareTestActorRuntime({}); + const ui32 partitionCount = multipartition ? 2 : 1; + + TVolumeClient volume(*runtime); + volume.UpdateVolumeConfig( + 0, + 0, + 0, + 0, + false, + 1, + NCloud::NProto::STORAGE_MEDIA_HYBRID, + 7 * 1024, // block count per partition + "vol0", + "cloud", + "folder", + partitionCount, // partition count + 2, // blocksPerStripe + trackUsed ? "track-used" : "" // tags + ); + volume.WaitReady(); + + auto clientInfo = CreateVolumeClientInfo( + NProto::VOLUME_ACCESS_READ_WRITE, + NProto::VOLUME_MOUNT_LOCAL, + 0); + + volume.AddClient(clientInfo); + + // Make the interceptor for WriteBlocks responses from the partition. + ui32 droppedResponseCount = 0; + auto dropPartitionResponses = + [&](TTestActorRuntimeBase&, TAutoPtr& event) + { + if (event->GetTypeRewrite() == TEvService::EvWriteBlocksResponse) { + ++droppedResponseCount; + return true; + } + + return false; + }; + auto oldFilter = runtime->SetEventFilter(dropPartitionResponses); + + // Send write request. + volume.SendWriteBlocksRequest( + TBlockRange64::WithLength(0, 1024), + clientInfo.GetClientId(), + 1); + + // Waiting for the interception of all responses from the partitions. + TDispatchOptions options; + options.CustomFinalCondition = [&]() + { + return droppedResponseCount == partitionCount; + }; + runtime->DispatchEvents(options, TDuration::Seconds(1)); + + // Remove the responses interceptor. + runtime->SetEventFilter(oldFilter); + + // Restarting the volume tablet. It must respond to requests that have + // not yet been completed. + volume.RebootTablet(); + + // Check response. + auto response = volume.RecvWriteBlocksResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL( + "Shutting down", + response->GetError().GetMessage()); + } + + Y_UNIT_TEST(ShouldRejectRequestsWhenSinglePartitionVolumesIsKilled) + { + DoShouldRejectRequestsWhenVolumeIsKilled(false, false); + } + + Y_UNIT_TEST(ShouldRejectRequestsWhenMultiPartitionVolumesIsKilled) + { + DoShouldRejectRequestsWhenVolumeIsKilled(true, false); + } + + Y_UNIT_TEST(ShouldRejectRequestsWhenTrackUsedAndSinglePartitionVolumesIsKilled) + { + DoShouldRejectRequestsWhenVolumeIsKilled(false, true); + } + + Y_UNIT_TEST(ShouldRejectRequestsWhenTrackUsedAndMultiPartitionVolumesIsKilled) + { + DoShouldRejectRequestsWhenVolumeIsKilled(true, true); + } } } // namespace NCloud::NBlockStore::NStorage