Skip to content

Commit

Permalink
cherry pick (#492)
Browse files Browse the repository at this point in the history 
* NBS-4683 tests for multiple block size (#456)

* NBS-4683 tests for multiple block size

* Fix review issues

* NBS-4905 simplify SendRequestToPartitionWithUsedBlockTracking (#490)
  • Loading branch information
@drbasic
drbasic authored Feb 20, 2024
1 parent febbb2f commit f78ce3d
Show file tree
Hide file tree
Showing 5 changed files with 136 additions and 92 deletions.
18 changes: 4 additions & 14 deletions cloud/blockstore/libs/storage/disk_registry/disk_registry_state.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,20 +269,10 @@ TString TCheckpointInfo::MakeCheckpointDiskId(
ui64 TDiskInfo::GetBlocksCount() const
{
ui64 result = 0;
if (MasterDiskId) {
for (const auto& replica: Replicas) {
for (const auto& device: replica) {
result += device.GetBlocksCount();
}
break;
}
} else {
for (const auto& device: Devices) {
const ui64 logicalBlockCount = device.GetBlockSize() *
device.GetBlocksCount() /
LogicalBlockSize;
result += logicalBlockCount;
}
for (const auto& device: Devices) {
const ui64 logicalBlockCount =
device.GetBlockSize() * device.GetBlocksCount() / LogicalBlockSize;
result += logicalBlockCount;
}
return result;
}
Expand Down
55 changes: 35 additions & 20 deletions cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut_checkpoint.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ namespace {

////////////////////////////////////////////////////////////////////////////////

constexpr ui32 AvailableBlockSizes[] =
{4_KB, 8_KB, 16_KB, 32_KB, 64_KB, 128_KB};

TDiskRegistryState MakeDiskRegistryState()
{
auto agentConfig1 = AgentConfig(
Expand Down Expand Up @@ -62,7 +65,8 @@ Y_UNIT_TEST_SUITE(TDiskRegistryStateCheckpointTest)
using TCreateDiskFunc = std::function<
void(TDiskRegistryState & state, TDiskRegistryDatabase db)>;

void DoShouldCreateCheckpointForMirrorDisk(TCreateDiskFunc createDisk)
void DoShouldCreateCheckpointForDiskRegistryBasedDisk(
TCreateDiskFunc createDisk)
{
TTestExecutor executor;
executor.WriteTx([&](TDiskRegistryDatabase db) { db.InitSchema(); });
Expand Down Expand Up @@ -148,25 +152,32 @@ Y_UNIT_TEST_SUITE(TDiskRegistryStateCheckpointTest)

Y_UNIT_TEST(ShouldCreateCheckpointForDiskRegistryBasedDisk)
{
DoShouldCreateCheckpointForMirrorDisk(
[&](TDiskRegistryState& state, TDiskRegistryDatabase db)
{
TVector<TDeviceConfig> devices;
auto error = AllocateDisk(
db,
state,
"disk-1",
"", // placementGroupId
0, // placementPartitionIndex
30_GB,
devices);
UNIT_ASSERT_SUCCESS(error);
});
for (ui32 blockSize: AvailableBlockSizes) {
DoShouldCreateCheckpointForDiskRegistryBasedDisk(
[&](TDiskRegistryState& state, TDiskRegistryDatabase db)
{
TVector<TDeviceConfig> devices;
auto error = AllocateDisk(
db,
state,
"disk-1",
"", // placementGroupId
0, // placementPartitionIndex
30_GB,
devices,
TInstant::Seconds(100),
NProto::STORAGE_MEDIA_SSD_NONREPLICATED,
TString(), // poolName
blockSize);
UNIT_ASSERT_SUCCESS(error);
});
}
}

Y_UNIT_TEST(ShouldCreateCheckpointForMirrorDisk)
Y_UNIT_TEST(ShouldCreateCheckpointForMirrorDisk8192)
{
DoShouldCreateCheckpointForMirrorDisk(
for (ui32 blockSize: AvailableBlockSizes) {
DoShouldCreateCheckpointForDiskRegistryBasedDisk(
[&](TDiskRegistryState& state, TDiskRegistryDatabase db)
{
TVector<TDeviceConfig> devices;
Expand All @@ -182,12 +193,16 @@ Y_UNIT_TEST_SUITE(TDiskRegistryStateCheckpointTest)
devices,
replicas,
migrations,
deviceReplacementIds);
deviceReplacementIds,
TInstant::Seconds(100),
NProto::STORAGE_MEDIA_SSD_MIRROR2,
blockSize);
UNIT_ASSERT_SUCCESS(error);
});
}
}

Y_UNIT_TEST(ShouldNotCreatWhenNotEnoughFreeSpace)
Y_UNIT_TEST(ShouldNotCreateWhenNotEnoughFreeSpace)
{
TTestExecutor executor;
executor.WriteTx([&](TDiskRegistryDatabase db) { db.InitSchema(); });
Expand Down Expand Up @@ -233,7 +248,7 @@ Y_UNIT_TEST_SUITE(TDiskRegistryStateCheckpointTest)
});
}

Y_UNIT_TEST(ShouldCreatMultipleCheckpoints)
Y_UNIT_TEST(ShouldCreateMultipleCheckpoints)
{
TTestExecutor executor;
executor.WriteTx([&](TDiskRegistryDatabase db) { db.InitSchema(); });
Expand Down
14 changes: 8 additions & 6 deletions cloud/blockstore/libs/storage/disk_registry/testlib/test_state.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,8 @@ NProto::TError AllocateMirroredDisk(
TVector<NProto::TDeviceMigration>& migrations,
TVector<TString>& deviceReplacementIds,
TInstant now,
NProto::EStorageMediaKind mediaKind)
NProto::EStorageMediaKind mediaKind,
ui32 logicalBlockSize)
{
TDiskRegistryState::TAllocateDiskResult result {};

Expand All @@ -233,8 +234,8 @@ NProto::TError AllocateMirroredDisk(
.DiskId = diskId,
.CloudId = "cloud-1",
.FolderId = "folder-1",
.BlockSize = DefaultLogicalBlockSize,
.BlocksCount = totalSize / DefaultLogicalBlockSize,
.BlockSize = logicalBlockSize,
.BlocksCount = totalSize / logicalBlockSize,
.ReplicaCount = replicaCount,
.MediaKind = mediaKind
},
Expand All @@ -261,7 +262,8 @@ NProto::TError AllocateDisk(
TVector<TDeviceConfig>& devices,
TInstant now,
NProto::EStorageMediaKind mediaKind,
TString poolName)
TString poolName,
ui32 logicalBlockSize)
{
TDiskRegistryState::TAllocateDiskResult result {};

Expand All @@ -274,8 +276,8 @@ NProto::TError AllocateDisk(
.FolderId = "folder-1",
.PlacementGroupId = placementGroupId,
.PlacementPartitionIndex = placementPartitionIndex,
.BlockSize = DefaultLogicalBlockSize,
.BlocksCount = totalSize / DefaultLogicalBlockSize,
.BlockSize = logicalBlockSize,
.BlocksCount = totalSize / logicalBlockSize,
.ReplicaCount = 0,
.PoolName = poolName,
.MediaKind = mediaKind,
Expand Down
6 changes: 4 additions & 2 deletions cloud/blockstore/libs/storage/disk_registry/testlib/test_state.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,8 @@ NProto::TError AllocateMirroredDisk(
TVector<NProto::TDeviceMigration>& migrations,
TVector<TString>& deviceReplacementIds,
TInstant now = TInstant::Seconds(100),
NProto::EStorageMediaKind mediaKind = NProto::STORAGE_MEDIA_SSD_MIRROR2);
NProto::EStorageMediaKind mediaKind = NProto::STORAGE_MEDIA_SSD_MIRROR2,
ui32 logicalBlockSize = DefaultLogicalBlockSize);

NProto::TError AllocateDisk(
TDiskRegistryDatabase& db,
Expand All @@ -168,7 +169,8 @@ NProto::TError AllocateDisk(
TInstant now = TInstant::Seconds(100),
NProto::EStorageMediaKind mediaKind =
NProto::STORAGE_MEDIA_SSD_NONREPLICATED,
TString poolName = "");
TString poolName = "",
ui32 logicalBlockSize = DefaultLogicalBlockSize);

NProto::TError AllocateCheckpoint(
TInstant now,
Expand Down
135 changes: 85 additions & 50 deletions cloud/blockstore/libs/storage/volume/volume_actor_forward_trackused.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ bool TVolumeActor::SendRequestToPartitionWithUsedBlockTracking(
const TActorId& partActorId,
const ui64 volumeRequestId)
{
static_assert(IsReadMethod<TMethod> || IsWriteMethod<TMethod>);

const auto* msg = ev->Get();

const auto& volumeConfig = State->GetMeta().GetVolumeConfig();
Expand Down Expand Up @@ -73,7 +75,11 @@ bool TVolumeActor::SendRequestToPartitionWithUsedBlockTracking(
== msg->Record.GetBlocksCount()
: false;

if (overlayDiskRegistryBasedDisk && !isOnlyOverlayDisk) {
if (isOnlyOverlayDisk) {
return false;
}

if (overlayDiskRegistryBasedDisk) {
NCloud::Register<TReadDiskRegistryBasedOverlayActor<TMethod>>(
ctx,
CreateRequestInfo(ev->Sender, ev->Cookie, msg->CallContext),
Expand All @@ -94,53 +100,65 @@ bool TVolumeActor::SendRequestToPartitionWithUsedBlockTracking(
return true;
}

if (!isOnlyOverlayDisk) {
NCloud::Register<TReadMarkedActor<TMethod>>(
ctx,
CreateRequestInfo(ev->Sender, ev->Cookie, msg->CallContext),
std::move(msg->Record),
State->GetUsedBlocks(),
State->GetMaskUnusedBlocks(),
encryptedDiskRegistryBasedDisk,
partActorId,
TabletID(),
SelfId());
NCloud::Register<TReadMarkedActor<TMethod>>(
ctx,
CreateRequestInfo(ev->Sender, ev->Cookie, msg->CallContext),
std::move(msg->Record),
State->GetUsedBlocks(),
State->GetMaskUnusedBlocks(),
encryptedDiskRegistryBasedDisk,
partActorId,
TabletID(),
SelfId());

return true;
}
return true;
}
}
return false;
}

if constexpr (std::is_same_v<TMethod, TEvService::TGetChangedBlocksMethod>) {
const auto highCheckpointType = State->GetCheckpointStore().GetCheckpointType(
template <>
bool TVolumeActor::SendRequestToPartitionWithUsedBlockTracking<
TEvService::TGetChangedBlocksMethod>(
const TActorContext& ctx,
const TEvService::TGetChangedBlocksMethod::TRequest::TPtr& ev,
const TActorId& partActorId,
const ui64 volumeRequestId)
{
Y_UNUSED(partActorId);
Y_UNUSED(volumeRequestId);

const auto* msg = ev->Get();

const auto highCheckpointType =
State->GetCheckpointStore().GetCheckpointType(
msg->Record.GetHighCheckpointId());
const auto lowCheckpointType = State->GetCheckpointStore().GetCheckpointType(
const auto lowCheckpointType =
State->GetCheckpointStore().GetCheckpointType(
msg->Record.GetLowCheckpointId());

if (highCheckpointType && *highCheckpointType == ECheckpointType::Light
|| lowCheckpointType && *lowCheckpointType == ECheckpointType::Light)
{
GetChangedBlocksForLightCheckpoints(ev, ctx);
return true;
}
if (highCheckpointType && *highCheckpointType == ECheckpointType::Light ||
lowCheckpointType && *lowCheckpointType == ECheckpointType::Light)
{
GetChangedBlocksForLightCheckpoints(ev, ctx);
return true;
}

// TODO: NBS-3228: remove checks for disk registry based disks after normal checkpoints
// for disk registry based disks are implemented completely.
if (IsDiskRegistryMediaKind(State->GetConfig().GetStorageMediaKind())
&& msg->Record.GetHighCheckpointId() == ""
&& msg->Record.GetLowCheckpointId() == "")
{
GetChangedBlocksForLightCheckpoints(ev, ctx);
return true;
}
// TODO: NBS-3228: remove checks for disk registry based disks after normal
// checkpoints for disk registry based disks are implemented completely.
if (!IsDiskRegistryMediaKind(State->GetConfig().GetStorageMediaKind())) {
return false;
}

if (IsDiskRegistryMediaKind(State->GetConfig().GetStorageMediaKind())) {
ReplyErrorOnNormalGetChangedBlocksRequestForDiskRegistryBasedDisk(ev, ctx);
return true;
}
if (msg->Record.GetHighCheckpointId() == "" &&
msg->Record.GetLowCheckpointId() == "")
{
GetChangedBlocksForLightCheckpoints(ev, ctx);
return true;
}

return false;
ReplyErrorOnNormalGetChangedBlocksRequestForDiskRegistryBasedDisk(ev, ctx);
return true;
}

////////////////////////////////////////////////////////////////////////////////
Expand All @@ -154,25 +172,42 @@ template bool TVolumeActor::SendRequestToPartitionWithUsedBlockTracking< \
const ui64 volumeRequestId); \
// GENERATE_IMPL

#define GENERATE_NO_IMPL(name, ns) \
template <> \
bool TVolumeActor::SendRequestToPartitionWithUsedBlockTracking< \
ns::T##name##Method>( \
const TActorContext& ctx, \
const ns::T##name##Method::TRequest::TPtr& ev, \
const TActorId& partActorId, \
const ui64 volumeRequestId) \
{ \
Y_UNUSED(ctx); \
Y_UNUSED(ev); \
Y_UNUSED(partActorId); \
Y_UNUSED(volumeRequestId); \
return false; \
} \
// GENERATE_NO_IMPL

GENERATE_IMPL(ReadBlocks, TEvService)
GENERATE_IMPL(WriteBlocks, TEvService)
GENERATE_IMPL(ZeroBlocks, TEvService)
GENERATE_IMPL(CreateCheckpoint, TEvService)
GENERATE_IMPL(DeleteCheckpoint, TEvService)
GENERATE_IMPL(GetChangedBlocks, TEvService)
GENERATE_IMPL(ReadBlocksLocal, TEvService)
GENERATE_IMPL(WriteBlocksLocal, TEvService)

GENERATE_IMPL(DescribeBlocks, TEvVolume)
GENERATE_IMPL(GetUsedBlocks, TEvVolume)
GENERATE_IMPL(GetPartitionInfo, TEvVolume)
GENERATE_IMPL(CompactRange, TEvVolume)
GENERATE_IMPL(GetCompactionStatus, TEvVolume)
GENERATE_IMPL(DeleteCheckpointData, TEvVolume)
GENERATE_IMPL(RebuildMetadata, TEvVolume)
GENERATE_IMPL(GetRebuildMetadataStatus, TEvVolume)
GENERATE_IMPL(ScanDisk, TEvVolume)
GENERATE_IMPL(GetScanDiskStatus, TEvVolume)
GENERATE_NO_IMPL(CreateCheckpoint, TEvService)
GENERATE_NO_IMPL(DeleteCheckpoint, TEvService)

GENERATE_NO_IMPL(DescribeBlocks, TEvVolume)
GENERATE_NO_IMPL(GetUsedBlocks, TEvVolume)
GENERATE_NO_IMPL(GetPartitionInfo, TEvVolume)
GENERATE_NO_IMPL(CompactRange, TEvVolume)
GENERATE_NO_IMPL(GetCompactionStatus, TEvVolume)
GENERATE_NO_IMPL(DeleteCheckpointData, TEvVolume)
GENERATE_NO_IMPL(RebuildMetadata, TEvVolume)
GENERATE_NO_IMPL(GetRebuildMetadataStatus, TEvVolume)
GENERATE_NO_IMPL(ScanDisk, TEvVolume)
GENERATE_NO_IMPL(GetScanDiskStatus, TEvVolume)

#undef GENERATE_IMPL

Expand Down

0 comments on commit f78ce3d

Please sign in to comment.