Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Restore indexes from backup with the original partitioning #7589 #9815

Open
wants to merge 1 commit into
base: stable-24-3
Choose a base branch
from
+181 −64
Open

Restore indexes from backup with the original partitioning #7589 #9815

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ydb/core/protos/flat_scheme_op.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1805,6 +1805,7 @@ message TDescribeOptions {
optional bool ReturnChannelsBinding = 8 [default = false];
optional bool ReturnRangeKey = 9 [default = true];
optional bool ReturnSetVal = 10 [default = false];
optional bool ReturnIndexTableBoundaries = 11 [default = false];
}

// Request to read scheme for a specific path
Expand Down
1 change: 1 addition & 0 deletions ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ static NKikimrSchemeOp::TPathDescription GetTableDescription(TSchemeShard* ss, c
opts.SetReturnPartitioningInfo(false);
opts.SetReturnPartitionConfig(true);
opts.SetReturnBoundaries(true);
opts.SetReturnIndexTableBoundaries(true);

auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId, opts);
auto record = desc->GetRecord();
Expand Down
141 changes: 81 additions & 60 deletions ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,78 @@ static void FillTableStats(NKikimrSchemeOp::TPathDescription& pathDescription, c
FillTableMetrics(pathDescription.MutableTabletMetrics(), stats);
}

static void FillColumns(
const TTableInfo& tableInfo,
google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TColumnDescription>& out
) {
bool familyNamesBuilt = false;
THashMap<ui32, TString> familyNames;

out.Reserve(tableInfo.Columns.size());
for (const auto& col : tableInfo.Columns) {
const auto& cinfo = col.second;
if (cinfo.IsDropped())
continue;

auto* colDescr = out.Add();
colDescr->SetName(cinfo.Name);
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
colDescr->SetTypeId(columnType.TypeId);
if (columnType.TypeInfo) {
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
}
colDescr->SetId(cinfo.Id);
colDescr->SetNotNull(cinfo.NotNull);

if (cinfo.Family != 0) {
colDescr->SetFamily(cinfo.Family);

if (!familyNamesBuilt) {
for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) {
if (family.HasName() && family.HasId()) {
familyNames[family.GetId()] = family.GetName();
}
}
familyNamesBuilt = true;
}

auto it = familyNames.find(cinfo.Family);
if (it != familyNames.end() && !it->second.empty()) {
colDescr->SetFamilyName(it->second);
}
}

colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);

switch (cinfo.DefaultKind) {
case ETableColumnDefaultKind::None:
break;
case ETableColumnDefaultKind::FromSequence:
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
break;
case ETableColumnDefaultKind::FromLiteral:
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
cinfo.DefaultValue));
break;
}
}
}

static void FillKeyColumns(
const TTableInfo& tableInfo,
google::protobuf::RepeatedPtrField<TProtoStringType>& names,
google::protobuf::RepeatedField<ui32>& ids
) {
Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty());
names.Reserve(tableInfo.KeyColumnIds.size());
ids.Reserve(tableInfo.KeyColumnIds.size());
for (ui32 keyColId : tableInfo.KeyColumnIds) {
*names.Add() = tableInfo.Columns.at(keyColId).Name;
*ids.Add() = keyColId;
}
}

void TPathDescriber::FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElement::TPtr pathEl, TPathElement::EPathSubType subType) {
FillChildDescr(descr, pathEl);

Expand Down Expand Up @@ -292,6 +364,7 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa
bool returnBoundaries = false;
bool returnRangeKey = true;
bool returnSetVal = Params.GetOptions().GetReturnSetVal();
bool returnIndexTableBoundaries = Params.GetOptions().GetReturnIndexTableBoundaries();
if (Params.HasOptions()) {
returnConfig = Params.GetOptions().GetReturnPartitionConfig();
returnPartitioning = Params.GetOptions().GetReturnPartitioningInfo();
Expand Down Expand Up @@ -416,7 +489,9 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa

switch (childPath->PathType) {
case NKikimrSchemeOp::EPathTypeTableIndex:
Self->DescribeTableIndex(childPathId, childName, returnConfig, false, *entry->AddTableIndexes());
Self->DescribeTableIndex(
childPathId, childName, returnConfig, returnIndexTableBoundaries, *entry->AddTableIndexes()
);
break;
case NKikimrSchemeOp::EPathTypeCdcStream:
Self->DescribeCdcStream(childPathId, childName, *entry->AddCdcStreams());
Expand Down Expand Up @@ -1175,67 +1250,10 @@ void TSchemeShard::DescribeTable(
) const
{
Y_UNUSED(typeRegistry);
THashMap<ui32, TString> familyNames;
bool familyNamesBuilt = false;

entry->SetTableSchemaVersion(tableInfo->AlterVersion);
entry->MutableColumns()->Reserve(tableInfo->Columns.size());
for (auto col : tableInfo->Columns) {
const auto& cinfo = col.second;
if (cinfo.IsDropped())
continue;

auto colDescr = entry->AddColumns();
colDescr->SetName(cinfo.Name);
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
colDescr->SetTypeId(columnType.TypeId);
if (columnType.TypeInfo) {
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
}
colDescr->SetId(cinfo.Id);
colDescr->SetNotNull(cinfo.NotNull);

if (cinfo.Family != 0) {
colDescr->SetFamily(cinfo.Family);

if (!familyNamesBuilt) {
for (const auto& family : tableInfo->PartitionConfig().GetColumnFamilies()) {
if (family.HasName() && family.HasId()) {
familyNames[family.GetId()] = family.GetName();
}
}
familyNamesBuilt = true;
}

auto it = familyNames.find(cinfo.Family);
if (it != familyNames.end() && !it->second.empty()) {
colDescr->SetFamilyName(it->second);
}
}

colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);

switch (cinfo.DefaultKind) {
case ETableColumnDefaultKind::None:
break;
case ETableColumnDefaultKind::FromSequence:
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
break;
case ETableColumnDefaultKind::FromLiteral:
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
cinfo.DefaultValue));
break;
}
}
Y_ABORT_UNLESS(!tableInfo->KeyColumnIds.empty());

entry->MutableKeyColumnNames()->Reserve(tableInfo->KeyColumnIds.size());
entry->MutableKeyColumnIds()->Reserve(tableInfo->KeyColumnIds.size());
for (ui32 keyColId : tableInfo->KeyColumnIds) {
entry->AddKeyColumnNames(tableInfo->Columns[keyColId].Name);
entry->AddKeyColumnIds(keyColId);
}
FillColumns(*tableInfo, *entry->MutableColumns());
FillKeyColumns(*tableInfo, *entry->MutableKeyColumnNames(), *entry->MutableKeyColumnIds());

if (fillConfig) {
FillPartitionConfig(tableInfo->PartitionConfig(), *entry->MutablePartitionConfig());
Expand Down Expand Up @@ -1299,6 +1317,9 @@ void TSchemeShard::DescribeTableIndex(const TPathId& pathId, const TString& name
FillPartitionConfig(tableInfo->PartitionConfig(), *tableDescription->MutablePartitionConfig());
}
if (fillBoundaries) {
// column info is necessary for split boundary type conversion
FillColumns(*tableInfo, *tableDescription->MutableColumns());
FillKeyColumns(*tableInfo, *tableDescription->MutableKeyColumnNames(), *tableDescription->MutableKeyColumnIds());
FillTableBoundaries(tableDescription->MutableSplitBoundary(), tableInfo);
}
}
Expand Down
4 changes: 2 additions & 2 deletions ydb/core/ydb_convert/table_description.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -839,8 +839,8 @@ void FillGlobalIndexSettings(Ydb::Table::GlobalIndexSettings& settings,
NKikimrMiniKQL::TType splitKeyType;
Ydb::Table::DescribeTableResult unused;
FillColumnDescription(unused, splitKeyType, indexImplTableDescription);
FillTableBoundaryImpl(
*settings.mutable_partition_at_keys(),
FillTableBoundaryImpl<Ydb::Table::GlobalIndexSettings>(
settings,
indexImplTableDescription,
splitKeyType
);
Expand Down
13 changes: 13 additions & 0 deletions ydb/public/sdk/cpp/client/ydb_table/table.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,10 @@ class TTableDescription::TImpl {
Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns));
}

void AddSecondaryIndex(const TIndexDescription& indexDescription) {
Indexes_.emplace_back(indexDescription);
}

void AddChangefeed(const TString& name, EChangefeedMode mode, EChangefeedFormat format) {
Changefeeds_.emplace_back(name, mode, format);
}
Expand Down Expand Up @@ -757,6 +761,10 @@ void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType t
Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
}

void TTableDescription::AddSecondaryIndex(const TIndexDescription& indexDescription) {
Impl_->AddSecondaryIndex(indexDescription);
}

void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) {
AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns);
}
Expand Down Expand Up @@ -1191,6 +1199,11 @@ TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const TString& primaryKeyColum
return *this;
}

TTableBuilder& TTableBuilder::AddSecondaryIndex(const TIndexDescription& indexDescription) {
TableDescription_.AddSecondaryIndex(indexDescription);
return *this;
}

TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) {
TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
return *this;
Expand Down
6 changes: 4 additions & 2 deletions ydb/public/sdk/cpp/client/ydb_table/table.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,10 +183,10 @@ struct TExplicitPartitions {
using TSelf = TExplicitPartitions;

FLUENT_SETTING_VECTOR(TValue, SplitPoints);

template <typename TProto>
static TExplicitPartitions FromProto(const TProto& proto);

void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const;
};

Expand Down Expand Up @@ -609,6 +609,7 @@ class TTableDescription {
// common
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
void AddSecondaryIndex(const TIndexDescription& indexDescription);
// sync
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns);
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
Expand Down Expand Up @@ -820,6 +821,7 @@ class TTableBuilder {
TTableBuilder& AddSerialColumn(const TString& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const TString& family = TString());

// common
TTableBuilder& AddSecondaryIndex(const TIndexDescription& indexDescription);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn);
Expand Down
79 changes: 79 additions & 0 deletions ydb/services/ydb/backup_ut/ydb_backup_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,68 @@ void TestTableSplitBoundariesArePreserved(
UNIT_ASSERT_EQUAL(restoredTableDescription.GetKeyRanges(), originalKeyRanges);
}

void TestIndexTableSplitBoundariesArePreserved(
const char* table, const char* index, ui64 indexPartitions, TSession& session,
TBackupFunction&& backup, TRestoreFunction&& restore
) {
const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable");

{
TExplicitPartitions indexPartitionBoundaries;
for (ui32 boundary : {1, 2, 4, 8, 16, 32, 64, 128, 256}) {
indexPartitionBoundaries.AppendSplitPoints(
// split boundary is technically always a tuple
TValueBuilder().BeginTuple().AddElement().OptionalUint32(boundary).EndTuple().Build()
);
}
// By default indexImplTable has auto partitioning by size enabled,
// so you must set min partition count for partitions to not merge immediately after indexImplTable is built.
TPartitioningSettingsBuilder partitioningSettingsBuilder;
partitioningSettingsBuilder
.SetMinPartitionsCount(indexPartitions)
.SetMaxPartitionsCount(indexPartitions);

const auto indexSettings = TGlobalIndexSettings{
.PartitioningSettings = partitioningSettingsBuilder.Build(),
.Partitions = std::move(indexPartitionBoundaries)
};

auto tableBuilder = TTableBuilder()
.AddNullableColumn("Key", EPrimitiveType::Uint32)
.AddNullableColumn("Value", EPrimitiveType::Uint32)
.SetPrimaryKeyColumn("Key")
.AddSecondaryIndex(TIndexDescription("byValue", EIndexType::GlobalSync, { "Value" }, {}, { indexSettings }));

const auto result = session.CreateTable(table, tableBuilder.Build()).ExtractValueSync();
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
}
const auto describeSettings = TDescribeTableSettings()
.WithTableStatistics(true)
.WithKeyShardBoundary(true);
const auto originalIndexTableDescription = GetTableDescription(
session, indexTablePath, describeSettings
);
UNIT_ASSERT_VALUES_EQUAL(originalIndexTableDescription.GetPartitionsCount(), indexPartitions);
const auto& originalKeyRanges = originalIndexTableDescription.GetKeyRanges();
UNIT_ASSERT_VALUES_EQUAL(originalKeyRanges.size(), indexPartitions);

backup(table);

ExecuteDataDefinitionQuery(session, Sprintf(R"(
DROP TABLE `%s`;
)", table
));

restore(table);
const auto restoredIndexTableDescription = GetTableDescription(
session, indexTablePath, describeSettings
);
UNIT_ASSERT_VALUES_EQUAL(restoredIndexTableDescription.GetPartitionsCount(), indexPartitions);
const auto& restoredKeyRanges = restoredIndexTableDescription.GetKeyRanges();
UNIT_ASSERT_VALUES_EQUAL(restoredKeyRanges.size(), indexPartitions);
UNIT_ASSERT_EQUAL(restoredKeyRanges, originalKeyRanges);
}

}

Y_UNIT_TEST_SUITE(BackupRestore) {
Expand Down Expand Up @@ -354,6 +416,7 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
);
}

// TO DO: test index impl table split boundaries restoration from a backup
}

Y_UNIT_TEST_SUITE(BackupRestoreS3) {
Expand Down Expand Up @@ -544,4 +607,20 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
);
}

Y_UNIT_TEST(RestoreIndexTableSplitBoundaries) {
TS3TestEnv testEnv;
constexpr const char* table = "/Root/table";
constexpr const char* index = "byValue";
constexpr ui64 indexPartitions = 10;

TestIndexTableSplitBoundariesArePreserved(
table,
index,
indexPartitions,
testEnv.GetSession(),
CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()),
CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port())
);
}

}
Loading