Skip to content

Commit

Permalink
Restore indexes from backup with the original partitioning (#7589)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jepett0
jepett0 committed Sep 26, 2024
1 parent e8fd738 commit 8470c00
Show file tree
Hide file tree
Showing 7 changed files with 181 additions and 64 deletions.
1 change: 1 addition & 0 deletions ydb/core/protos/flat_scheme_op.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1804,6 +1804,7 @@ message TDescribeOptions {
optional bool ReturnChannelsBinding = 8 [default = false];
optional bool ReturnRangeKey = 9 [default = true];
optional bool ReturnSetVal = 10 [default = false];
optional bool ReturnIndexTableBoundaries = 11 [default = false];
}

// Request to read scheme for a specific path
Expand Down
1 change: 1 addition & 0 deletions ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ static NKikimrSchemeOp::TPathDescription GetTableDescription(TSchemeShard* ss, c
opts.SetReturnPartitioningInfo(false);
opts.SetReturnPartitionConfig(true);
opts.SetReturnBoundaries(true);
opts.SetReturnIndexTableBoundaries(true);

auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId, opts);
auto record = desc->GetRecord();
Expand Down
141 changes: 81 additions & 60 deletions ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,78 @@ static void FillTableStats(NKikimrSchemeOp::TPathDescription& pathDescription, c
FillTableMetrics(pathDescription.MutableTabletMetrics(), stats);
}

static void FillColumns(
const TTableInfo& tableInfo,
google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TColumnDescription>& out
) {
bool familyNamesBuilt = false;
THashMap<ui32, TString> familyNames;

out.Reserve(tableInfo.Columns.size());
for (const auto& col : tableInfo.Columns) {
const auto& cinfo = col.second;
if (cinfo.IsDropped())
continue;

auto* colDescr = out.Add();
colDescr->SetName(cinfo.Name);
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
colDescr->SetTypeId(columnType.TypeId);
if (columnType.TypeInfo) {
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
}
colDescr->SetId(cinfo.Id);
colDescr->SetNotNull(cinfo.NotNull);

if (cinfo.Family != 0) {
colDescr->SetFamily(cinfo.Family);

if (!familyNamesBuilt) {
for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) {
if (family.HasName() && family.HasId()) {
familyNames[family.GetId()] = family.GetName();
}
}
familyNamesBuilt = true;
}

auto it = familyNames.find(cinfo.Family);
if (it != familyNames.end() && !it->second.empty()) {
colDescr->SetFamilyName(it->second);
}
}

colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);

switch (cinfo.DefaultKind) {
case ETableColumnDefaultKind::None:
break;
case ETableColumnDefaultKind::FromSequence:
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
break;
case ETableColumnDefaultKind::FromLiteral:
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
cinfo.DefaultValue));
break;
}
}
}

static void FillKeyColumns(
const TTableInfo& tableInfo,
google::protobuf::RepeatedPtrField<TProtoStringType>& names,
google::protobuf::RepeatedField<ui32>& ids
) {
Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty());
names.Reserve(tableInfo.KeyColumnIds.size());
ids.Reserve(tableInfo.KeyColumnIds.size());
for (ui32 keyColId : tableInfo.KeyColumnIds) {
*names.Add() = tableInfo.Columns.at(keyColId).Name;
*ids.Add() = keyColId;
}
}

void TPathDescriber::FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElement::TPtr pathEl, TPathElement::EPathSubType subType) {
FillChildDescr(descr, pathEl);

Expand Down Expand Up @@ -292,6 +364,7 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa
bool returnBoundaries = false;
bool returnRangeKey = true;
bool returnSetVal = Params.GetOptions().GetReturnSetVal();
bool returnIndexTableBoundaries = Params.GetOptions().GetReturnIndexTableBoundaries();
if (Params.HasOptions()) {
returnConfig = Params.GetOptions().GetReturnPartitionConfig();
returnPartitioning = Params.GetOptions().GetReturnPartitioningInfo();
Expand Down Expand Up @@ -416,7 +489,9 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa

switch (childPath->PathType) {
case NKikimrSchemeOp::EPathTypeTableIndex:
Self->DescribeTableIndex(childPathId, childName, returnConfig, false, *entry->AddTableIndexes());
Self->DescribeTableIndex(
childPathId, childName, returnConfig, returnIndexTableBoundaries, *entry->AddTableIndexes()
);
break;
case NKikimrSchemeOp::EPathTypeCdcStream:
Self->DescribeCdcStream(childPathId, childName, *entry->AddCdcStreams());
Expand Down Expand Up @@ -1174,67 +1249,10 @@ void TSchemeShard::DescribeTable(
) const
{
Y_UNUSED(typeRegistry);
THashMap<ui32, TString> familyNames;
bool familyNamesBuilt = false;

entry->SetTableSchemaVersion(tableInfo->AlterVersion);
entry->MutableColumns()->Reserve(tableInfo->Columns.size());
for (auto col : tableInfo->Columns) {
const auto& cinfo = col.second;
if (cinfo.IsDropped())
continue;

auto colDescr = entry->AddColumns();
colDescr->SetName(cinfo.Name);
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
colDescr->SetTypeId(columnType.TypeId);
if (columnType.TypeInfo) {
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
}
colDescr->SetId(cinfo.Id);
colDescr->SetNotNull(cinfo.NotNull);

if (cinfo.Family != 0) {
colDescr->SetFamily(cinfo.Family);

if (!familyNamesBuilt) {
for (const auto& family : tableInfo->PartitionConfig().GetColumnFamilies()) {
if (family.HasName() && family.HasId()) {
familyNames[family.GetId()] = family.GetName();
}
}
familyNamesBuilt = true;
}

auto it = familyNames.find(cinfo.Family);
if (it != familyNames.end() && !it->second.empty()) {
colDescr->SetFamilyName(it->second);
}
}

colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);

switch (cinfo.DefaultKind) {
case ETableColumnDefaultKind::None:
break;
case ETableColumnDefaultKind::FromSequence:
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
break;
case ETableColumnDefaultKind::FromLiteral:
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
cinfo.DefaultValue));
break;
}
}
Y_ABORT_UNLESS(!tableInfo->KeyColumnIds.empty());

entry->MutableKeyColumnNames()->Reserve(tableInfo->KeyColumnIds.size());
entry->MutableKeyColumnIds()->Reserve(tableInfo->KeyColumnIds.size());
for (ui32 keyColId : tableInfo->KeyColumnIds) {
entry->AddKeyColumnNames(tableInfo->Columns[keyColId].Name);
entry->AddKeyColumnIds(keyColId);
}
FillColumns(*tableInfo, *entry->MutableColumns());
FillKeyColumns(*tableInfo, *entry->MutableKeyColumnNames(), *entry->MutableKeyColumnIds());

if (fillConfig) {
FillPartitionConfig(tableInfo->PartitionConfig(), *entry->MutablePartitionConfig());
Expand Down Expand Up @@ -1298,6 +1316,9 @@ void TSchemeShard::DescribeTableIndex(const TPathId& pathId, const TString& name
FillPartitionConfig(tableInfo->PartitionConfig(), *tableDescription->MutablePartitionConfig());
}
if (fillBoundaries) {
// column info is necessary for split boundary type conversion
FillColumns(*tableInfo, *tableDescription->MutableColumns());
FillKeyColumns(*tableInfo, *tableDescription->MutableKeyColumnNames(), *tableDescription->MutableKeyColumnIds());
FillTableBoundaries(tableDescription->MutableSplitBoundary(), tableInfo);
}
}
Expand Down
4 changes: 2 additions & 2 deletions ydb/core/ydb_convert/table_description.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -839,8 +839,8 @@ void FillGlobalIndexSettings(Ydb::Table::GlobalIndexSettings& settings,
NKikimrMiniKQL::TType splitKeyType;
Ydb::Table::DescribeTableResult unused;
FillColumnDescription(unused, splitKeyType, indexImplTableDescription);
FillTableBoundaryImpl(
*settings.mutable_partition_at_keys(),
FillTableBoundaryImpl<Ydb::Table::GlobalIndexSettings>(
settings,
indexImplTableDescription,
splitKeyType
);
Expand Down
13 changes: 13 additions & 0 deletions ydb/public/sdk/cpp/client/ydb_table/table.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,10 @@ class TTableDescription::TImpl {
Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns));
}

void AddSecondaryIndex(const TIndexDescription& indexDescription) {
Indexes_.emplace_back(indexDescription);
}

void AddChangefeed(const TString& name, EChangefeedMode mode, EChangefeedFormat format) {
Changefeeds_.emplace_back(name, mode, format);
}
Expand Down Expand Up @@ -757,6 +761,10 @@ void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType t
Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
}

void TTableDescription::AddSecondaryIndex(const TIndexDescription& indexDescription) {
Impl_->AddSecondaryIndex(indexDescription);
}

void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) {
AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns);
}
Expand Down Expand Up @@ -1191,6 +1199,11 @@ TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const TString& primaryKeyColum
return *this;
}

TTableBuilder& TTableBuilder::AddSecondaryIndex(const TIndexDescription& indexDescription) {
TableDescription_.AddSecondaryIndex(indexDescription);
return *this;
}

TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) {
TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
return *this;
Expand Down
6 changes: 4 additions & 2 deletions ydb/public/sdk/cpp/client/ydb_table/table.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,10 +183,10 @@ struct TExplicitPartitions {
using TSelf = TExplicitPartitions;

FLUENT_SETTING_VECTOR(TValue, SplitPoints);

template <typename TProto>
static TExplicitPartitions FromProto(const TProto& proto);

void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const;
};

Expand Down Expand Up @@ -609,6 +609,7 @@ class TTableDescription {
// common
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
void AddSecondaryIndex(const TIndexDescription& indexDescription);
// sync
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns);
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
Expand Down Expand Up @@ -820,6 +821,7 @@ class TTableBuilder {
TTableBuilder& AddSerialColumn(const TString& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const TString& family = TString());

// common
TTableBuilder& AddSecondaryIndex(const TIndexDescription& indexDescription);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn);
Expand Down
79 changes: 79 additions & 0 deletions ydb/services/ydb/backup_ut/ydb_backup_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,68 @@ void TestTableSplitBoundariesArePreserved(
UNIT_ASSERT_EQUAL(restoredTableDescription.GetKeyRanges(), originalKeyRanges);
}

void TestIndexTableSplitBoundariesArePreserved(
const char* table, const char* index, ui64 indexPartitions, TSession& session,
TBackupFunction&& backup, TRestoreFunction&& restore
) {
const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable");

{
TExplicitPartitions indexPartitionBoundaries;
for (ui32 boundary : {1, 2, 4, 8, 16, 32, 64, 128, 256}) {
indexPartitionBoundaries.AppendSplitPoints(
// split boundary is technically always a tuple
TValueBuilder().BeginTuple().AddElement().OptionalUint32(boundary).EndTuple().Build()
);
}
// By default indexImplTable has auto partitioning by size enabled,
// so you must set min partition count for partitions to not merge immediately after indexImplTable is built.
TPartitioningSettingsBuilder partitioningSettingsBuilder;
partitioningSettingsBuilder
.SetMinPartitionsCount(indexPartitions)
.SetMaxPartitionsCount(indexPartitions);

const auto indexSettings = TGlobalIndexSettings{
.PartitioningSettings = partitioningSettingsBuilder.Build(),
.Partitions = std::move(indexPartitionBoundaries)
};

auto tableBuilder = TTableBuilder()
.AddNullableColumn("Key", EPrimitiveType::Uint32)
.AddNullableColumn("Value", EPrimitiveType::Uint32)
.SetPrimaryKeyColumn("Key")
.AddSecondaryIndex(TIndexDescription("byValue", EIndexType::GlobalSync, { "Value" }, {}, { indexSettings }));

const auto result = session.CreateTable(table, tableBuilder.Build()).ExtractValueSync();
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
}
const auto describeSettings = TDescribeTableSettings()
.WithTableStatistics(true)
.WithKeyShardBoundary(true);
const auto originalIndexTableDescription = GetTableDescription(
session, indexTablePath, describeSettings
);
UNIT_ASSERT_VALUES_EQUAL(originalIndexTableDescription.GetPartitionsCount(), indexPartitions);
const auto& originalKeyRanges = originalIndexTableDescription.GetKeyRanges();
UNIT_ASSERT_VALUES_EQUAL(originalKeyRanges.size(), indexPartitions);

backup(table);

ExecuteDataDefinitionQuery(session, Sprintf(R"(
DROP TABLE `%s`;
)", table
));

restore(table);
const auto restoredIndexTableDescription = GetTableDescription(
session, indexTablePath, describeSettings
);
UNIT_ASSERT_VALUES_EQUAL(restoredIndexTableDescription.GetPartitionsCount(), indexPartitions);
const auto& restoredKeyRanges = restoredIndexTableDescription.GetKeyRanges();
UNIT_ASSERT_VALUES_EQUAL(restoredKeyRanges.size(), indexPartitions);
UNIT_ASSERT_EQUAL(restoredKeyRanges, originalKeyRanges);
}

}

Y_UNIT_TEST_SUITE(BackupRestore) {
Expand Down Expand Up @@ -354,6 +416,7 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
);
}

// TO DO: test index impl table split boundaries restoration from a backup
}

Y_UNIT_TEST_SUITE(BackupRestoreS3) {
Expand Down Expand Up @@ -544,4 +607,20 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
);
}

Y_UNIT_TEST(RestoreIndexTableSplitBoundaries) {
TS3TestEnv testEnv;
constexpr const char* table = "/Root/table";
constexpr const char* index = "byValue";
constexpr ui64 indexPartitions = 10;

TestIndexTableSplitBoundariesArePreserved(
table,
index,
indexPartitions,
testEnv.GetSession(),
CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()),
CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port())
);
}

}

0 comments on commit 8470c00

Please sign in to comment.