Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[bug](backup) Add table/partition id into convert_rowset_ids() #37077

Merged
merged 7 commits into from
Jul 14, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion be/src/olap/snapshot_manager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,8 @@ Status SnapshotManager::release_snapshot(const string& snapshot_path) {
}

Status SnapshotManager::convert_rowset_ids(const std::string& clone_dir, int64_t tablet_id,
int64_t replica_id, const int32_t& schema_hash) {
int64_t replica_id, int64_t table_id,
int64_t partition_id, const int32_t& schema_hash) {
SCOPED_CONSUME_MEM_TRACKER(_mem_tracker);
Status res = Status::OK();
// check clone dir existed
Expand Down Expand Up @@ -149,6 +150,12 @@ Status SnapshotManager::convert_rowset_ids(const std::string& clone_dir, int64_t
// equal to tablet id in meta
new_tablet_meta_pb.set_tablet_id(tablet_id);
new_tablet_meta_pb.set_replica_id(replica_id);
if (table_id > 0) {
new_tablet_meta_pb.set_table_id(table_id);
}
if (partition_id > 0) {
new_tablet_meta_pb.set_partition_id(partition_id);
}
new_tablet_meta_pb.set_schema_hash(schema_hash);
TabletSchemaSPtr tablet_schema;
tablet_schema =
Expand Down Expand Up @@ -177,6 +184,9 @@ Status SnapshotManager::convert_rowset_ids(const std::string& clone_dir, int64_t
}
// FIXME(cyx): Redundant?
rowset_meta->set_tablet_id(tablet_id);
if (partition_id > 0) {
rowset_meta->set_partition_id(partition_id);
}
rowset_meta->set_tablet_schema_hash(schema_hash);
Version rowset_version = {visible_rowset.start_version(), visible_rowset.end_version()};
rs_version_map[rowset_version] = rowset_meta;
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/snapshot_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ class SnapshotManager {
static SnapshotManager* instance();

Status convert_rowset_ids(const std::string& clone_dir, int64_t tablet_id, int64_t replica_id,
const int32_t& schema_hash);
int64_t table_id, int64_t partition_id, const int32_t& schema_hash);

private:
SnapshotManager() : _snapshot_base_id(0) {
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/task/engine_clone_task.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir,
// change all rowset ids because they maybe its id same with local rowset
status = SnapshotManager::instance()->convert_rowset_ids(
local_data_path, _clone_req.tablet_id, _clone_req.replica_id,
_clone_req.schema_hash);
_clone_req.table_id, _clone_req.partition_id, _clone_req.schema_hash);
} else {
LOG_WARNING("failed to download snapshot from remote BE")
.tag("url", remote_url_prefix)
Expand Down
5 changes: 3 additions & 2 deletions be/src/olap/task/engine_storage_migration_task.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,9 @@ Status EngineStorageMigrationTask::_gen_and_write_header_to_hdr_file(

// it will change rowset id and its create time
// rowset create time is useful when load tablet from meta to check which tablet is the tablet to load
return SnapshotManager::instance()->convert_rowset_ids(full_path, tablet_id,
_tablet->replica_id(), schema_hash);
return SnapshotManager::instance()->convert_rowset_ids(
full_path, tablet_id, _tablet->replica_id(), _tablet->table_id(),
_tablet->partition_id(), schema_hash);
}

Status EngineStorageMigrationTask::_reload_tablet(const std::string& full_path) {
Expand Down
3 changes: 2 additions & 1 deletion be/src/runtime/snapshot_loader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -406,7 +406,8 @@ Status SnapshotLoader::move(const std::string& snapshot_path, TabletSharedPtr ta

// rename the rowset ids and tabletid info in rowset meta
Status convert_status = SnapshotManager::instance()->convert_rowset_ids(
snapshot_path, tablet_id, tablet->replica_id(), schema_hash);
snapshot_path, tablet_id, tablet->replica_id(), tablet->table_id(),
tablet->partition_id(), schema_hash);
if (convert_status != Status::OK()) {
std::stringstream ss;
ss << "failed to convert rowsetids in snapshot: " << snapshot_path
Expand Down
2 changes: 2 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ public TCloneReq toThrift() {
request.setDestPathHash(destPathHash);
}
request.setTimeoutS(timeoutS);
request.setPartitionId(partitionId);
request.setTableId(tableId);

return request;
}
Expand Down
2 changes: 2 additions & 0 deletions gensrc/thrift/AgentService.thrift
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,8 @@ struct TCloneReq {
9: optional i64 dest_path_hash;
10: optional i32 timeout_s;
11: optional Types.TReplicaId replica_id = 0
12: optional i64 partition_id
13: optional i64 table_id = -1
}

struct TCompactionReq {
Expand Down
Loading