From 916d7fd1fc898b2e1f5c8371bb3c36f46d26fbf8 Mon Sep 17 00:00:00 2001 From: walter Date: Mon, 23 Sep 2024 17:58:53 +0800 Subject: [PATCH] [fix](restore) Fix view signature (#41120) 1. reset with dbName instead of dbFullName, to be compatible with doris 2.0(full name has prefix `default_cluster:`) 2. since the referred db name of the restored view has changed, the next time to restore the view, the signature is not matched, so the db names should be reset and compared with the signature again. --- .../org/apache/doris/backup/RestoreJob.java | 21 ++- .../org/apache/doris/catalog/OlapTable.java | 2 +- .../java/org/apache/doris/catalog/View.java | 4 +- .../test_backup_restore_with_view.groovy | 137 ++++++++++++++++++ 4 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 regression-test/suites/backup_restore/test_backup_restore_with_view.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index cca381cc90d7871..738c249d583da1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -751,16 +751,23 @@ private void checkAndPrepareMeta() { if (localTbl != null) { Preconditions.checkState(localTbl.getType() == TableType.VIEW); View localView = (View) localTbl; - if (!localView.getSignature(BackupHandler.SIGNATURE_VERSION) - .equals(remoteView.getSignature(BackupHandler.SIGNATURE_VERSION))) { - status = new Status(ErrCode.COMMON_ERROR, "View " - + jobInfo.getAliasByOriginNameIfSet(backupViewName) - + " already exist but with different schema"); - return; + String localViewSignature = localView.getSignature(BackupHandler.SIGNATURE_VERSION); + // keep compatible with old version, compare the signature without reset view def + if (!localViewSignature.equals(remoteView.getSignature(BackupHandler.SIGNATURE_VERSION))) { + // reset view def to dest db name and compare signature again + String srcDbName = jobInfo.dbName; + remoteView.resetViewDefForRestore(srcDbName, db.getName()); + if (!localViewSignature.equals(remoteView.getSignature(BackupHandler.SIGNATURE_VERSION))) { + status = new Status(ErrCode.COMMON_ERROR, "View " + + jobInfo.getAliasByOriginNameIfSet(backupViewName) + + " already exist but with different schema"); + return; + } } } else { String srcDbName = jobInfo.dbName; - remoteView.resetIdsForRestore(env, srcDbName, db.getFullName()); + remoteView.resetViewDefForRestore(srcDbName, db.getName()); + remoteView.resetIdsForRestore(env); restoredTbls.add(remoteView); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 83759106e476b18..912286d63b407ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -591,7 +591,7 @@ public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation restore baseIndexId = newIdxId; } MaterializedIndexMeta indexMeta = origIdxIdToMeta.get(entry.getKey()); - indexMeta.resetIndexIdForRestore(newIdxId, srcDbName, db.getFullName()); + indexMeta.resetIndexIdForRestore(newIdxId, srcDbName, db.getName()); indexIdToMeta.put(newIdxId, indexMeta); indexNameToId.put(entry.getValue(), newIdxId); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java index c65297cd86463c9..870912906cd85c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java @@ -247,9 +247,11 @@ public View clone() { return copied; } - public void resetIdsForRestore(Env env, String srcDbName, String dbName) { + public void resetIdsForRestore(Env env) { id = env.getNextId(); + } + public void resetViewDefForRestore(String srcDbName, String dbName) { // the source db name is not setted in old BackupMeta, keep compatible with the old one. if (srcDbName != null) { inlineViewDef = inlineViewDef.replaceAll(srcDbName, dbName); diff --git a/regression-test/suites/backup_restore/test_backup_restore_with_view.groovy b/regression-test/suites/backup_restore/test_backup_restore_with_view.groovy new file mode 100644 index 000000000000000..10b21bb3442082e --- /dev/null +++ b/regression-test/suites/backup_restore/test_backup_restore_with_view.groovy @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_with_view", "backup_restore") { + String suiteName = "backup_restore_with_view" + String dbName = "${suiteName}_db" + String dbName1 = "${suiteName}_db_1" + String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "") + String snapshotName = "${suiteName}_snapshot" + String tableName = "${suiteName}_table" + String viewName = "${suiteName}_view" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "CREATE DATABASE IF NOT EXISTS ${dbName1}" + + int numRows = 10; + sql "DROP TABLE IF EXISTS ${dbName}.${tableName} FORCE" + sql "DROP VIEW IF EXISTS ${dbName}.${viewName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `count` LARGEINT SUM DEFAULT "0" + ) + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "replication_num" = "1" + ) + """ + List values = [] + for (int j = 1; j <= numRows; ++j) { + values.add("(${j}, ${j})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + + sql """CREATE VIEW ${dbName}.${viewName} (id, count) + AS + SELECT * FROM ${dbName}.${tableName} WHERE count > 5 + """ + + qt_sql "SELECT * FROM ${dbName}.${tableName} ORDER BY id ASC" + qt_sql "SELECT * FROM ${dbName}.${viewName} ORDER BY id ASC" + + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + """ + + syncer.waitSnapshotFinish(dbName) + + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + sql "DROP TABLE IF EXISTS ${dbName1}.${tableName} FORCE" + sql "DROP VIEW IF EXISTS ${dbName1}.${viewName}" + + sql """ + RESTORE SNAPSHOT ${dbName1}.${snapshotName} + FROM `${repoName}` + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true" + ) + """ + + syncer.waitAllRestoreFinish(dbName1) + + qt_sql "SELECT * FROM ${dbName1}.${tableName} ORDER BY id ASC" + qt_sql "SELECT * FROM ${dbName1}.${viewName} ORDER BY id ASC" + def show_view_result = sql_return_maparray "SHOW VIEW FROM ${tableName} FROM ${dbName1}" + logger.info("show view result: ${show_view_result}") + assertTrue(show_view_result.size() == 1); + def show_view = show_view_result[0]['Create View'] + assertTrue(show_view.contains("${dbName1}")) + assertTrue(show_view.contains("${tableName}")) + + // restore to db, test the view signature. + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + def restore_result = sql_return_maparray """ SHOW RESTORE FROM ${dbName} WHERE Label ="${snapshotName}" """ + restore_result.last() + logger.info("show restore result: ${restore_result}") + assertTrue(restore_result.last().State == "FINISHED") + + // restore to db1, test the view signature. + sql """ + RESTORE SNAPSHOT ${dbName1}.${snapshotName} + FROM `${repoName}` + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true" + ) + """ + + syncer.waitAllRestoreFinish(dbName1) + restore_result = sql_return_maparray """ SHOW RESTORE FROM ${dbName1} WHERE Label ="${snapshotName}" """ + restore_result.last() + logger.info("show restore result: ${restore_result}") + assertTrue(restore_result.last().State == "FINISHED") + + sql "DROP TABLE ${dbName}.${tableName} FORCE" + sql "DROP VIEW ${dbName}.${viewName}" + sql "DROP DATABASE ${dbName} FORCE" + sql "DROP TABLE ${dbName1}.${tableName} FORCE" + sql "DROP VIEW ${dbName1}.${viewName}" + sql "DROP DATABASE ${dbName1} FORCE" + sql "DROP REPOSITORY `${repoName}`" +} +