Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[fix](restore) Fix clean restore with view #40620 #41185

Merged
merged 1 commit into from
Sep 24, 2024
Merged

[fix](restore) Fix clean restore with view #40620 #41185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 17 additions & 11 deletions fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2003,22 +2003,28 @@ private Status allTabletCommitted(boolean isReplay) {
}

private Status dropAllNonRestoredTableAndPartitions(Database db) {
Set<String> restoredViews = jobInfo.newBackupObjects.views.stream()
.map(view -> view.name).collect(Collectors.toSet());

try {
for (Table table : db.getTables()) {
long tableId = table.getId();
String tableName = table.getName();
TableType tableType = table.getType();
BackupOlapTableInfo backupTableInfo = jobInfo.backupOlapTableObjects.get(tableName);
if (tableType != TableType.OLAP && tableType != TableType.ODBC && tableType != TableType.VIEW) {
continue;
}
if (tableType == TableType.OLAP && backupTableInfo != null) {
// drop the non restored partitions.
dropNonRestoredPartitions(db, (OlapTable) table, backupTableInfo);
} else if (isCleanTables) {
// otherwise drop the entire table.
LOG.info("drop non restored table {}({}). {}", tableName, tableId, this);
boolean isForceDrop = false; // move this table into recyclebin.
if (tableType == TableType.OLAP) {
BackupOlapTableInfo backupTableInfo = jobInfo.backupOlapTableObjects.get(tableName);
if (tableType == TableType.OLAP && backupTableInfo != null) {
// drop the non restored partitions.
dropNonRestoredPartitions(db, (OlapTable) table, backupTableInfo);
} else if (isCleanTables) {
// otherwise drop the entire table.
LOG.info("drop non restored table {}, table id: {}. {}", tableName, tableId, this);
boolean isForceDrop = false; // move this table into recyclebin.
env.getInternalCatalog().dropTableWithoutCheck(db, table, isForceDrop);
}
} else if (tableType == TableType.VIEW && isCleanTables && !restoredViews.contains(tableName)) {
LOG.info("drop non restored view {}, table id: {}. {}", tableName, tableId, this);
boolean isForceDrop = false; // move this view into recyclebin.
env.getInternalCatalog().dropTableWithoutCheck(db, table, isForceDrop);
}
}
Expand Down
36 changes: 34 additions & 2 deletions regression-test/suites/backup_restore/test_backup_restore_clean_restore.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ suite("test_backup_restore_clean_restore", "backup_restore") {
)
"""


sql "INSERT INTO ${dbName}.${tableName2} VALUES ${values.join(",")}"
result = sql "SELECT * FROM ${dbName}.${tableName2}"
assertEquals(result.size(), numRows);
Expand Down Expand Up @@ -106,6 +107,25 @@ suite("test_backup_restore_clean_restore", "backup_restore") {
result = sql "SELECT * FROM ${dbName}.${tableName3}"
assertEquals(result.size(), numRows);

// view 1 must exists
String viewName1 = "${tableNamePrefix}_4"
sql "DROP VIEW IF EXISTS ${dbName}.${viewName1}"
sql """
CREATE VIEW ${dbName}.${viewName1} (k1, k2)
AS
SELECT id as k1, count as k2 FROM ${dbName}.${tableName1}
WHERE id in (1,3,5,7,9)
"""

// view 2 will be deleted
String viewName2 = "${tableNamePrefix}_5"
sql "DROP VIEW IF EXISTS ${dbName}.${viewName2}"
sql """
CREATE VIEW ${dbName}.${viewName2} (k1, k2)
AS
SELECT id as k1, count as k2 FROM ${dbName}.${tableName3}
WHERE id in (1,3,5,7,9)
"""

sql """
BACKUP SNAPSHOT ${dbName}.${snapshotName}
Expand All @@ -119,13 +139,14 @@ suite("test_backup_restore_clean_restore", "backup_restore") {
def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
assertTrue(snapshot != null)

// restore table1, partition 3 of table2
// restore table1, partition 3 of table2, view1
sql """
RESTORE SNAPSHOT ${dbName}.${snapshotName}
FROM `${repoName}`
ON (
`${tableName1}`,
`${tableName2}` PARTITION (`p3`)
`${tableName2}` PARTITION (`p3`),
`${viewName1}`
)
PROPERTIES
(
Expand All @@ -148,12 +169,23 @@ suite("test_backup_restore_clean_restore", "backup_restore") {
result = sql "SELECT * FROM ${dbName}.${tableName2}"
assertEquals(result.size(), numRows-10)

// view1 are exists
result = sql """ SHOW VIEW FROM ${tableName1} FROM ${dbName} """
assertEquals(result.size(), 1)

// view2 are dropped
result = sql """
SHOW TABLE STATUS FROM ${dbName} LIKE "${viewName2}"
"""
assertEquals(result.size(), 0)

// table3 are dropped
result = sql """
SHOW TABLE STATUS FROM ${dbName} LIKE "${tableName3}"
"""
assertEquals(result.size(), 0)

sql "DROP VIEW ${dbName}.${viewName1}"
sql "DROP TABLE ${dbName}.${tableName1} FORCE"
sql "DROP TABLE ${dbName}.${tableName2} FORCE"
sql "DROP DATABASE ${dbName} FORCE"
Expand Down
Loading