[fix](cases) Add check status timeout for backup/restore cases

apache · Mar 28, 2024 · 11eda71 · 11eda71
1 parent fce88d7
commit 11eda71
Show file tree

Hide file tree

Showing 25 changed files with 91 additions and 198 deletions.
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
@@ -365,6 +365,17 @@ class Syncer {
         true
     }
 
+    void waitSnapshotFinish(String dbName = null) {
+        int count = 0;
+        while (!checkSnapshotFinish(dbName)) {
+            if (++count >= 600) {  // 30min
+                logger.error('BACKUP task is timeouted')
+                throw new Exception("BACKUP task is timeouted after 30mins")
+            }
+            Thread.sleep(3000)
+        }
+    }
+
     String getSnapshotTimestamp(String repoName, String snapshotName) {
         def filterShowSnapshot = { records, name ->
             for (row in records) {
@@ -403,6 +414,17 @@ class Syncer {
         true
     }
 
+    void waitAllRestoreFinish(String dbName = null) {
+        int count = 0;
+        while (!checkAllRestoreFinish(dbName)) {
+            if (++count >= 600) {  // 30min
+                logger.error('RESTORE task is timeouted')
+                throw new Exception("RESTORE task is timeouted after 30mins")
+            }
+            Thread.sleep(3000)
+        }
+    }
+
     Boolean checkRestoreFinish() {
         String checkSQL = "SHOW RESTORE FROM TEST_" + context.db
         List<Object> row = suite.sql(checkSQL)[0]

diff --git a/regression-test/suites/backup_restore/test_MoW_backup_restore.groovy b/regression-test/suites/backup_restore/test_MoW_backup_restore.groovy
@@ -35,63 +35,47 @@ suite("test_MoW_backup_restore", "p1") {
     // version1 (1,1)(2,2)
     sql """insert into ${tableName} values(1,1),(2,2)"""
     sql """backup snapshot ${context.dbName}.snapshot1 to ${repo} on (${tableName}) properties("type"="full")"""
-    while(checkSnapshotFinish()==false){
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish()
     qt_3 """select * from ${tableName} order by user_id"""
 
     // version2 (1,10)(2,2)
     sql """insert into ${tableName} values(1,10)"""
     sql """backup snapshot ${context.dbName}.snapshot2 to ${repo} on (${tableName}) properties("type"="full")"""
-    while(checkSnapshotFinish()==false){
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish()
     qt_4 """select * from ${tableName} order by user_id"""
 
     // version3 (1,100)(2,2)
     sql """update ${tableName} set value = 100 where user_id = 1"""
     sql """backup snapshot ${context.dbName}.snapshot3 to ${repo} on (${tableName}) properties("type"="full")"""
-    while(checkSnapshotFinish()==false){
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish()
     qt_5 """select * from ${tableName} order by user_id"""
 
     // version4 (2,2)
     sql """delete from ${tableName} where user_id = 1"""
     sql """backup snapshot ${context.dbName}.snapshot4 to ${repo} on (${tableName}) properties("type"="full")"""
-    while(checkSnapshotFinish()==false){
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish()
     qt_6 """select * from ${tableName} order by user_id"""
 
     // version1 (1,1)(2,2)
     assertTrue(syncer.getSnapshot("snapshot1", "${tableName}"))
     assertTrue(syncer.restoreSnapshot())
-    while (checkRestoreFinish() == false) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish()
     qt_7 """select * from ${tableName} order by user_id"""
 
     // version2 (1,10)(2,2)
     assertTrue(syncer.getSnapshot("snapshot2", "${tableName}"))
     assertTrue(syncer.restoreSnapshot())
-    while (checkRestoreFinish() == false) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish()
     qt_8 """select * from ${tableName} order by user_id"""
     // version3 (1,100)(2,2)
     assertTrue(syncer.getSnapshot("snapshot3", "${tableName}"))
     assertTrue(syncer.restoreSnapshot())
-    while (checkRestoreFinish() == false) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish()
     qt_9 """select * from ${tableName} order by user_id"""
     // version4 (2,2)
     assertTrue(syncer.getSnapshot("snapshot4", "${tableName}"))
     assertTrue(syncer.restoreSnapshot())
-    while (checkRestoreFinish() == false) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish()
     qt_10 """select * from ${tableName} order by user_id"""
 
     sql """drop table if exists ${tableName}"""
@@ -107,30 +91,22 @@ suite("test_MoW_backup_restore", "p1") {
     // version1 (1,1)(2,2)
     assertTrue(syncer.getSnapshot("snapshot1", "${tableName}"))
     assertTrue(syncer.restoreSnapshot())
-    while (checkRestoreFinish() == false) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish()
     qt_11 """select * from ${tableName} order by user_id"""
 
     // version2 (1,10)(2,2)
     assertTrue(syncer.getSnapshot("snapshot2", "${tableName}"))
     assertTrue(syncer.restoreSnapshot())
-    while (checkRestoreFinish() == false) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish()
     qt_12 """select * from ${tableName} order by user_id"""
     // version3 (1,100)(2,2)
     assertTrue(syncer.getSnapshot("snapshot3", "${tableName}"))
     assertTrue(syncer.restoreSnapshot())
-    while (checkRestoreFinish() == false) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish()
     qt_13 """select * from ${tableName} order by user_id"""
     // version4 (2,2)
     assertTrue(syncer.getSnapshot("snapshot4", "${tableName}"))
     assertTrue(syncer.restoreSnapshot())
-    while (checkRestoreFinish() == false) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish()
     qt_14 """select * from ${tableName} order by user_id"""
 }
diff --git a/regression-test/suites/backup_restore/test_backup_restore.groovy b/regression-test/suites/backup_restore/test_backup_restore.groovy
@@ -52,9 +52,7 @@ suite("test_backup_restore", "backup_restore") {
         ON (${tableName})
     """
 
-    while (!syncer.checkSnapshotFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish(dbName)
 
     def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
     assertTrue(snapshot != null)
@@ -72,9 +70,7 @@ suite("test_backup_restore", "backup_restore") {
         )
     """
 
-    while (!syncer.checkAllRestoreFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish(dbName)
 
     result = sql "SELECT * FROM ${dbName}.${tableName}"
     assertEquals(result.size(), values.size());

diff --git a/regression-test/suites/backup_restore/test_backup_restore_alias.groovy b/regression-test/suites/backup_restore/test_backup_restore_alias.groovy
@@ -53,9 +53,7 @@ suite("test_backup_restore_alias", "backup_restore") {
         ON (${tableName})
     """
 
-    while (!syncer.checkSnapshotFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish(dbName)
 
     def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
     assertTrue(snapshot != null)
@@ -74,9 +72,7 @@ suite("test_backup_restore_alias", "backup_restore") {
         )
     """
 
-    while (!syncer.checkAllRestoreFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish(dbName)
 
     qt_select "SELECT * FROM ${dbName}.${tableName} ORDER BY id"
     qt_select "SELECT * FROM ${dbName}.${aliasName} ORDER BY id"

diff --git a/regression-test/suites/backup_restore/test_backup_restore_bloom_filter.groovy b/regression-test/suites/backup_restore/test_backup_restore_bloom_filter.groovy
@@ -54,9 +54,7 @@ suite("test_backup_restore_bloom_filter", "backup_restore") {
         PROPERTIES ("type" = "full")
         """
 
-    while (!syncer.checkSnapshotFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish(dbName)
 
     def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
 
@@ -74,9 +72,7 @@ suite("test_backup_restore_bloom_filter", "backup_restore") {
         )
         """
 
-    while (!syncer.checkAllRestoreFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish(dbName)
 
     def restore_index_comment = sql "SHOW CREATE TABLE ${dbName}.${tableName}"
 

diff --git a/regression-test/suites/backup_restore/test_backup_restore_db.groovy b/regression-test/suites/backup_restore/test_backup_restore_db.groovy
@@ -59,9 +59,7 @@ suite("test_backup_restore_db", "backup_restore") {
         TO `${repoName}`
     """
 
-    while (!syncer.checkSnapshotFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish(dbName)
 
     def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
     assertTrue(snapshot != null)
@@ -80,9 +78,7 @@ suite("test_backup_restore_db", "backup_restore") {
         )
     """
 
-    while (!syncer.checkAllRestoreFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish(dbName)
 
     for (def tableName in tables) {
         result = sql "SELECT * FROM ${dbName}.${tableName}"

diff --git a/regression-test/suites/backup_restore/test_backup_restore_diff_repo_same_snapshot.groovy b/regression-test/suites/backup_restore/test_backup_restore_diff_repo_same_snapshot.groovy
@@ -70,12 +70,8 @@ suite("test_backup_restore_diff_repo_same_snapshot", "backup_restore") {
         ON (${tableName}_2)
     """
 
-    while (!syncer.checkSnapshotFinish("${dbName}_1")) {
-        Thread.sleep(3000)
-    }
-    while (!syncer.checkSnapshotFinish("${dbName}_2")) {
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish("${dbName}_1")
+    syncer.waitSnapshotFinish("${dbName}_2")
 
     // Restore snapshot from repo_1 to db_1
     def snapshot = syncer.getSnapshotTimestamp("${repoName}_1", snapshotName)
@@ -93,9 +89,7 @@ suite("test_backup_restore_diff_repo_same_snapshot", "backup_restore") {
         )
     """
 
-    while (!syncer.checkAllRestoreFinish("${dbName}_1")) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish("${dbName}_1")
 
     result = sql "SELECT * FROM ${dbName}_1.${tableName}_1"
     assertEquals(result.size(), values.size());
@@ -116,9 +110,7 @@ suite("test_backup_restore_diff_repo_same_snapshot", "backup_restore") {
         )
     """
 
-    while (!syncer.checkAllRestoreFinish("${dbName}_2")) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish("${dbName}_2")
 
     result = sql "SELECT * FROM ${dbName}_2.${tableName}_2"
     assertEquals(result.size(), values.size());

diff --git a/regression-test/suites/backup_restore/test_backup_restore_dup_without_default_keys.groovy b/regression-test/suites/backup_restore/test_backup_restore_dup_without_default_keys.groovy
@@ -54,9 +54,7 @@ suite("test_backup_restore_dup_without_default_keys", "backup_restore") {
         ON (${tableName})
     """    
 
-    while (!syncer.checkSnapshotFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish(dbName)
     def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
     assertTrue(snapshot != null)
 
@@ -72,9 +70,7 @@ suite("test_backup_restore_dup_without_default_keys", "backup_restore") {
             "reserve_replica" = "true"
         )
     """
-    while (!syncer.checkAllRestoreFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish(dbName)
     result = sql "SELECT * FROM ${dbName}.${tableName}"
     assertEquals(result.size(), values.size());
 

diff --git a/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy b/regression-test/suites/backup_restore/test_backup_restore_exclude.groovy
@@ -61,9 +61,7 @@ suite("test_backup_restore_exclude", "backup_restore") {
         EXCLUDE (${backupExcludeTable})
     """
 
-    while (!syncer.checkSnapshotFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish(dbName)
 
     def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
     assertTrue(snapshot != null)
@@ -86,9 +84,7 @@ suite("test_backup_restore_exclude", "backup_restore") {
         )
     """
 
-    while (!syncer.checkAllRestoreFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish(dbName)
 
     qt_select "SELECT * FROM ${dbName}.${backupExcludeTable} ORDER BY id"
     for (def tableName in tables) {

diff --git a/regression-test/suites/backup_restore/test_backup_restore_inverted_index.groovy b/regression-test/suites/backup_restore/test_backup_restore_inverted_index.groovy
@@ -52,9 +52,7 @@ suite("test_backup_restore_inverted_index", "backup_restore") {
         PROPERTIES ("type" = "full")
         """
 
-    while (!syncer.checkSnapshotFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish(dbName)
 
     def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
 
@@ -72,9 +70,7 @@ suite("test_backup_restore_inverted_index", "backup_restore") {
         )
         """
 
-    while (!syncer.checkAllRestoreFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish(dbName)
 
     def restore_index_comment = sql "SHOW CREATE TABLE ${dbName}.${tableName}"
 

diff --git a/regression-test/suites/backup_restore/test_backup_restore_keep_on_local.groovy b/regression-test/suites/backup_restore/test_backup_restore_keep_on_local.groovy
@@ -51,9 +51,7 @@ suite("test_backup_restore_keep_on_local", "backup_restore") {
         ON (${tableName})
     """
 
-    while (!syncer.checkSnapshotFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish(dbName)
 
     sql "TRUNCATE TABLE ${dbName}.${tableName}"
 

diff --git a/regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy b/regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy
@@ -61,9 +61,7 @@ suite("test_backup_restore_multi_tables", "backup_restore") {
         ON (${backupTables.join(",")})
     """
 
-    while (!syncer.checkSnapshotFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish(dbName)
 
     def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
     assertTrue(snapshot != null)
@@ -83,9 +81,7 @@ suite("test_backup_restore_multi_tables", "backup_restore") {
         )
     """
 
-    while (!syncer.checkAllRestoreFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish(dbName)
 
     for (def tableName in tables) {
         result = sql "SELECT * FROM ${dbName}.${tableName}"

diff --git a/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy b/regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy
@@ -61,9 +61,7 @@ suite("test_backup_restore_multi_tables_overwrite", "backup_restore") {
         ON (${backupTables.join(",")})
     """
 
-    while (!syncer.checkSnapshotFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitSnapshotFinish(dbName)
 
     def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
     assertTrue(snapshot != null)
@@ -84,9 +82,7 @@ suite("test_backup_restore_multi_tables_overwrite", "backup_restore") {
         )
     """
 
-    while (!syncer.checkAllRestoreFinish(dbName)) {
-        Thread.sleep(3000)
-    }
+    syncer.waitAllRestoreFinish(dbName)
 
     qt_select "SELECT * FROM ${dbName}.${firstTableName} ORDER BY id"
     for (def tableName in tables) {