diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index e1157b1f432f8d9..ad54af5f61dcbd7 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -447,7 +447,7 @@ unsupportedRecoverStatement
| RECOVER TABLE name=multipartIdentifier
id=INTEGER_VALUE? (AS alias=identifier)? #recoverTable
| RECOVER PARTITION name=identifier id=INTEGER_VALUE? (AS alias=identifier)?
- FROM tableName=multipartIdentifier #recoverPartition
+ FROM tableName=multipartIdentifier (AS alias=identifier)? #recoverPartition
;
unsupportedAdminStatement
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 29a05856ff3b8e5..78ffeab84387327 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -3309,9 +3309,9 @@ recover_stmt ::=
{:
RESULT = new RecoverTableStmt(dbTblName, tableId, alias);
:}
- | KW_RECOVER KW_PARTITION ident:partitionName opt_id:partitionId opt_alias:alias KW_FROM table_name:dbTblName
+ | KW_RECOVER KW_PARTITION ident:partitionName opt_id:partitionId opt_alias:alias KW_FROM table_name:dbTblName opt_alias:newTableName
{:
- RESULT = new RecoverPartitionStmt(dbTblName, partitionName, partitionId, alias);
+ RESULT = new RecoverPartitionStmt(dbTblName, partitionName, partitionId, alias, newTableName);
:}
;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java
index def1b57f1b8ee47..73f9c5b6ed0dea0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java
@@ -33,14 +33,20 @@ public class RecoverPartitionStmt extends DdlStmt implements NotFallbackInParser
private String partitionName;
private long partitionId = -1;
private String newPartitionName = "";
+ private String newTableName = "";
- public RecoverPartitionStmt(TableName dbTblName, String partitionName, long partitionId, String newPartitionName) {
+
+ public RecoverPartitionStmt(TableName dbTblName, String partitionName, long partitionId, String newPartitionName,
+ String newTableName) {
this.dbTblName = dbTblName;
this.partitionName = partitionName;
this.partitionId = partitionId;
if (newPartitionName != null) {
this.newPartitionName = newPartitionName;
}
+ if (newTableName != null) {
+ this.newTableName = newTableName;
+ }
}
public String getDbName() {
@@ -63,6 +69,10 @@ public String getNewPartitionName() {
return newPartitionName;
}
+ public String getNewTableName() {
+ return newTableName;
+ }
+
@Override
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
dbTblName.analyze(analyzer);
@@ -95,7 +105,12 @@ public String toSql() {
sb.append(getDbName()).append(".");
}
sb.append(getTableName());
+ if (!Strings.isNullOrEmpty(newTableName)) {
+ sb.append(" AS ");
+ sb.append(this.newTableName);
+ }
return sb.toString();
+
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
index 745c1c8a3516864..8f9281dad522084 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
@@ -24,6 +24,7 @@
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.DebugUtil;
@@ -57,6 +58,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -787,8 +789,58 @@ private synchronized boolean innerRecoverTable(Database db, Table table, String
return true;
}
+ public synchronized void recoverPartitionUsingNewTable(String newTableName,
+ RecyclePartitionInfo recoverPartitionInfo, PartitionItem recoverItem,
+ OlapTable origTable, String newPartitionName, Database db) throws DdlException {
+ try {
+ Optional
newTable = db.getTable(newTableName);
+ if (!newTable.isPresent()) {
+ Env.getCurrentEnv().createTableUsingPartitionInfo(newTableName,
+ recoverPartitionInfo, origTable);
+ newTable = db.getTable(newTableName);
+ }
+ if (!newTable.isPresent()) {
+ // raise exeception
+ throw new DdlException("Unable to create " + newTableName);
+ }
+
+ OlapTable table = (OlapTable) newTable.get();
+ Partition recoverPartition = recoverPartitionInfo.getPartition();
+ // check if partition name exists
+ if (!Strings.isNullOrEmpty(newPartitionName)) {
+ if (table.checkPartitionNameExist(newPartitionName)) {
+ throw new DdlException("Partition name[" + newPartitionName + "] is already used");
+ }
+ recoverPartition.setName(newPartitionName);
+ }
+ // recover partition
+ table.addPartition(recoverPartition);
+
+ PartitionInfo partitionInfo = table.getPartitionInfo();
+ // recover partition info
+ long partitionId = recoverPartition.getId();
+ partitionInfo.setItem(partitionId, false, recoverItem);
+ partitionInfo.setDataProperty(partitionId, recoverPartitionInfo.getDataProperty());
+ partitionInfo.setReplicaAllocation(partitionId, recoverPartitionInfo.getReplicaAlloc());
+ partitionInfo.setIsInMemory(partitionId, recoverPartitionInfo.isInMemory());
+ partitionInfo.setIsMutable(partitionId, recoverPartitionInfo.isMutable());
+
+ // remove from recycle bin
+ idToPartition.remove(partitionId);
+ idToRecycleTime.remove(partitionId);
+
+ // log
+ long dbId = db.getId();
+ RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(), partitionId, "", "", newPartitionName);
+ Env.getCurrentEnv().getEditLog().logRecoverPartition(recoverInfo);
+ LOG.info("recover partition[{}]", partitionId);
+ } catch (UserException e) {
+ throw new DdlException(e.getMessage());
+ }
+ }
+
public synchronized void recoverPartition(long dbId, OlapTable table, String partitionName,
- long partitionIdToRecover, String newPartitionName) throws DdlException {
+ long partitionIdToRecover, String newPartitionName, String newTableName, Database db) throws DdlException {
if (table.getType() == TableType.MATERIALIZED_VIEW) {
throw new DdlException("Can not recover partition in materialized view: " + table.getName());
}
@@ -833,49 +885,54 @@ public synchronized void recoverPartition(long dbId, OlapTable table, String par
} else if (partitionInfo.getType() == PartitionType.LIST) {
recoverItem = recoverPartitionInfo.getListPartitionItem();
}
- // check if partition item is invalid
- if (partitionInfo.getAnyIntersectItem(recoverItem, false) != null) {
- throw new DdlException("Can not recover partition[" + partitionName + "]. Partition item conflict.");
- }
-
- // check if schema change
- Partition recoverPartition = recoverPartitionInfo.getPartition();
- Set tableIndex = table.getIndexIdToMeta().keySet();
- Set partitionIndex = recoverPartition.getMaterializedIndices(IndexExtState.ALL).stream()
- .map(i -> i.getId()).collect(Collectors.toSet());
- if (!tableIndex.equals(partitionIndex)) {
- throw new DdlException("table's index not equal with partition's index. table's index=" + tableIndex
- + ", partition's index=" + partitionIndex);
- }
+ if (newTableName != null) {
+ // use new table to recover
+ recoverPartitionUsingNewTable(newTableName, recoverPartitionInfo, recoverItem, table, newPartitionName, db);
+ } else {
+ // check if partition item is invalid
+ if (partitionInfo.getAnyIntersectItem(recoverItem, false) != null) {
+ throw new DdlException("Can not recover partition[" + partitionName + "]. Partition item conflict.");
+ }
+ // check if schema change
+ Partition recoverPartition = recoverPartitionInfo.getPartition();
+ Set tableIndex = table.getIndexIdToMeta().keySet();
+ Set partitionIndex = recoverPartition.getMaterializedIndices(IndexExtState.ALL).stream()
+ .map(i -> i.getId()).collect(Collectors.toSet());
+ if (!tableIndex.equals(partitionIndex)) {
+ throw new DdlException("table's index not equal with partition's index. table's index=" + tableIndex
+ + ", partition's index=" + partitionIndex);
+ }
- // check if partition name exists
- Preconditions.checkState(recoverPartition.getName().equalsIgnoreCase(partitionName));
- if (!Strings.isNullOrEmpty(newPartitionName)) {
- if (table.checkPartitionNameExist(newPartitionName)) {
- throw new DdlException("Partition name[" + newPartitionName + "] is already used");
+ // check if partition name exists
+ Preconditions.checkState(recoverPartition.getName().equalsIgnoreCase(partitionName));
+ if (!Strings.isNullOrEmpty(newPartitionName)) {
+ if (table.checkPartitionNameExist(newPartitionName)) {
+ throw new DdlException("Partition name[" + newPartitionName + "] is already used");
+ }
+ recoverPartition.setName(newPartitionName);
}
- recoverPartition.setName(newPartitionName);
- }
- // recover partition
- table.addPartition(recoverPartition);
+ // recover partition
+ table.addPartition(recoverPartition);
- // recover partition info
- long partitionId = recoverPartition.getId();
- partitionInfo.setItem(partitionId, false, recoverItem);
- partitionInfo.setDataProperty(partitionId, recoverPartitionInfo.getDataProperty());
- partitionInfo.setReplicaAllocation(partitionId, recoverPartitionInfo.getReplicaAlloc());
- partitionInfo.setIsInMemory(partitionId, recoverPartitionInfo.isInMemory());
- partitionInfo.setIsMutable(partitionId, recoverPartitionInfo.isMutable());
+ // recover partition info
+ long partitionId = recoverPartition.getId();
+ partitionInfo.setItem(partitionId, false, recoverItem);
+ partitionInfo.setDataProperty(partitionId, recoverPartitionInfo.getDataProperty());
+ partitionInfo.setReplicaAllocation(partitionId, recoverPartitionInfo.getReplicaAlloc());
+ partitionInfo.setIsInMemory(partitionId, recoverPartitionInfo.isInMemory());
+ partitionInfo.setIsMutable(partitionId, recoverPartitionInfo.isMutable());
- // remove from recycle bin
- idToPartition.remove(partitionId);
- idToRecycleTime.remove(partitionId);
+ // remove from recycle bin
+ idToPartition.remove(partitionId);
+ idToRecycleTime.remove(partitionId);
+
+ // log
+ RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(), partitionId, "", "", newPartitionName);
+ Env.getCurrentEnv().getEditLog().logRecoverPartition(recoverInfo);
+ LOG.info("recover partition[{}]", partitionId);
+ }
- // log
- RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(), partitionId, "", "", newPartitionName);
- Env.getCurrentEnv().getEditLog().logRecoverPartition(recoverInfo);
- LOG.info("recover partition[{}]", partitionId);
}
// The caller should keep table write lock
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 35bf31ad3fcdb3b..6c214f581b5b581 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -82,6 +82,7 @@
import org.apache.doris.binlog.BinlogGcer;
import org.apache.doris.binlog.BinlogManager;
import org.apache.doris.blockrule.SqlBlockRuleMgr;
+import org.apache.doris.catalog.CatalogRecycleBin.RecyclePartitionInfo;
import org.apache.doris.catalog.ColocateTableIndex.GroupId;
import org.apache.doris.catalog.DistributionInfo.DistributionInfoType;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
@@ -125,6 +126,7 @@
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.SmallFileMgr;
+import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.consistency.ConsistencyChecker;
@@ -6641,4 +6643,107 @@ private void replayJournalsAndExit() {
System.exit(0);
}
+
+ /**
+ * Get table ddl stmt from partition Info.
+ * Only olap table
+ *
+ */
+ public static void getDdlStmtFromPartitionInfo(TableIf table,
+ RecyclePartitionInfo recovRecyclePartitionInfo, String createTableStmt) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("CREATE ");
+
+ sb.append(table.getType() != TableType.MATERIALIZED_VIEW ? "TABLE " : "MATERIALIZED VIEW ");
+ sb.append("`").append(table.getName()).append("`");
+ sb.append(" (\n");
+ int idx = 0;
+ OlapTable olapTable = (OlapTable) table;
+
+ // this getting always latest column set.
+ // how to get old one?
+ long oldIndex = recovRecyclePartitionInfo.getPartition().getBaseIndex().getId();
+ List columns = olapTable.getSchemaByIndexId(oldIndex);
+ for (Column column : columns) {
+ if (idx++ != 0) {
+ sb.append(",\n");
+ }
+ // There MUST BE 2 space in front of each column description line
+ // sqlalchemy requires this to parse SHOW CREATE TABLE stmt.
+ if (table.isManagedTable()) {
+ sb.append(" ").append(
+ column.toSql(((OlapTable) table).getKeysType() == KeysType.UNIQUE_KEYS, true));
+ } else {
+ sb.append(" ").append(column.toSql());
+ }
+ }
+ if (table instanceof OlapTable) {
+ if (CollectionUtils.isNotEmpty(olapTable.getIndexes())) {
+ for (Index index : olapTable.getIndexes()) {
+ sb.append(",\n");
+ sb.append(" ").append(index.toSql());
+ }
+ }
+ }
+ sb.append("\n) ENGINE=");
+ sb.append(table.getType().name());
+
+ if (table instanceof OlapTable) {
+ // keys
+ String keySql = olapTable.getKeysType().toSql();
+ if (olapTable.isDuplicateWithoutKey()) {
+ // after #18621, use can create a DUP_KEYS olap table without key columns
+ // and get a ddl schema without key type and key columns
+ } else {
+ sb.append("\n").append(keySql).append("(");
+ List keysColumnNames = Lists.newArrayList();
+ Map clusterKeysColumnNamesToId = new TreeMap<>();
+ for (Column column : olapTable.getBaseSchema()) {
+ if (column.isKey()) {
+ keysColumnNames.add("`" + column.getName() + "`");
+ }
+ if (column.isClusterKey()) {
+ clusterKeysColumnNamesToId.put(column.getClusterKeyId(), column.getName());
+ }
+ }
+ sb.append(Joiner.on(", ").join(keysColumnNames)).append(")");
+ // show cluster keys
+ if (!clusterKeysColumnNamesToId.isEmpty()) {
+ sb.append("\n").append("CLUSTER BY (`");
+ sb.append(Joiner.on("`, `").join(clusterKeysColumnNamesToId.values())).append("`)");
+ }
+ }
+
+ // partition
+ PartitionInfo partitionInfo = olapTable.getPartitionInfo();
+ List partitionId = null;
+ if ((partitionInfo.getType() == PartitionType.RANGE
+ || partitionInfo.getType() == PartitionType.LIST)) {
+ sb.append("\n").append(partitionInfo.toSql(olapTable, partitionId));
+ }
+
+ // distribution
+ DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo();
+ sb.append("\n").append(distributionInfo.toSql());
+
+ // properties
+ sb.append("\nPROPERTIES (\n");
+ addOlapTablePropertyInfo(olapTable, sb, false, false, partitionId);
+ sb.append("\n)");
+ }
+ sb.append(";");
+ createTableStmt = sb.toString();
+ }
+
+ public void createTableUsingPartitionInfo(String newTableName,
+ RecyclePartitionInfo recovRecyclePartitionInfo, OlapTable origTable) throws UserException {
+ String createTableStmt = null;
+ Env.getDdlStmtFromPartitionInfo(origTable, recovRecyclePartitionInfo, createTableStmt);
+ LOG.info("create table stmt: {}", createTableStmt);
+ CreateTableStmt parsedCreateTableStmt = (CreateTableStmt) SqlParserUtils.parseAndAnalyzeStmt(
+ createTableStmt, ConnectContext.get());
+ parsedCreateTableStmt.setTableName(newTableName);
+ parsedCreateTableStmt.setIfNotExists(true);
+ createTable(parsedCreateTableStmt);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 16e0106b3185b1a..05dc757377af8ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -685,31 +685,46 @@ public void recoverTable(RecoverTableStmt recoverStmt) throws DdlException {
}
public void recoverPartition(RecoverPartitionStmt recoverStmt) throws DdlException {
+ boolean needCreateNewTable = false;
String dbName = recoverStmt.getDbName();
String tableName = recoverStmt.getTableName();
Database db = getDbOrDdlException(dbName);
OlapTable olapTable = db.getOlapTableOrDdlException(tableName);
- olapTable.writeLockOrDdlException();
+ String newTableName = recoverStmt.getNewTableName();
+ if (!Strings.isNullOrEmpty(newTableName)) {
+ needCreateNewTable = true;
+ }
try {
+ // if we are going to create a new table , then we need to take writelock
+ // on db level. else just need table level.
String partitionName = recoverStmt.getPartitionName();
String newPartitionName = recoverStmt.getNewPartitionName();
- if (Strings.isNullOrEmpty(newPartitionName)) {
- if (olapTable.getPartition(partitionName) != null) {
- throw new DdlException("partition[" + partitionName + "] "
- + "already exist in table[" + tableName + "]");
- }
+
+ if (needCreateNewTable) {
+ db.writeLockOrDdlException();
} else {
- if (olapTable.getPartition(newPartitionName) != null) {
- throw new DdlException("partition[" + newPartitionName + "] "
- + "already exist in table[" + tableName + "]");
+ olapTable.writeLockOrDdlException();
+ if (Strings.isNullOrEmpty(newPartitionName)) {
+ if (olapTable.getPartition(partitionName) != null) {
+ throw new DdlException("partition[" + partitionName + "] "
+ + "already exist in table[" + tableName + "]");
+ }
+ } else {
+ if (olapTable.getPartition(newPartitionName) != null) {
+ throw new DdlException("partition[" + newPartitionName + "] "
+ + "already exist in table[" + tableName + "]");
+ }
}
}
-
Env.getCurrentRecycleBin().recoverPartition(db.getId(), olapTable, partitionName,
- recoverStmt.getPartitionId(), newPartitionName);
+ recoverStmt.getPartitionId(), newPartitionName, newTableName, db);
} finally {
- olapTable.writeUnlock();
+ if (needCreateNewTable) {
+ db.writeUnlock();
+ } else {
+ olapTable.writeUnlock();
+ }
}
}
diff --git a/regression-test/data/catalog_recycle_bin_p0/recover_partition_with_newtable.out b/regression-test/data/catalog_recycle_bin_p0/recover_partition_with_newtable.out
new file mode 100644
index 000000000000000..7d3480549f142c8
--- /dev/null
+++ b/regression-test/data/catalog_recycle_bin_p0/recover_partition_with_newtable.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_check_1 --
+1 a 2022-01-02
+2 a 2023-01-02
+3 a 2024-01-02
+
diff --git a/regression-test/suites/catalog_recycle_bin_p0/recover_partition_with_newtable.groovy b/regression-test/suites/catalog_recycle_bin_p0/recover_partition_with_newtable.groovy
new file mode 100644
index 000000000000000..7bf79e5f4c671dc
--- /dev/null
+++ b/regression-test/suites/catalog_recycle_bin_p0/recover_partition_with_newtable.groovy
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("recover_partition_with_newtable") {
+ def dbName = "recover_partition_with_newtable_db"
+ sql "drop database if exists ${dbName} force"
+ sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+ sql "use ${dbName}"
+
+ def table = "test_table"
+ def table_new = "test_table_new"
+ // create table and insert data
+ sql """ drop table if exists ${table} """
+ sql """
+ create table ${table} (
+ `id` int(11),
+ `name` varchar(128),
+ `da` date
+ )
+ engine=olap
+ duplicate key(id)
+ partition by range(da)(
+ PARTITION p1 VALUES LESS THAN ('2023-01-01'),
+ PARTITION p2 VALUES LESS THAN ('2024-01-01'),
+ PARTITION p3 VALUES LESS THAN ('2025-01-01')
+ )
+ distributed by hash(id) buckets 2
+ properties(
+ "replication_num"="1",
+ "light_schema_change"="true"
+ );
+ """
+
+ sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
+ sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
+ sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
+
+ qt_select_check_1 """select * from ${table} order by id,name,da; """
+
+ // drop partition
+ sql """ ALTER TABLE ${table} DROP PARTITION p1; """
+ sql """ ALTER TABLE ${table} DROP PARTITION p2; """
+ sql """ ALTER TABLE ${table} DROP PARTITION p3; """
+
+
+ sql """ RECOVER PARTITION p1 from ${table} as ${table_new}; """
+ sql """ RECOVER PARTITION p2 from ${table} as ${table_new}; """
+ sql """ RECOVER PARTITION p3 from ${table} as ${table_new}; """
+
+ qt_select_check_1 """select * from ${table_new} order by id,name,da; """
+
+}