diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index e1157b1f432f8d9..ad54af5f61dcbd7 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -447,7 +447,7 @@ unsupportedRecoverStatement | RECOVER TABLE name=multipartIdentifier id=INTEGER_VALUE? (AS alias=identifier)? #recoverTable | RECOVER PARTITION name=identifier id=INTEGER_VALUE? (AS alias=identifier)? - FROM tableName=multipartIdentifier #recoverPartition + FROM tableName=multipartIdentifier (AS alias=identifier)? #recoverPartition ; unsupportedAdminStatement diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 29a05856ff3b8e5..78ffeab84387327 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -3309,9 +3309,9 @@ recover_stmt ::= {: RESULT = new RecoverTableStmt(dbTblName, tableId, alias); :} - | KW_RECOVER KW_PARTITION ident:partitionName opt_id:partitionId opt_alias:alias KW_FROM table_name:dbTblName + | KW_RECOVER KW_PARTITION ident:partitionName opt_id:partitionId opt_alias:alias KW_FROM table_name:dbTblName opt_alias:newTableName {: - RESULT = new RecoverPartitionStmt(dbTblName, partitionName, partitionId, alias); + RESULT = new RecoverPartitionStmt(dbTblName, partitionName, partitionId, alias, newTableName); :} ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java index def1b57f1b8ee47..73f9c5b6ed0dea0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java @@ -33,14 +33,20 @@ public class RecoverPartitionStmt extends DdlStmt implements NotFallbackInParser private String partitionName; private long partitionId = -1; private String newPartitionName = ""; + private String newTableName = ""; - public RecoverPartitionStmt(TableName dbTblName, String partitionName, long partitionId, String newPartitionName) { + + public RecoverPartitionStmt(TableName dbTblName, String partitionName, long partitionId, String newPartitionName, + String newTableName) { this.dbTblName = dbTblName; this.partitionName = partitionName; this.partitionId = partitionId; if (newPartitionName != null) { this.newPartitionName = newPartitionName; } + if (newTableName != null) { + this.newTableName = newTableName; + } } public String getDbName() { @@ -63,6 +69,10 @@ public String getNewPartitionName() { return newPartitionName; } + public String getNewTableName() { + return newTableName; + } + @Override public void analyze(Analyzer analyzer) throws AnalysisException, UserException { dbTblName.analyze(analyzer); @@ -95,7 +105,12 @@ public String toSql() { sb.append(getDbName()).append("."); } sb.append(getTableName()); + if (!Strings.isNullOrEmpty(newTableName)) { + sb.append(" AS "); + sb.append(this.newTableName); + } return sb.toString(); + } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index 745c1c8a3516864..8f9281dad522084 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -24,6 +24,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.DebugUtil; @@ -57,6 +58,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -787,8 +789,58 @@ private synchronized boolean innerRecoverTable(Database db, Table table, String return true; } + public synchronized void recoverPartitionUsingNewTable(String newTableName, + RecyclePartitionInfo recoverPartitionInfo, PartitionItem recoverItem, + OlapTable origTable, String newPartitionName, Database db) throws DdlException { + try { + Optional newTable = db.getTable(newTableName); + if (!newTable.isPresent()) { + Env.getCurrentEnv().createTableUsingPartitionInfo(newTableName, + recoverPartitionInfo, origTable); + newTable = db.getTable(newTableName); + } + if (!newTable.isPresent()) { + // raise exeception + throw new DdlException("Unable to create " + newTableName); + } + + OlapTable table = (OlapTable) newTable.get(); + Partition recoverPartition = recoverPartitionInfo.getPartition(); + // check if partition name exists + if (!Strings.isNullOrEmpty(newPartitionName)) { + if (table.checkPartitionNameExist(newPartitionName)) { + throw new DdlException("Partition name[" + newPartitionName + "] is already used"); + } + recoverPartition.setName(newPartitionName); + } + // recover partition + table.addPartition(recoverPartition); + + PartitionInfo partitionInfo = table.getPartitionInfo(); + // recover partition info + long partitionId = recoverPartition.getId(); + partitionInfo.setItem(partitionId, false, recoverItem); + partitionInfo.setDataProperty(partitionId, recoverPartitionInfo.getDataProperty()); + partitionInfo.setReplicaAllocation(partitionId, recoverPartitionInfo.getReplicaAlloc()); + partitionInfo.setIsInMemory(partitionId, recoverPartitionInfo.isInMemory()); + partitionInfo.setIsMutable(partitionId, recoverPartitionInfo.isMutable()); + + // remove from recycle bin + idToPartition.remove(partitionId); + idToRecycleTime.remove(partitionId); + + // log + long dbId = db.getId(); + RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(), partitionId, "", "", newPartitionName); + Env.getCurrentEnv().getEditLog().logRecoverPartition(recoverInfo); + LOG.info("recover partition[{}]", partitionId); + } catch (UserException e) { + throw new DdlException(e.getMessage()); + } + } + public synchronized void recoverPartition(long dbId, OlapTable table, String partitionName, - long partitionIdToRecover, String newPartitionName) throws DdlException { + long partitionIdToRecover, String newPartitionName, String newTableName, Database db) throws DdlException { if (table.getType() == TableType.MATERIALIZED_VIEW) { throw new DdlException("Can not recover partition in materialized view: " + table.getName()); } @@ -833,49 +885,54 @@ public synchronized void recoverPartition(long dbId, OlapTable table, String par } else if (partitionInfo.getType() == PartitionType.LIST) { recoverItem = recoverPartitionInfo.getListPartitionItem(); } - // check if partition item is invalid - if (partitionInfo.getAnyIntersectItem(recoverItem, false) != null) { - throw new DdlException("Can not recover partition[" + partitionName + "]. Partition item conflict."); - } - - // check if schema change - Partition recoverPartition = recoverPartitionInfo.getPartition(); - Set tableIndex = table.getIndexIdToMeta().keySet(); - Set partitionIndex = recoverPartition.getMaterializedIndices(IndexExtState.ALL).stream() - .map(i -> i.getId()).collect(Collectors.toSet()); - if (!tableIndex.equals(partitionIndex)) { - throw new DdlException("table's index not equal with partition's index. table's index=" + tableIndex - + ", partition's index=" + partitionIndex); - } + if (newTableName != null) { + // use new table to recover + recoverPartitionUsingNewTable(newTableName, recoverPartitionInfo, recoverItem, table, newPartitionName, db); + } else { + // check if partition item is invalid + if (partitionInfo.getAnyIntersectItem(recoverItem, false) != null) { + throw new DdlException("Can not recover partition[" + partitionName + "]. Partition item conflict."); + } + // check if schema change + Partition recoverPartition = recoverPartitionInfo.getPartition(); + Set tableIndex = table.getIndexIdToMeta().keySet(); + Set partitionIndex = recoverPartition.getMaterializedIndices(IndexExtState.ALL).stream() + .map(i -> i.getId()).collect(Collectors.toSet()); + if (!tableIndex.equals(partitionIndex)) { + throw new DdlException("table's index not equal with partition's index. table's index=" + tableIndex + + ", partition's index=" + partitionIndex); + } - // check if partition name exists - Preconditions.checkState(recoverPartition.getName().equalsIgnoreCase(partitionName)); - if (!Strings.isNullOrEmpty(newPartitionName)) { - if (table.checkPartitionNameExist(newPartitionName)) { - throw new DdlException("Partition name[" + newPartitionName + "] is already used"); + // check if partition name exists + Preconditions.checkState(recoverPartition.getName().equalsIgnoreCase(partitionName)); + if (!Strings.isNullOrEmpty(newPartitionName)) { + if (table.checkPartitionNameExist(newPartitionName)) { + throw new DdlException("Partition name[" + newPartitionName + "] is already used"); + } + recoverPartition.setName(newPartitionName); } - recoverPartition.setName(newPartitionName); - } - // recover partition - table.addPartition(recoverPartition); + // recover partition + table.addPartition(recoverPartition); - // recover partition info - long partitionId = recoverPartition.getId(); - partitionInfo.setItem(partitionId, false, recoverItem); - partitionInfo.setDataProperty(partitionId, recoverPartitionInfo.getDataProperty()); - partitionInfo.setReplicaAllocation(partitionId, recoverPartitionInfo.getReplicaAlloc()); - partitionInfo.setIsInMemory(partitionId, recoverPartitionInfo.isInMemory()); - partitionInfo.setIsMutable(partitionId, recoverPartitionInfo.isMutable()); + // recover partition info + long partitionId = recoverPartition.getId(); + partitionInfo.setItem(partitionId, false, recoverItem); + partitionInfo.setDataProperty(partitionId, recoverPartitionInfo.getDataProperty()); + partitionInfo.setReplicaAllocation(partitionId, recoverPartitionInfo.getReplicaAlloc()); + partitionInfo.setIsInMemory(partitionId, recoverPartitionInfo.isInMemory()); + partitionInfo.setIsMutable(partitionId, recoverPartitionInfo.isMutable()); - // remove from recycle bin - idToPartition.remove(partitionId); - idToRecycleTime.remove(partitionId); + // remove from recycle bin + idToPartition.remove(partitionId); + idToRecycleTime.remove(partitionId); + + // log + RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(), partitionId, "", "", newPartitionName); + Env.getCurrentEnv().getEditLog().logRecoverPartition(recoverInfo); + LOG.info("recover partition[{}]", partitionId); + } - // log - RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(), partitionId, "", "", newPartitionName); - Env.getCurrentEnv().getEditLog().logRecoverPartition(recoverInfo); - LOG.info("recover partition[{}]", partitionId); } // The caller should keep table write lock diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 35bf31ad3fcdb3b..6c214f581b5b581 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -82,6 +82,7 @@ import org.apache.doris.binlog.BinlogGcer; import org.apache.doris.binlog.BinlogManager; import org.apache.doris.blockrule.SqlBlockRuleMgr; +import org.apache.doris.catalog.CatalogRecycleBin.RecyclePartitionInfo; import org.apache.doris.catalog.ColocateTableIndex.GroupId; import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; @@ -125,6 +126,7 @@ import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.SmallFileMgr; +import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.consistency.ConsistencyChecker; @@ -6641,4 +6643,107 @@ private void replayJournalsAndExit() { System.exit(0); } + + /** + * Get table ddl stmt from partition Info. + * Only olap table + * + */ + public static void getDdlStmtFromPartitionInfo(TableIf table, + RecyclePartitionInfo recovRecyclePartitionInfo, String createTableStmt) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE "); + + sb.append(table.getType() != TableType.MATERIALIZED_VIEW ? "TABLE " : "MATERIALIZED VIEW "); + sb.append("`").append(table.getName()).append("`"); + sb.append(" (\n"); + int idx = 0; + OlapTable olapTable = (OlapTable) table; + + // this getting always latest column set. + // how to get old one? + long oldIndex = recovRecyclePartitionInfo.getPartition().getBaseIndex().getId(); + List columns = olapTable.getSchemaByIndexId(oldIndex); + for (Column column : columns) { + if (idx++ != 0) { + sb.append(",\n"); + } + // There MUST BE 2 space in front of each column description line + // sqlalchemy requires this to parse SHOW CREATE TABLE stmt. + if (table.isManagedTable()) { + sb.append(" ").append( + column.toSql(((OlapTable) table).getKeysType() == KeysType.UNIQUE_KEYS, true)); + } else { + sb.append(" ").append(column.toSql()); + } + } + if (table instanceof OlapTable) { + if (CollectionUtils.isNotEmpty(olapTable.getIndexes())) { + for (Index index : olapTable.getIndexes()) { + sb.append(",\n"); + sb.append(" ").append(index.toSql()); + } + } + } + sb.append("\n) ENGINE="); + sb.append(table.getType().name()); + + if (table instanceof OlapTable) { + // keys + String keySql = olapTable.getKeysType().toSql(); + if (olapTable.isDuplicateWithoutKey()) { + // after #18621, use can create a DUP_KEYS olap table without key columns + // and get a ddl schema without key type and key columns + } else { + sb.append("\n").append(keySql).append("("); + List keysColumnNames = Lists.newArrayList(); + Map clusterKeysColumnNamesToId = new TreeMap<>(); + for (Column column : olapTable.getBaseSchema()) { + if (column.isKey()) { + keysColumnNames.add("`" + column.getName() + "`"); + } + if (column.isClusterKey()) { + clusterKeysColumnNamesToId.put(column.getClusterKeyId(), column.getName()); + } + } + sb.append(Joiner.on(", ").join(keysColumnNames)).append(")"); + // show cluster keys + if (!clusterKeysColumnNamesToId.isEmpty()) { + sb.append("\n").append("CLUSTER BY (`"); + sb.append(Joiner.on("`, `").join(clusterKeysColumnNamesToId.values())).append("`)"); + } + } + + // partition + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + List partitionId = null; + if ((partitionInfo.getType() == PartitionType.RANGE + || partitionInfo.getType() == PartitionType.LIST)) { + sb.append("\n").append(partitionInfo.toSql(olapTable, partitionId)); + } + + // distribution + DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo(); + sb.append("\n").append(distributionInfo.toSql()); + + // properties + sb.append("\nPROPERTIES (\n"); + addOlapTablePropertyInfo(olapTable, sb, false, false, partitionId); + sb.append("\n)"); + } + sb.append(";"); + createTableStmt = sb.toString(); + } + + public void createTableUsingPartitionInfo(String newTableName, + RecyclePartitionInfo recovRecyclePartitionInfo, OlapTable origTable) throws UserException { + String createTableStmt = null; + Env.getDdlStmtFromPartitionInfo(origTable, recovRecyclePartitionInfo, createTableStmt); + LOG.info("create table stmt: {}", createTableStmt); + CreateTableStmt parsedCreateTableStmt = (CreateTableStmt) SqlParserUtils.parseAndAnalyzeStmt( + createTableStmt, ConnectContext.get()); + parsedCreateTableStmt.setTableName(newTableName); + parsedCreateTableStmt.setIfNotExists(true); + createTable(parsedCreateTableStmt); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 16e0106b3185b1a..05dc757377af8ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -685,31 +685,46 @@ public void recoverTable(RecoverTableStmt recoverStmt) throws DdlException { } public void recoverPartition(RecoverPartitionStmt recoverStmt) throws DdlException { + boolean needCreateNewTable = false; String dbName = recoverStmt.getDbName(); String tableName = recoverStmt.getTableName(); Database db = getDbOrDdlException(dbName); OlapTable olapTable = db.getOlapTableOrDdlException(tableName); - olapTable.writeLockOrDdlException(); + String newTableName = recoverStmt.getNewTableName(); + if (!Strings.isNullOrEmpty(newTableName)) { + needCreateNewTable = true; + } try { + // if we are going to create a new table , then we need to take writelock + // on db level. else just need table level. String partitionName = recoverStmt.getPartitionName(); String newPartitionName = recoverStmt.getNewPartitionName(); - if (Strings.isNullOrEmpty(newPartitionName)) { - if (olapTable.getPartition(partitionName) != null) { - throw new DdlException("partition[" + partitionName + "] " - + "already exist in table[" + tableName + "]"); - } + + if (needCreateNewTable) { + db.writeLockOrDdlException(); } else { - if (olapTable.getPartition(newPartitionName) != null) { - throw new DdlException("partition[" + newPartitionName + "] " - + "already exist in table[" + tableName + "]"); + olapTable.writeLockOrDdlException(); + if (Strings.isNullOrEmpty(newPartitionName)) { + if (olapTable.getPartition(partitionName) != null) { + throw new DdlException("partition[" + partitionName + "] " + + "already exist in table[" + tableName + "]"); + } + } else { + if (olapTable.getPartition(newPartitionName) != null) { + throw new DdlException("partition[" + newPartitionName + "] " + + "already exist in table[" + tableName + "]"); + } } } - Env.getCurrentRecycleBin().recoverPartition(db.getId(), olapTable, partitionName, - recoverStmt.getPartitionId(), newPartitionName); + recoverStmt.getPartitionId(), newPartitionName, newTableName, db); } finally { - olapTable.writeUnlock(); + if (needCreateNewTable) { + db.writeUnlock(); + } else { + olapTable.writeUnlock(); + } } } diff --git a/regression-test/data/catalog_recycle_bin_p0/recover_partition_with_newtable.out b/regression-test/data/catalog_recycle_bin_p0/recover_partition_with_newtable.out new file mode 100644 index 000000000000000..7d3480549f142c8 --- /dev/null +++ b/regression-test/data/catalog_recycle_bin_p0/recover_partition_with_newtable.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_check_1 -- +1 a 2022-01-02 +2 a 2023-01-02 +3 a 2024-01-02 + diff --git a/regression-test/suites/catalog_recycle_bin_p0/recover_partition_with_newtable.groovy b/regression-test/suites/catalog_recycle_bin_p0/recover_partition_with_newtable.groovy new file mode 100644 index 000000000000000..7bf79e5f4c671dc --- /dev/null +++ b/regression-test/suites/catalog_recycle_bin_p0/recover_partition_with_newtable.groovy @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("recover_partition_with_newtable") { + def dbName = "recover_partition_with_newtable_db" + sql "drop database if exists ${dbName} force" + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "use ${dbName}" + + def table = "test_table" + def table_new = "test_table_new" + // create table and insert data + sql """ drop table if exists ${table} """ + sql """ + create table ${table} ( + `id` int(11), + `name` varchar(128), + `da` date + ) + engine=olap + duplicate key(id) + partition by range(da)( + PARTITION p1 VALUES LESS THAN ('2023-01-01'), + PARTITION p2 VALUES LESS THAN ('2024-01-01'), + PARTITION p3 VALUES LESS THAN ('2025-01-01') + ) + distributed by hash(id) buckets 2 + properties( + "replication_num"="1", + "light_schema_change"="true" + ); + """ + + sql """ insert into ${table} values(1, 'a', '2022-01-02'); """ + sql """ insert into ${table} values(2, 'a', '2023-01-02'); """ + sql """ insert into ${table} values(3, 'a', '2024-01-02'); """ + + qt_select_check_1 """select * from ${table} order by id,name,da; """ + + // drop partition + sql """ ALTER TABLE ${table} DROP PARTITION p1; """ + sql """ ALTER TABLE ${table} DROP PARTITION p2; """ + sql """ ALTER TABLE ${table} DROP PARTITION p3; """ + + + sql """ RECOVER PARTITION p1 from ${table} as ${table_new}; """ + sql """ RECOVER PARTITION p2 from ${table} as ${table_new}; """ + sql """ RECOVER PARTITION p3 from ${table} as ${table_new}; """ + + qt_select_check_1 """select * from ${table_new} order by id,name,da; """ + +}