From a8f2bad7b3ffc696669571a0d47f9553e53812dc Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Tue, 24 Sep 2024 16:58:22 +0800 Subject: [PATCH] [feat](metatable) support table$partitions for hive table (#40774) Support new grammar: `table_name$partitions` User can query partition info by using: ``` select * from table_name$partitions ``` `table_name$partitions` is a special meta table corresponding to the table. Its schema is the partition columns of the table, and column type is always "String". And the value is the partition column's value You can use `DESC table_name$partitions` to get schema: ``` mysql> desc partition_values_all_types$partitions; +-------+----------------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +-------+----------------+------+------+---------+-------+ | p1 | boolean | Yes | true | NULL | NONE | | p2 | tinyint | Yes | true | NULL | NONE | | p3 | smallint | Yes | true | NULL | NONE | | p4 | int | Yes | true | NULL | NONE | | p5 | bigint | Yes | true | NULL | NONE | | p6 | date | Yes | true | NULL | NONE | | p7 | datetime(6) | Yes | true | NULL | NONE | | p8 | varchar(65533) | Yes | true | NULL | NONE | | p9 | decimal(9,2) | Yes | true | NULL | NONE | | p10 | decimal(18,10) | Yes | true | NULL | NONE | | p11 | decimal(38,9) | Yes | true | NULL | NONE | +-------+----------------+------+------+---------+-------+ ``` Where `px` are partition columns of table `partition_values_all_types`; ``` mysql> select * from partition_values_all_types$partitions order by p1,p2,p3; +------+------+--------+-------------+----------------------+------------+----------------------------+--------+-------------+----------------------+------------------------------------------+ | p1 | p2 | p3 | p4 | p5 | p6 | p7 | p8 | p9 | p10 | p11 | +------+------+--------+-------------+----------------------+------------+----------------------------+--------+-------------+----------------------+------------------------------------------+ | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | | 0 | -128 | -32768 | -2147483648 | -9223372036854775808 | 1900-01-01 | 1899-01-01 23:59:59.000000 | NULL | -9999999.99 | -99999999.9999999999 | -99999999999999999999999999999.999999999 | | 0 | 127 | 32767 | 2147483647 | 9223372036854775807 | 9999-12-31 | 0001-01-01 00:00:01.321000 | boston | 9999999.99 | 99999999.9999999999 | 99999999999999999999999999999.999999999 | +------+------+--------+-------------+----------------------+------------+----------------------------+--------+-------------+----------------------+------------------------------------------+ ``` Currently, this grammar can only be used for partition table's in Hive Catalog. Table in other catalogs or non partition table can not use this grammar. Internally, it is implemented as a table valued function: `partition_values` ``` mysql> select * from partition_values("catalog" = "hive", "database" = "multi_catalog", "table" = "partition_values_all_types"); +------+------+--------+-------------+----------------------+------------+----------------------------+--------+-------------+----------------------+------------------------------------------+ | p1 | p2 | p3 | p4 | p5 | p6 | p7 | p8 | p9 | p10 | p11 | +------+------+--------+-------------+----------------------+------------+----------------------------+--------+-------------+----------------------+------------------------------------------+ | 0 | 127 | 32767 | 2147483647 | 9223372036854775807 | 9999-12-31 | 0001-01-01 00:00:01.321000 | boston | 9999999.99 | 99999999.9999999999 | 99999999999999999999999999999.999999999 | | 0 | -128 | -32768 | -2147483648 | -9223372036854775808 | 1900-01-01 | 1899-01-01 23:59:59.000000 | NULL | -9999999.99 | -99999999.9999999999 | -99999999999999999999999999999.999999999 | | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | +------+------+--------+-------------+----------------------+------------+----------------------------+--------+-------------+----------------------+------------------------------------------+ ``` --- be/src/vec/exec/scan/vmeta_scanner.cpp | 170 +++++++++++------ be/src/vec/exec/scan/vmeta_scanner.h | 2 + .../org/apache/doris/nereids/DorisParser.g4 | 8 +- .../apache/doris/analysis/DescribeStmt.java | 28 ++- .../catalog/BuiltinTableValuedFunctions.java | 4 +- .../org/apache/doris/catalog/TableIf.java | 2 - .../apache/doris/common/util/TimeUtils.java | 35 ++++ .../apache/doris/datasource/CatalogIf.java | 23 +++ .../datasource/hive/HMSExternalCatalog.java | 92 +++++++++ .../datasource/hive/HMSExternalTable.java | 2 +- .../nereids/parser/LogicalPlanBuilder.java | 7 +- .../nereids/rules/analysis/BindRelation.java | 24 ++- .../functions/table/PartitionValues.java | 76 ++++++++ .../visitor/TableValuedFunctionVisitor.java | 5 + .../doris/nereids/util/RelationUtil.java | 6 +- .../tablefunction/MetadataGenerator.java | 143 +++++++++++++- .../PartitionValuesTableValuedFunction.java | 180 ++++++++++++++++++ .../PartitionsTableValuedFunction.java | 2 +- .../tablefunction/TableValuedFunctionIf.java | 2 + .../doris/common/util/TimeUtilsTest.java | 51 +++++ .../rules/analysis/BindRelationTest.java | 9 + gensrc/thrift/Data.thrift | 1 + gensrc/thrift/FrontendService.thrift | 1 + gensrc/thrift/PlanNodes.thrift | 8 + gensrc/thrift/Types.thrift | 3 +- .../hive/test_hive_partition_values_tvf.out | 120 ++++++++++++ .../test_hive_partition_values_tvf.groovy | 142 ++++++++++++++ 27 files changed, 1053 insertions(+), 93 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/PartitionValues.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionValuesTableValuedFunction.java create mode 100644 regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out create mode 100644 regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp b/be/src/vec/exec/scan/vmeta_scanner.cpp index f5864924a389fa..74fed8c80c73cd 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.cpp +++ b/be/src/vec/exec/scan/vmeta_scanner.cpp @@ -74,16 +74,6 @@ Status VMetaScanner::prepare(RuntimeState* state, const VExprContextSPtrs& conju VLOG_CRITICAL << "VMetaScanner::prepare"; RETURN_IF_ERROR(VScanner::prepare(_state, conjuncts)); _tuple_desc = state->desc_tbl().get_tuple_descriptor(_tuple_id); - bool has_col_nullable = - std::any_of(std::begin(_tuple_desc->slots()), std::end(_tuple_desc->slots()), - [](SlotDescriptor* slot_desc) { return slot_desc->is_nullable(); }); - - if (has_col_nullable) { - // We do not allow any columns to be Nullable here, since FE can not - // transmit a NULL value to BE, so we can not distinguish a empty string - // from a NULL value. - return Status::InternalError("Logical error, VMetaScanner do not allow ColumnNullable"); - } return Status::OK(); } @@ -146,60 +136,90 @@ Status VMetaScanner::_fill_block_with_remote_data(const std::vectortype().type) { - case TYPE_BOOLEAN: { - bool data = _batch_data[_row_idx].column_value[col_idx].boolVal; - reinterpret_cast*>(col_ptr) - ->insert_value((uint8_t)data); - break; - } - case TYPE_INT: { - int64_t data = _batch_data[_row_idx].column_value[col_idx].intVal; - reinterpret_cast*>(col_ptr) - ->insert_value(data); - break; - } - case TYPE_BIGINT: { - int64_t data = _batch_data[_row_idx].column_value[col_idx].longVal; - reinterpret_cast*>(col_ptr) - ->insert_value(data); - break; - } - case TYPE_FLOAT: { - double data = _batch_data[_row_idx].column_value[col_idx].doubleVal; - reinterpret_cast*>(col_ptr) - ->insert_value(data); - break; - } - case TYPE_DOUBLE: { - double data = _batch_data[_row_idx].column_value[col_idx].doubleVal; - reinterpret_cast*>(col_ptr) - ->insert_value(data); - break; - } - case TYPE_DATETIMEV2: { - uint64_t data = _batch_data[_row_idx].column_value[col_idx].longVal; - reinterpret_cast*>(col_ptr) - ->insert_value(data); - break; - } - case TYPE_STRING: - case TYPE_CHAR: - case TYPE_VARCHAR: { - std::string data = _batch_data[_row_idx].column_value[col_idx].stringVal; - reinterpret_cast(col_ptr)->insert_data(data.c_str(), - data.length()); - break; - } - default: { - std::string error_msg = - fmt::format("Invalid column type {} on column: {}.", - slot_desc->type().debug_string(), slot_desc->col_name()); - return Status::InternalError(std::string(error_msg)); - } + TCell& cell = _batch_data[_row_idx].column_value[col_idx]; + if (cell.__isset.isNull && cell.isNull) { + DCHECK(slot_desc->is_nullable()) + << "cell is null but column is not nullable: " << slot_desc->col_name(); + auto& null_col = reinterpret_cast(*col_ptr); + null_col.get_nested_column().insert_default(); + null_col.get_null_map_data().push_back(1); + } else { + if (slot_desc->is_nullable()) { + auto& null_col = reinterpret_cast(*col_ptr); + null_col.get_null_map_data().push_back(0); + col_ptr = null_col.get_nested_column_ptr(); + } + switch (slot_desc->type().type) { + case TYPE_BOOLEAN: { + bool data = cell.boolVal; + reinterpret_cast*>(col_ptr) + ->insert_value((uint8_t)data); + break; + } + case TYPE_TINYINT: { + int8_t data = (int8_t)cell.intVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_SMALLINT: { + int16_t data = (int16_t)cell.intVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_INT: { + int32_t data = cell.intVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_BIGINT: { + int64_t data = cell.longVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_FLOAT: { + double data = cell.doubleVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_DOUBLE: { + double data = cell.doubleVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_DATEV2: { + uint32_t data = (uint32_t)cell.longVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_DATETIMEV2: { + uint64_t data = cell.longVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_STRING: + case TYPE_CHAR: + case TYPE_VARCHAR: { + std::string data = cell.stringVal; + reinterpret_cast(col_ptr)->insert_data( + data.c_str(), data.length()); + break; + } + default: { + std::string error_msg = + fmt::format("Invalid column type {} on column: {}.", + slot_desc->type().debug_string(), slot_desc->col_name()); + return Status::InternalError(std::string(error_msg)); + } + } } } } @@ -241,6 +261,9 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) { case TMetadataType::TASKS: RETURN_IF_ERROR(_build_tasks_metadata_request(meta_scan_range, &request)); break; + case TMetadataType::PARTITION_VALUES: + RETURN_IF_ERROR(_build_partition_values_metadata_request(meta_scan_range, &request)); + break; default: _meta_eos = true; return Status::OK(); @@ -461,6 +484,27 @@ Status VMetaScanner::_build_tasks_metadata_request(const TMetaScanRange& meta_sc return Status::OK(); } +Status VMetaScanner::_build_partition_values_metadata_request( + const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request) { + VLOG_CRITICAL << "VMetaScanner::_build_partition_values_metadata_request"; + if (!meta_scan_range.__isset.partition_values_params) { + return Status::InternalError( + "Can not find TPartitionValuesMetadataParams from meta_scan_range."); + } + + // create request + request->__set_schema_table_name(TSchemaTableName::METADATA_TABLE); + + // create TMetadataTableRequestParams + TMetadataTableRequestParams metadata_table_params; + metadata_table_params.__set_metadata_type(TMetadataType::PARTITION_VALUES); + metadata_table_params.__set_partition_values_metadata_params( + meta_scan_range.partition_values_params); + + request->__set_metada_table_params(metadata_table_params); + return Status::OK(); +} + Status VMetaScanner::close(RuntimeState* state) { VLOG_CRITICAL << "VMetaScanner::close"; RETURN_IF_ERROR(VScanner::close(state)); diff --git a/be/src/vec/exec/scan/vmeta_scanner.h b/be/src/vec/exec/scan/vmeta_scanner.h index a9975300cdc500..350e0fbf807acf 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.h +++ b/be/src/vec/exec/scan/vmeta_scanner.h @@ -86,6 +86,8 @@ class VMetaScanner : public VScanner { TFetchSchemaTableDataRequest* request); Status _build_queries_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request); + Status _build_partition_values_metadata_request(const TMetaScanRange& meta_scan_range, + TFetchSchemaTableDataRequest* request); bool _meta_eos; TupleId _tuple_id; TUserIdentity _user_identity; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 2616f96af6d43c..94f00f0efd0bd3 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1270,12 +1270,12 @@ optScanParams relationPrimary : multipartIdentifier optScanParams? materializedViewName? tableSnapshot? specifiedPartition? - tabletList? tableAlias sample? relationHint? lateralView* #tableName - | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery + tabletList? tableAlias sample? relationHint? lateralView* #tableName + | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery | tvfName=identifier LEFT_PAREN (properties=propertyItemList)? - RIGHT_PAREN tableAlias #tableValuedFunction - | LEFT_PAREN relations RIGHT_PAREN #relationList + RIGHT_PAREN tableAlias #tableValuedFunction + | LEFT_PAREN relations RIGHT_PAREN #relationList ; materializedViewName diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java index 7e503d525868c7..f5f7cdaf7490b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java @@ -32,6 +32,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.proc.IndexSchemaProcNode; import org.apache.doris.common.proc.ProcNodeInterface; @@ -44,6 +45,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.lang3.StringUtils; @@ -55,6 +57,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; public class DescribeStmt extends ShowStmt implements NotFallbackInParser { @@ -123,6 +126,25 @@ public boolean isAllTables() { @Override public void analyze(Analyzer analyzer) throws UserException { + // First handle meta table. + // It will convert this to corresponding table valued functions + // eg: DESC table$partitions -> partition_values(...) + if (dbTableName != null) { + dbTableName.analyze(analyzer); + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(dbTableName.getCtl()); + Pair sourceTableNameWithMetaName = catalog.getSourceTableNameWithMetaTableName( + dbTableName.getTbl()); + if (!Strings.isNullOrEmpty(sourceTableNameWithMetaName.second)) { + isTableValuedFunction = true; + Optional optTvfRef = catalog.getMetaTableFunctionRef( + dbTableName.getDb(), dbTableName.getTbl()); + if (!optTvfRef.isPresent()) { + throw new AnalysisException("meta table not found: " + sourceTableNameWithMetaName.second); + } + tableValuedFunctionRef = optTvfRef.get(); + } + } + if (!isAllTables && isTableValuedFunction) { tableValuedFunctionRef.analyze(analyzer); List columns = tableValuedFunctionRef.getTable().getBaseSchema(); @@ -148,8 +170,6 @@ public void analyze(Analyzer analyzer) throws UserException { } } - dbTableName.analyze(analyzer); - if (!Env.getCurrentEnv().getAccessManager() .checkTblPriv(ConnectContext.get(), dbTableName, PrivPredicate.SHOW)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "DESCRIBE", @@ -159,8 +179,7 @@ public void analyze(Analyzer analyzer) throws UserException { CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(dbTableName.getCtl()); DatabaseIf db = catalog.getDbOrAnalysisException(dbTableName.getDb()); - TableIf table = db.getTableOrAnalysisException(dbTableName.getTbl()); - + TableIf table = db.getTableOrDdlException(dbTableName.getTbl()); table.readLock(); try { if (!isAllTables) { @@ -387,3 +406,4 @@ private static List initEmptyRow() { return emptyRow; } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java index db66c260e5670c..88c9816209365a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos; import org.apache.doris.nereids.trees.expressions.functions.table.Numbers; +import org.apache.doris.nereids.trees.expressions.functions.table.PartitionValues; import org.apache.doris.nereids.trees.expressions.functions.table.Partitions; import org.apache.doris.nereids.trees.expressions.functions.table.Query; import org.apache.doris.nereids.trees.expressions.functions.table.S3; @@ -59,7 +60,8 @@ public class BuiltinTableValuedFunctions implements FunctionHelper { tableValued(Partitions.class, "partitions"), tableValued(Jobs.class, "jobs"), tableValued(Tasks.class, "tasks"), - tableValued(Query.class, "query") + tableValued(Query.class, "query"), + tableValued(PartitionValues.class, "partition_values") ); public static final BuiltinTableValuedFunctions INSTANCE = new BuiltinTableValuedFunctions(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index cb9d780d55a80e..1ddd083cff4b64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -65,8 +65,6 @@ default boolean tryReadLock(long timeout, TimeUnit unit) { default void readUnlock() { } - ; - default void writeLock() { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java index 55b2ed1c58d972..e7066846c30919 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -340,4 +340,39 @@ private static String formatDateStr(String dateStr) { parts.length > 3 ? String.format(" %02d:%02d:%02d", Integer.parseInt(parts[3]), Integer.parseInt(parts[4]), Integer.parseInt(parts[5])) : ""); } + + + // Refer to be/src/vec/runtime/vdatetime_value.h + public static long convertToDateTimeV2( + int year, int month, int day, int hour, int minute, int second, int microsecond) { + return (long) microsecond | (long) second << 20 | (long) minute << 26 | (long) hour << 32 + | (long) day << 37 | (long) month << 42 | (long) year << 46; + } + + // Refer to be/src/vec/runtime/vdatetime_value.h + public static long convertToDateV2( + int year, int month, int day) { + return (long) day | (long) month << 5 | (long) year << 9; + } + + public static long convertStringToDateTimeV2(String dateTimeStr, int scale) { + String format = "yyyy-MM-dd HH:mm:ss"; + if (scale > 0) { + format += "."; + for (int i = 0; i < scale; i++) { + format += "S"; + } + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(format); + LocalDateTime dateTime = TimeUtils.formatDateTimeAndFullZero(dateTimeStr, formatter); + return convertToDateTimeV2(dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth(), + dateTime.getHour(), dateTime.getMinute(), dateTime.getSecond(), dateTime.getNano() / 1000); + } + + public static long convertStringToDateV2(String dateStr) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + LocalDateTime dateTime = TimeUtils.formatDateTimeAndFullZero(dateStr, formatter); + return convertToDateV2(dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth()); + } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index a79897c67df3eb..41cb44ef0b5fa3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.DropDbStmt; import org.apache.doris.analysis.DropTableStmt; import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.TableValuedFunctionRef; import org.apache.doris.analysis.TruncateTableStmt; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; @@ -30,7 +31,9 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -197,4 +200,24 @@ default CatalogLog constructEditLog() { void dropTable(DropTableStmt stmt) throws DdlException; void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlException; + + /** + * Try to parse meta table name from table name. + * Some catalog allow querying meta table like "table_name$partitions". + * Catalog can override this method to parse meta table name from table name. + * + * @param tableName table name like "table_name" or "table_name$partitions" + * @return pair of source table name and meta table name + */ + default Pair getSourceTableNameWithMetaTableName(String tableName) { + return Pair.of(tableName, ""); + } + + default Optional getMetaTableFunction(String dbName, String sourceNameWithMetaName) { + return Optional.empty(); + } + + default Optional getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) { + return Optional.empty(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 5faf1f2bb6e723..5173b414b8acc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -17,11 +17,13 @@ package org.apache.doris.datasource.hive; +import org.apache.doris.analysis.TableValuedFunctionRef; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.HdfsResource; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.security.authentication.AuthenticationConfig; import org.apache.doris.common.security.authentication.HadoopAuthenticator; @@ -39,10 +41,15 @@ import org.apache.doris.fs.FileSystemProvider; import org.apache.doris.fs.FileSystemProviderImpl; import org.apache.doris.fs.remote.dfs.DFSFileSystem; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.functions.table.PartitionValues; +import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.transaction.TransactionManagerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import lombok.Getter; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.hive.conf.HiveConf; @@ -52,6 +59,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ThreadPoolExecutor; /** @@ -277,6 +285,36 @@ public void setDefaultPropsIfMissing(boolean isReplay) { } } + @Override + public Pair getSourceTableNameWithMetaTableName(String tableName) { + for (MetaTableFunction metaFunction : MetaTableFunction.values()) { + if (metaFunction.containsMetaTable(tableName)) { + return Pair.of(metaFunction.getSourceTableName(tableName), metaFunction.name().toLowerCase()); + } + } + return Pair.of(tableName, ""); + } + + @Override + public Optional getMetaTableFunction(String dbName, String sourceNameWithMetaName) { + for (MetaTableFunction metaFunction : MetaTableFunction.values()) { + if (metaFunction.containsMetaTable(sourceNameWithMetaName)) { + return Optional.of(metaFunction.createFunction(name, dbName, sourceNameWithMetaName)); + } + } + return Optional.empty(); + } + + @Override + public Optional getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) { + for (MetaTableFunction metaFunction : MetaTableFunction.values()) { + if (metaFunction.containsMetaTable(sourceNameWithMetaName)) { + return Optional.of(metaFunction.createFunctionRef(name, dbName, sourceNameWithMetaName)); + } + } + return Optional.empty(); + } + public String getHiveMetastoreUris() { return catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, ""); } @@ -292,4 +330,58 @@ public int getHmsEventsBatchSizePerRpc() { public boolean isEnableHmsEventsIncrementalSync() { return enableHmsEventsIncrementalSync; } + + /** + * Enum for meta tables in hive catalog. + * eg: tbl$partitions + */ + private enum MetaTableFunction { + PARTITIONS("partition_values"); + + private final String suffix; + private final String tvfName; + + MetaTableFunction(String tvfName) { + this.suffix = "$" + name().toLowerCase(); + this.tvfName = tvfName; + } + + boolean containsMetaTable(String tableName) { + return tableName.endsWith(suffix) && (tableName.length() > suffix.length()); + } + + String getSourceTableName(String tableName) { + return tableName.substring(0, tableName.length() - suffix.length()); + } + + public TableValuedFunction createFunction(String ctlName, String dbName, String sourceNameWithMetaName) { + switch (this) { + case PARTITIONS: + List nameParts = Lists.newArrayList(ctlName, dbName, + getSourceTableName(sourceNameWithMetaName)); + return PartitionValues.create(nameParts); + default: + throw new AnalysisException("Unsupported meta function type: " + this); + } + } + + public TableValuedFunctionRef createFunctionRef(String ctlName, String dbName, String sourceNameWithMetaName) { + switch (this) { + case PARTITIONS: + Map params = Maps.newHashMap(); + params.put("catalog", ctlName); + params.put("database", dbName); + params.put("table", getSourceTableName(sourceNameWithMetaName)); + try { + return new TableValuedFunctionRef(tvfName, null, params); + } catch (org.apache.doris.common.AnalysisException e) { + LOG.warn("should not happen. {}.{}.{}", ctlName, dbName, sourceNameWithMetaName); + return null; + } + default: + throw new AnalysisException("Unsupported meta function type: " + this); + } + } + } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 3968c61c3f3de3..f012601b82f283 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -923,6 +923,6 @@ private List getFilesForPartitions( @Override public boolean isPartitionedTable() { makeSureInitialized(); - return remoteTable.getPartitionKeysSize() > 0; + return !isView() && remoteTable.getPartitionKeysSize() > 0; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 8a4c28ef036539..3f84f6b9e5229a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1408,7 +1408,7 @@ protected LogicalPlan withTableAlias(LogicalPlan plan, TableAliasContext ctx) { @Override public LogicalPlan visitTableName(TableNameContext ctx) { - List tableId = visitMultipartIdentifier(ctx.multipartIdentifier()); + List nameParts = visitMultipartIdentifier(ctx.multipartIdentifier()); List partitionNames = new ArrayList<>(); boolean isTempPart = false; if (ctx.specifiedPartition() != null) { @@ -1456,8 +1456,9 @@ public LogicalPlan visitTableName(TableNameContext ctx) { TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample()); UnboundRelation relation = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), - tableId, partitionNames, isTempPart, tabletIdLists, relationHints, - Optional.ofNullable(tableSample), indexName, scanParams, Optional.ofNullable(tableSnapshot)); + nameParts, partitionNames, isTempPart, tabletIdLists, relationHints, + Optional.ofNullable(tableSample), indexName, scanParams, Optional.ofNullable(tableSnapshot)); + LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy(relation); LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias()); for (LateralViewContext lateralViewContext : ctx.lateralView()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index d266ffb712a8d4..2d592a02e97c67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -65,6 +65,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.functions.agg.QuantileUnion; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PreAggStatus; @@ -81,6 +82,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalTestScan; import org.apache.doris.nereids.trees.plans.logical.LogicalView; import org.apache.doris.nereids.util.RelationUtil; @@ -366,14 +368,32 @@ public static LogicalPlan checkAndAddDeleteSignFilter(LogicalOlapScan scan, Conn return scan; } + private Optional handleMetaTable(TableIf table, UnboundRelation unboundRelation, + List qualifiedTableName) { + Optional tvf = table.getDatabase().getCatalog().getMetaTableFunction( + qualifiedTableName.get(1), qualifiedTableName.get(2)); + if (tvf.isPresent()) { + return Optional.of(new LogicalTVFRelation(unboundRelation.getRelationId(), tvf.get())); + } + return Optional.empty(); + } + private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelation, - List qualifiedTableName, CascadesContext cascadesContext) { - // for create view stmt replace tablNname to ctl.db.tableName + List qualifiedTableName, CascadesContext cascadesContext) { + // for create view stmt replace tableName to ctl.db.tableName unboundRelation.getIndexInSqlString().ifPresent(pair -> { StatementContext statementContext = cascadesContext.getStatementContext(); statementContext.addIndexInSqlToString(pair, Utils.qualifiedNameWithBackquote(qualifiedTableName)); }); + + // Handle meta table like "table_name$partitions" + // qualifiedTableName should be like "ctl.db.tbl$partitions" + Optional logicalPlan = handleMetaTable(table, unboundRelation, qualifiedTableName); + if (logicalPlan.isPresent()) { + return logicalPlan.get(); + } + List qualifierWithoutTableName = Lists.newArrayList(); qualifierWithoutTableName.addAll(qualifiedTableName.subList(0, qualifiedTableName.size() - 1)); boolean isView = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/PartitionValues.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/PartitionValues.java new file mode 100644 index 00000000000000..0389bb626371aa --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/PartitionValues.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.table; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Properties; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.coercion.AnyDataType; +import org.apache.doris.tablefunction.PartitionValuesTableValuedFunction; +import org.apache.doris.tablefunction.TableValuedFunctionIf; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; + +/** + * partition_values + */ +public class PartitionValues extends TableValuedFunction { + public PartitionValues(Properties properties) { + super("partition_values", properties); + } + + /** + * Create PartitionValues function. + * @param qualifiedTableName ctl.db.tbl + * @return PartitionValues function + */ + public static TableValuedFunction create(List qualifiedTableName) { + Preconditions.checkArgument(qualifiedTableName != null && qualifiedTableName.size() == 3); + Map parameters = Maps.newHashMap(); + parameters.put(PartitionValuesTableValuedFunction.CATALOG, qualifiedTableName.get(0)); + parameters.put(PartitionValuesTableValuedFunction.DB, qualifiedTableName.get(1)); + parameters.put(PartitionValuesTableValuedFunction.TABLE, qualifiedTableName.get(2)); + return new PartitionValues(new Properties(parameters)); + } + + @Override + public FunctionSignature customSignature() { + return FunctionSignature.of(AnyDataType.INSTANCE_WITHOUT_INDEX, getArgumentsTypes()); + } + + @Override + protected TableValuedFunctionIf toCatalogFunction() { + try { + Map arguments = getTVFProperties().getMap(); + return new PartitionValuesTableValuedFunction(arguments); + } catch (Throwable t) { + throw new AnalysisException("Can not build PartitionsTableValuedFunction by " + + this + ": " + t.getMessage(), t); + } + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitPartitionValues(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java index ca14edd87def39..0b4b57e11dce16 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos; import org.apache.doris.nereids.trees.expressions.functions.table.Numbers; +import org.apache.doris.nereids.trees.expressions.functions.table.PartitionValues; import org.apache.doris.nereids.trees.expressions.functions.table.Partitions; import org.apache.doris.nereids.trees.expressions.functions.table.Query; import org.apache.doris.nereids.trees.expressions.functions.table.S3; @@ -59,6 +60,10 @@ default R visitPartitions(Partitions partitions, C context) { return visitTableValuedFunction(partitions, context); } + default R visitPartitionValues(PartitionValues partitionValues, C context) { + return visitTableValuedFunction(partitionValues, context); + } + default R visitJobs(Jobs jobs, C context) { return visitTableValuedFunction(jobs, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java index b145338ff81c48..b72498a2227a2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java @@ -100,8 +100,10 @@ public static Pair, TableIf> getDbAndTable(List qualifierN try { DatabaseIf db = catalog.getDb(dbName).orElseThrow(() -> new AnalysisException( "Database [" + dbName + "] does not exist.")); - TableIf table = db.getTable(tableName).orElseThrow(() -> new AnalysisException( - "Table [" + tableName + "] does not exist in database [" + dbName + "].")); + Pair sourceTblNameWithMetaTblName = catalog.getSourceTableNameWithMetaTableName(tableName); + String sourceTableName = sourceTblNameWithMetaTblName.first; + TableIf table = db.getTable(sourceTableName).orElseThrow(() -> new AnalysisException( + "Table [" + sourceTableName + "] does not exist in database [" + dbName + "].")); return Pair.of(db, table); } catch (Throwable e) { throw new AnalysisException(e.getMessage(), e.getCause()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index ce2e564716f8c3..466d324c36951d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -30,10 +30,13 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.SchemaTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.TableProperty; +import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ClientPool; import org.apache.doris.common.Pair; @@ -42,11 +45,14 @@ import org.apache.doris.common.proc.PartitionsProcDir; import org.apache.doris.common.util.NetUtils; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveMetaStoreCache; import org.apache.doris.datasource.hudi.source.HudiCachedPartitionProcessor; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; @@ -78,6 +84,7 @@ import org.apache.doris.thrift.TMetadataTableRequestParams; import org.apache.doris.thrift.TMetadataType; import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPartitionValuesMetadataParams; import org.apache.doris.thrift.TPartitionsMetadataParams; import org.apache.doris.thrift.TPipelineWorkloadGroup; import org.apache.doris.thrift.TRow; @@ -204,7 +211,8 @@ public static TFetchSchemaTableDataResult getMetadataTable(TFetchSchemaTableData } TFetchSchemaTableDataResult result; TMetadataTableRequestParams params = request.getMetadaTableParams(); - switch (request.getMetadaTableParams().getMetadataType()) { + TMetadataType metadataType = request.getMetadaTableParams().getMetadataType(); + switch (metadataType) { case ICEBERG: result = icebergMetadataResult(params); break; @@ -232,11 +240,17 @@ public static TFetchSchemaTableDataResult getMetadataTable(TFetchSchemaTableData case TASKS: result = taskMetadataResult(params); break; + case PARTITION_VALUES: + result = partitionValuesMetadataResult(params); + break; default: return errorResult("Metadata table params is not set."); } if (result.getStatus().getStatusCode() == TStatusCode.OK) { - filterColumns(result, params.getColumnsName(), params.getMetadataType(), params); + if (metadataType != TMetadataType.PARTITION_VALUES) { + // partition_values' result already sorted by column names + filterColumns(result, params.getColumnsName(), params.getMetadataType(), params); + } } if (LOG.isDebugEnabled()) { LOG.debug("getMetadataTable() end."); @@ -329,7 +343,8 @@ private static TFetchSchemaTableDataResult icebergMetadataResult(TMetadataTableR TRow trow = new TRow(); LocalDateTime committedAt = LocalDateTime.ofInstant(Instant.ofEpochMilli( snapshot.timestampMillis()), TimeUtils.getTimeZone().toZoneId()); - long encodedDatetime = convertToDateTimeV2(committedAt.getYear(), committedAt.getMonthValue(), + long encodedDatetime = TimeUtils.convertToDateTimeV2(committedAt.getYear(), + committedAt.getMonthValue(), committedAt.getDayOfMonth(), committedAt.getHour(), committedAt.getMinute(), committedAt.getSecond(), committedAt.getNano() / 1000); @@ -785,12 +800,6 @@ private static void filterColumns(TFetchSchemaTableDataResult result, result.setDataBatch(filterColumnsRows); } - private static long convertToDateTimeV2( - int year, int month, int day, int hour, int minute, int second, int microsecond) { - return (long) microsecond | (long) second << 20 | (long) minute << 26 | (long) hour << 32 - | (long) day << 37 | (long) month << 42 | (long) year << 46; - } - private static TFetchSchemaTableDataResult mtmvMetadataResult(TMetadataTableRequestParams params) { if (LOG.isDebugEnabled()) { LOG.debug("mtmvMetadataResult() start"); @@ -1476,4 +1485,120 @@ private static void fillBatch(List dataBatch, Map dataBatch; + try { + TableIf table = PartitionValuesTableValuedFunction.analyzeAndGetTable(ctlName, dbName, tblName, false); + TableType tableType = table.getType(); + switch (tableType) { + case HMS_EXTERNAL_TABLE: + dataBatch = partitionValuesMetadataResultForHmsTable((HMSExternalTable) table, + params.getColumnsName()); + break; + default: + return errorResult("not support table type " + tableType.name()); + } + TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); + result.setDataBatch(dataBatch); + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } catch (Throwable t) { + LOG.warn("error when get partition values metadata. {}.{}.{}", ctlName, dbName, tblName, t); + return errorResult("error when get partition values metadata: " + Util.getRootCauseMessage(t)); + } + } + + private static List partitionValuesMetadataResultForHmsTable(HMSExternalTable tbl, List colNames) + throws AnalysisException { + List partitionCols = tbl.getPartitionColumns(); + List colIdxs = Lists.newArrayList(); + List types = Lists.newArrayList(); + for (String colName : colNames) { + for (int i = 0; i < partitionCols.size(); ++i) { + if (partitionCols.get(i).getName().equalsIgnoreCase(colName)) { + colIdxs.add(i); + types.add(partitionCols.get(i).getType()); + } + } + } + if (colIdxs.size() != colNames.size()) { + throw new AnalysisException( + "column " + colNames + " does not match partition columns of table " + tbl.getName()); + } + + HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .getMetaStoreCache((HMSExternalCatalog) tbl.getCatalog()); + HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( + tbl.getDbName(), tbl.getName(), tbl.getPartitionColumnTypes()); + Map> valuesMap = hivePartitionValues.getPartitionValuesMap(); + List dataBatch = Lists.newArrayList(); + for (Map.Entry> entry : valuesMap.entrySet()) { + TRow trow = new TRow(); + List values = entry.getValue(); + if (values.size() != partitionCols.size()) { + continue; + } + + for (int i = 0; i < colIdxs.size(); ++i) { + int idx = colIdxs.get(i); + String partitionValue = values.get(idx); + if (partitionValue == null || partitionValue.equals(TablePartitionValues.HIVE_DEFAULT_PARTITION)) { + trow.addToColumnValue(new TCell().setIsNull(true)); + } else { + Type type = types.get(i); + switch (type.getPrimitiveType()) { + case BOOLEAN: + trow.addToColumnValue(new TCell().setBoolVal(Boolean.valueOf(partitionValue))); + break; + case TINYINT: + case SMALLINT: + case INT: + trow.addToColumnValue(new TCell().setIntVal(Integer.valueOf(partitionValue))); + break; + case BIGINT: + trow.addToColumnValue(new TCell().setLongVal(Long.valueOf(partitionValue))); + break; + case FLOAT: + trow.addToColumnValue(new TCell().setDoubleVal(Float.valueOf(partitionValue))); + break; + case DOUBLE: + trow.addToColumnValue(new TCell().setDoubleVal(Double.valueOf(partitionValue))); + break; + case VARCHAR: + case CHAR: + case STRING: + trow.addToColumnValue(new TCell().setStringVal(partitionValue)); + break; + case DATE: + case DATEV2: + trow.addToColumnValue( + new TCell().setLongVal(TimeUtils.convertStringToDateV2(partitionValue))); + break; + case DATETIME: + case DATETIMEV2: + trow.addToColumnValue( + new TCell().setLongVal(TimeUtils.convertStringToDateTimeV2(partitionValue, + ((ScalarType) type).getScalarScale()))); + break; + default: + throw new AnalysisException( + "Unsupported partition column type for $partitions sys table " + type); + } + } + } + dataBatch.add(trow); + } + return dataBatch; + } + } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionValuesTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionValuesTableValuedFunction.java new file mode 100644 index 00000000000000..59efe9fcefde21 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionValuesTableValuedFunction.java @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TMetaScanRange; +import org.apache.doris.thrift.TMetadataType; +import org.apache.doris.thrift.TPartitionValuesMetadataParams; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The Implement of table valued function + * partition_values("catalog"="ctl1", "database" = "db1","table" = "table1"). + */ +public class PartitionValuesTableValuedFunction extends MetadataTableValuedFunction { + private static final Logger LOG = LogManager.getLogger(PartitionValuesTableValuedFunction.class); + + public static final String NAME = "partition_values"; + + public static final String CATALOG = "catalog"; + public static final String DB = "database"; + public static final String TABLE = "table"; + + private static final ImmutableSet PROPERTIES_SET = ImmutableSet.of(CATALOG, DB, TABLE); + + private final String catalogName; + private final String databaseName; + private final String tableName; + private TableIf table; + private List schema; + + public PartitionValuesTableValuedFunction(Map params) throws AnalysisException { + Map validParams = Maps.newHashMap(); + for (String key : params.keySet()) { + if (!PROPERTIES_SET.contains(key.toLowerCase())) { + throw new AnalysisException("'" + key + "' is invalid property"); + } + // check ctl, db, tbl + validParams.put(key.toLowerCase(), params.get(key)); + } + String catalogName = validParams.get(CATALOG); + String dbName = validParams.get(DB); + String tableName = validParams.get(TABLE); + if (StringUtils.isEmpty(catalogName) || StringUtils.isEmpty(dbName) || StringUtils.isEmpty(tableName)) { + throw new AnalysisException("catalog, database and table are required"); + } + this.table = analyzeAndGetTable(catalogName, dbName, tableName, true); + this.catalogName = catalogName; + this.databaseName = dbName; + this.tableName = tableName; + if (LOG.isDebugEnabled()) { + LOG.debug("PartitionsTableValuedFunction() end"); + } + } + + public static TableIf analyzeAndGetTable(String catalogName, String dbName, String tableName, boolean checkAuth) { + if (checkAuth) { + // This method will be called at 2 places: + // One is when planing the query, which should check the privilege of the user. + // the other is when BE call FE to fetch partition values, which should not check the privilege. + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ConnectContext.get(), catalogName, dbName, + tableName, PrivPredicate.SHOW)) { + String message = ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("SHOW PARTITIONS", + ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), + catalogName + ": " + dbName + ": " + tableName); + throw new AnalysisException(message); + } + } + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName); + if (catalog == null) { + throw new AnalysisException("can not find catalog: " + catalogName); + } + // disallow unsupported catalog + if (!(catalog.isInternalCatalog() || catalog instanceof HMSExternalCatalog + || catalog instanceof MaxComputeExternalCatalog)) { + throw new AnalysisException(String.format("Catalog of type '%s' is not allowed in ShowPartitionsStmt", + catalog.getType())); + } + + Optional db = catalog.getDb(dbName); + if (!db.isPresent()) { + throw new AnalysisException("can not find database: " + dbName); + } + TableIf table; + try { + table = db.get().getTableOrMetaException(tableName, TableType.OLAP, + TableType.HMS_EXTERNAL_TABLE, TableType.MAX_COMPUTE_EXTERNAL_TABLE); + } catch (MetaNotFoundException e) { + throw new AnalysisException(e.getMessage(), e); + } + + if (!(table instanceof HMSExternalTable)) { + throw new AnalysisException("Currently only support hive table's partition values meta table"); + } + HMSExternalTable hmsTable = (HMSExternalTable) table; + if (!hmsTable.isPartitionedTable()) { + throw new AnalysisException("Table " + tableName + " is not a partitioned table"); + } + return table; + } + + @Override + public TMetadataType getMetadataType() { + return TMetadataType.PARTITION_VALUES; + } + + @Override + public TMetaScanRange getMetaScanRange() { + if (LOG.isDebugEnabled()) { + LOG.debug("getMetaScanRange() start"); + } + TMetaScanRange metaScanRange = new TMetaScanRange(); + metaScanRange.setMetadataType(TMetadataType.PARTITION_VALUES); + TPartitionValuesMetadataParams partitionParam = new TPartitionValuesMetadataParams(); + partitionParam.setCatalog(catalogName); + partitionParam.setDatabase(databaseName); + partitionParam.setTable(tableName); + metaScanRange.setPartitionValuesParams(partitionParam); + return metaScanRange; + } + + @Override + public String getTableName() { + return "PartitionsValuesTableValuedFunction"; + } + + @Override + public List getTableColumns() throws AnalysisException { + Preconditions.checkNotNull(table); + // TODO: support other type of sys tables + if (schema == null) { + List partitionColumns = ((HMSExternalTable) table).getPartitionColumns(); + schema = Lists.newArrayList(); + for (Column column : partitionColumns) { + schema.add(new Column(column)); + } + } + return schema; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java index 1ceddeb89cf4a8..00169ad555f93e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java @@ -55,7 +55,7 @@ /** * The Implement of table valued function - * partitions("database" = "db1","table" = "table1"). + * partitions("catalog"="ctl1", "database" = "db1","table" = "table1"). */ public class PartitionsTableValuedFunction extends MetadataTableValuedFunction { private static final Logger LOG = LogManager.getLogger(PartitionsTableValuedFunction.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java index 6b6fda088a8a04..d4faa46019541c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java @@ -77,6 +77,8 @@ public static TableValuedFunctionIf getTableFunction(String funcName, Map TimeUtils.convertStringToDateV2("2021-1-1")); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateV2("1900-01-1")); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateV2("20210101")); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateV2("")); + ExceptionChecker.expectThrows(NullPointerException.class, () -> TimeUtils.convertStringToDateV2(null)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateV2("2024:12:31")); + } + + @Test + public void testConvertToBEDatetimeV2Type() { + long result = TimeUtils.convertStringToDateTimeV2("2021-01-01 10:10:10", 0); + Assert.assertEquals(142219811099770880L, result); + result = TimeUtils.convertStringToDateTimeV2("1900-01-01 00:00:00.12", 2); + Assert.assertEquals(133705149423146176L, result); + result = TimeUtils.convertStringToDateTimeV2("1899-12-31 23:59:59.000", 3); + Assert.assertEquals(133687385164611584L, result); + result = TimeUtils.convertStringToDateTimeV2("9999-12-31 23:59:59.123456", 6); + Assert.assertEquals(703674213003812984L, result); + + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2021-1-1", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("1900-01-1", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("20210101", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateTimeV2("", 0)); + ExceptionChecker.expectThrows(NullPointerException.class, () -> TimeUtils.convertStringToDateTimeV2(null, 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2024-10-10", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2024-10-10 10", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2024:12:31", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2024-10-10 11:11:11", 6)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java index 67115e676871b4..369a57017cba28 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java @@ -18,6 +18,8 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PartitionInfo; @@ -98,6 +100,8 @@ public void bindExternalRelation() { new Column("name", Type.VARCHAR) ); + Database externalDatabase = new Database(10000, DEFAULT_CLUSTER_PREFIX + DB1); + OlapTable externalOlapTable = new OlapTable(1, tableName, externalTableColumns, KeysType.DUP_KEYS, new PartitionInfo(), new RandomDistributionInfo(10)) { @Override @@ -109,6 +113,11 @@ public List getBaseSchema(boolean full) { public boolean hasDeleteSign() { return false; } + + @Override + public DatabaseIf getDatabase() { + return externalDatabase; + } }; CustomTableResolver customTableResolver = qualifiedTable -> { diff --git a/gensrc/thrift/Data.thrift b/gensrc/thrift/Data.thrift index d1163821076d8a..dc1190c6e42559 100644 --- a/gensrc/thrift/Data.thrift +++ b/gensrc/thrift/Data.thrift @@ -54,6 +54,7 @@ struct TCell { 3: optional i64 longVal 4: optional double doubleVal 5: optional string stringVal + 6: optional bool isNull // add type: date datetime } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index d106a734b2a81a..05c98b4a279085 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1031,6 +1031,7 @@ struct TMetadataTableRequestParams { 10: optional PlanNodes.TTasksMetadataParams tasks_metadata_params 11: optional PlanNodes.TPartitionsMetadataParams partitions_metadata_params 12: optional PlanNodes.TMetaCacheStatsParams meta_cache_stats_params + 13: optional PlanNodes.TPartitionValuesMetadataParams partition_values_metadata_params } struct TSchemaTableRequestParams { diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index e53289c1b865fd..b6161acd3c8bd9 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -535,6 +535,12 @@ struct TPartitionsMetadataParams { 3: optional string table } +struct TPartitionValuesMetadataParams { + 1: optional string catalog + 2: optional string database + 3: optional string table +} + struct TJobsMetadataParams { 1: optional string type 2: optional Types.TUserIdentity current_user_ident @@ -552,6 +558,7 @@ struct TQueriesMetadataParams { 4: optional TJobsMetadataParams jobs_params 5: optional TTasksMetadataParams tasks_params 6: optional TPartitionsMetadataParams partitions_params + 7: optional TPartitionValuesMetadataParams partition_values_params } struct TMetaCacheStatsParams { @@ -568,6 +575,7 @@ struct TMetaScanRange { 8: optional TTasksMetadataParams tasks_params 9: optional TPartitionsMetadataParams partitions_params 10: optional TMetaCacheStatsParams meta_cache_stats_params + 11: optional TPartitionValuesMetadataParams partition_values_params } // Specification of an individual data range which is held in its entirety diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index ee684a72df2e9a..a92fe883d6ebef 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -732,7 +732,8 @@ enum TMetadataType { JOBS, TASKS, WORKLOAD_SCHED_POLICY, - PARTITIONS; + PARTITIONS, + PARTITION_VALUES; } enum TIcebergQueryType { diff --git a/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out b/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out new file mode 100644 index 00000000000000..bc69c716277ccd --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out @@ -0,0 +1,120 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql01 -- +\N 0.1 test1 +\N 0.2 test2 +100 0.3 test3 + +-- !sql02 -- +\N 0.1 test1 +\N 0.2 test2 +100 0.3 test3 + +-- !sql03 -- +\N 0.1 test1 +\N 0.2 test2 +100 0.3 test3 + +-- !sql11 -- +\N 0.1 +\N 0.2 +100 0.3 + +-- !sql12 -- +0.1 \N +0.2 \N +0.3 100 + +-- !sql13 -- +test1 \N +test2 \N +test3 100 + +-- !sql21 -- +test3 100 0.3 + +-- !sql22 -- +test1 +test2 +test3 + +-- !sql22 -- +3 + +-- !sql22 -- +3 + +-- !sql31 -- +0.1 \N +0.2 \N +0.3 100 + +-- !sql41 -- +100 + +-- !sql42 -- +test3 + +-- !sql51 -- +test3 + +-- !sql61 -- + +-- !sql71 -- +\N 0.2 test2 +100 0.3 test3 +\N 0.1 test1 + +-- !sql72 -- + +-- !sql73 -- +test3 + +-- !sql81 -- +100 0.3 test3 100 0.3 test3 + +-- !sql91 -- +t_int int Yes true \N NONE +t_float float Yes true \N NONE +t_string varchar(65533) Yes true \N NONE + +-- !sql92 -- +t_int int Yes true \N NONE +t_float float Yes true \N NONE +t_string varchar(65533) Yes true \N NONE + +-- !sql93 -- +t_int int Yes true \N NONE + +-- !sql94 -- +t_int int Yes true \N NONE + +-- !sql95 -- +\N 0.1 test1 +\N 0.2 test2 +100 0.3 test3 + +-- !sql101 -- +k1 int Yes true \N + +-- !sql102 -- + +-- !sql111 -- +p1 boolean Yes true \N NONE +p2 tinyint Yes true \N NONE +p3 smallint Yes true \N NONE +p4 int Yes true \N NONE +p5 bigint Yes true \N NONE +p6 date Yes true \N NONE +p7 datetime(6) Yes true \N NONE +p8 varchar(65533) Yes true \N NONE + +-- !sql112 -- +1 test1 true -128 -32768 -2147483648 -9223372036854775808 1900-01-01 1899-01-01T23:59:59 \N +2 \N false 127 32767 2147483647 9223372036854775807 9999-12-31 0001-01-01T00:00:01.321 boston +3 \N \N \N \N \N \N \N \N + +-- !sql113 -- +\N \N \N \N \N \N \N \N +false -128 -32768 -2147483648 -9223372036854775808 1900-01-01 1899-01-01T23:59:59 \N +false 127 32767 2147483647 9223372036854775807 9999-12-31 0001-01-01T00:00:01.321 boston + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy b/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy new file mode 100644 index 00000000000000..7cb764d0165797 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_partition_values_tvf", "p0,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Hive test.") + return; + } + for (String hivePrefix : ["hive3"]) { + String extHiveHmsHost = context.config.otherConfigs.get("externalEnvIp") + String extHiveHmsPort = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = "${hivePrefix}_test_external_catalog_hive_partition" + + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + + // 1. test qualifier + qt_sql01 """ select * from ${catalog_name}.multi_catalog.orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + sql """ switch ${catalog_name} """ + qt_sql02 """ select * from multi_catalog.orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + sql """ use multi_catalog""" + qt_sql03 """ select * from orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + + // 2. test select order + qt_sql11 """ select * except(t_string) from orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + qt_sql12 """ select t_float, t_int from orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + qt_sql13 """ select t_string, t_int from orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + + // 3. test agg + qt_sql21 """ select max(t_string), max(t_int), max(t_float) from orc_partitioned_columns\$partitions""" + qt_sql22 """ select max(t_string) from orc_partitioned_columns\$partitions group by t_int, t_float order by t_int, t_float""" + qt_sql22 """ select count(*) from orc_partitioned_columns\$partitions;""" + qt_sql22 """ select count(1) from orc_partitioned_columns\$partitions;""" + + // 4. test alias + qt_sql31 """ select pv.t_float, pv.t_int from orc_partitioned_columns\$partitions as pv group by t_int, t_float order by t_int, t_float""" + + // 5. test CTE + qt_sql41 """ with v1 as (select t_string, t_int from orc_partitioned_columns\$partitions order by t_int, t_float, t_string) select max(t_int) from v1; """ + qt_sql42 """ with v1 as (select t_string, t_int from orc_partitioned_columns\$partitions order by t_int, t_float, t_string) select c1 from (select max(t_string) as c1 from v1) x; """ + + // 6. test subquery + qt_sql51 """select c1 from (select max(t_string) as c1 from (select * from multi_catalog.orc_partitioned_columns\$partitions)x)y;""" + + // 7. test where + qt_sql61 """select * from orc_partitioned_columns\$partitions where t_int != "__HIVE_DEFAULT_PARTITION__" order by t_int, t_float, t_string; """ + + // 8. test view + sql """drop database if exists internal.partition_values_db""" + sql """create database if not exists internal.partition_values_db""" + sql """create view internal.partition_values_db.v1 as select * from ${catalog_name}.multi_catalog.orc_partitioned_columns\$partitions""" + qt_sql71 """select * from internal.partition_values_db.v1""" + qt_sql72 """select t_string, t_int from internal.partition_values_db.v1 where t_int != "__HIVE_DEFAULT_PARTITION__"""" + qt_sql73 """with v1 as (select t_string, t_int from internal.partition_values_db.v1 order by t_int, t_float, t_string) select c1 from (select max(t_string) as c1 from v1) x;""" + + // 9. test join + qt_sql81 """select * from orc_partitioned_columns\$partitions p1 join orc_partitioned_columns\$partitions p2 on p1.t_int = p2.t_int order by p1.t_int, p1.t_float""" + + // 10. test desc + qt_sql91 """desc orc_partitioned_columns\$partitions""" + qt_sql92 """desc function partition_values("catalog" = "${catalog_name}", "database" = "multi_catalog", "table" = "orc_partitioned_columns");""" + qt_sql93 """desc orc_partitioned_one_column\$partitions""" + qt_sql94 """desc function partition_values("catalog" = "${catalog_name}", "database" = "multi_catalog", "table" = "orc_partitioned_one_column");""" + qt_sql95 """select * from partition_values("catalog" = "${catalog_name}", "database" = "multi_catalog", "table" = "orc_partitioned_columns") order by t_int, t_float""" + + // 11. test non partition table + test { + sql """select * from hive_text_complex_type\$partitions""" + exception "is not a partitioned table" + } + test { + sql """desc hive_text_complex_type\$partitions""" + exception "is not a partitioned table" + } + + // 12. test inner table + sql """create table internal.partition_values_db.pv_inner1 (k1 int) distributed by hash (k1) buckets 1 properties("replication_num" = "1")""" + qt_sql101 """desc internal.partition_values_db.pv_inner1""" + qt_sql102 """select * from internal.partition_values_db.pv_inner1""" + test { + sql """desc internal.partition_values_db.pv_inner1\$partitions""" + exception """Unknown table 'pv_inner1\$partitions'""" + } + + test { + sql """select * from internal.partition_values_db.pv_inner1\$partitions""" + exception """Table [pv_inner1\$partitions] does not exist in database [partition_values_db]""" + } + + // 13. test all types of partition columns + sql """switch ${catalog_name}""" + sql """drop database if exists partition_values_db"""; + sql """create database partition_values_db""" + sql """use partition_values_db""" + + sql """create table partition_values_all_types ( + k1 int, + k2 string, + p1 boolean, + p2 tinyint, + p3 smallint, + p4 int, + p5 bigint, + p6 date, + p7 datetime, + p8 string + ) partition by list(p1, p2, p3, p4, p5, p6, p7, p8)(); + """ + + qt_sql111 """desc partition_values_all_types\$partitions;""" + + sql """insert into partition_values_all_types values + (1, "test1", true, -128, -32768, -2147483648, -9223372036854775808, "1900-01-01", "1899-01-01 23:59:59", ""), + (2, null, false, 127, 32767, 2147483647, 9223372036854775807, "9999-12-31", "0001-01-01 00:00:01.321", "boston"), + (3, "", null, null, null, null, null, null, null, null); + """ + + qt_sql112 """select * from partition_values_all_types order by k1;""" + qt_sql113 """select * from partition_values_all_types\$partitions order by p1,p2,p3;""" + } +} +