diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp b/be/src/vec/exec/scan/vmeta_scanner.cpp index 02cf15e1af3871..64981d0f483c58 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.cpp +++ b/be/src/vec/exec/scan/vmeta_scanner.cpp @@ -85,16 +85,6 @@ Status VMetaScanner::prepare(RuntimeState* state, const VExprContextSPtrs& conju VLOG_CRITICAL << "VMetaScanner::prepare"; RETURN_IF_ERROR(VScanner::prepare(_state, conjuncts)); _tuple_desc = state->desc_tbl().get_tuple_descriptor(_tuple_id); - bool has_col_nullable = - std::any_of(std::begin(_tuple_desc->slots()), std::end(_tuple_desc->slots()), - [](SlotDescriptor* slot_desc) { return slot_desc->is_nullable(); }); - - if (has_col_nullable) { - // We do not allow any columns to be Nullable here, since FE can not - // transmit a NULL value to BE, so we can not distinguish a empty string - // from a NULL value. - return Status::InternalError("Logical error, VMetaScanner do not allow ColumnNullable"); - } return Status::OK(); } @@ -157,60 +147,90 @@ Status VMetaScanner::_fill_block_with_remote_data(const std::vectortype().type) { - case TYPE_BOOLEAN: { - bool data = _batch_data[_row_idx].column_value[col_idx].boolVal; - reinterpret_cast*>(col_ptr) - ->insert_value((uint8_t)data); - break; - } - case TYPE_INT: { - int64_t data = _batch_data[_row_idx].column_value[col_idx].intVal; - reinterpret_cast*>(col_ptr) - ->insert_value(data); - break; - } - case TYPE_BIGINT: { - int64_t data = _batch_data[_row_idx].column_value[col_idx].longVal; - reinterpret_cast*>(col_ptr) - ->insert_value(data); - break; - } - case TYPE_FLOAT: { - double data = _batch_data[_row_idx].column_value[col_idx].doubleVal; - reinterpret_cast*>(col_ptr) - ->insert_value(data); - break; - } - case TYPE_DOUBLE: { - double data = _batch_data[_row_idx].column_value[col_idx].doubleVal; - reinterpret_cast*>(col_ptr) - ->insert_value(data); - break; - } - case TYPE_DATETIMEV2: { - uint64_t data = _batch_data[_row_idx].column_value[col_idx].longVal; - reinterpret_cast*>(col_ptr) - ->insert_value(data); - break; - } - case TYPE_STRING: - case TYPE_CHAR: - case TYPE_VARCHAR: { - std::string data = _batch_data[_row_idx].column_value[col_idx].stringVal; - reinterpret_cast(col_ptr)->insert_data(data.c_str(), - data.length()); - break; - } - default: { - std::string error_msg = - fmt::format("Invalid column type {} on column: {}.", - slot_desc->type().debug_string(), slot_desc->col_name()); - return Status::InternalError(std::string(error_msg)); - } + TCell& cell = _batch_data[_row_idx].column_value[col_idx]; + if (cell.__isset.isNull && cell.isNull) { + DCHECK(slot_desc->is_nullable()) + << "cell is null but column is not nullable: " << slot_desc->col_name(); + auto& null_col = reinterpret_cast(*col_ptr); + null_col.get_nested_column().insert_default(); + null_col.get_null_map_data().push_back(1); + } else { + if (slot_desc->is_nullable()) { + auto& null_col = reinterpret_cast(*col_ptr); + null_col.get_null_map_data().push_back(0); + col_ptr = null_col.get_nested_column_ptr(); + } + switch (slot_desc->type().type) { + case TYPE_BOOLEAN: { + bool data = cell.boolVal; + reinterpret_cast*>(col_ptr) + ->insert_value((uint8_t)data); + break; + } + case TYPE_TINYINT: { + int8_t data = (int8_t)cell.intVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_SMALLINT: { + int16_t data = (int16_t)cell.intVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_INT: { + int32_t data = cell.intVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_BIGINT: { + int64_t data = cell.longVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_FLOAT: { + double data = cell.doubleVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_DOUBLE: { + double data = cell.doubleVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_DATEV2: { + uint32_t data = (uint32_t)cell.longVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_DATETIMEV2: { + uint64_t data = cell.longVal; + reinterpret_cast*>(col_ptr) + ->insert_value(data); + break; + } + case TYPE_STRING: + case TYPE_CHAR: + case TYPE_VARCHAR: { + std::string data = cell.stringVal; + reinterpret_cast(col_ptr)->insert_data( + data.c_str(), data.length()); + break; + } + default: { + std::string error_msg = + fmt::format("Invalid column type {} on column: {}.", + slot_desc->type().debug_string(), slot_desc->col_name()); + return Status::InternalError(std::string(error_msg)); + } + } } } } @@ -252,6 +272,9 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) { case TMetadataType::TASKS: RETURN_IF_ERROR(_build_tasks_metadata_request(meta_scan_range, &request)); break; + case TMetadataType::PARTITION_VALUES: + RETURN_IF_ERROR(_build_partition_values_metadata_request(meta_scan_range, &request)); + break; default: _meta_eos = true; return Status::OK(); @@ -472,6 +495,27 @@ Status VMetaScanner::_build_tasks_metadata_request(const TMetaScanRange& meta_sc return Status::OK(); } +Status VMetaScanner::_build_partition_values_metadata_request( + const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request) { + VLOG_CRITICAL << "VMetaScanner::_build_partition_values_metadata_request"; + if (!meta_scan_range.__isset.partition_values_params) { + return Status::InternalError( + "Can not find TPartitionValuesMetadataParams from meta_scan_range."); + } + + // create request + request->__set_schema_table_name(TSchemaTableName::METADATA_TABLE); + + // create TMetadataTableRequestParams + TMetadataTableRequestParams metadata_table_params; + metadata_table_params.__set_metadata_type(TMetadataType::PARTITION_VALUES); + metadata_table_params.__set_partition_values_metadata_params( + meta_scan_range.partition_values_params); + + request->__set_metada_table_params(metadata_table_params); + return Status::OK(); +} + Status VMetaScanner::close(RuntimeState* state) { VLOG_CRITICAL << "VMetaScanner::close"; RETURN_IF_ERROR(VScanner::close(state)); diff --git a/be/src/vec/exec/scan/vmeta_scanner.h b/be/src/vec/exec/scan/vmeta_scanner.h index b44511826f2c50..600624ee636c6e 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.h +++ b/be/src/vec/exec/scan/vmeta_scanner.h @@ -93,6 +93,8 @@ class VMetaScanner : public VScanner { TFetchSchemaTableDataRequest* request); Status _build_queries_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request); + Status _build_partition_values_metadata_request(const TMetaScanRange& meta_scan_range, + TFetchSchemaTableDataRequest* request); bool _meta_eos; TupleId _tuple_id; TUserIdentity _user_identity; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index a0e3a4582d91c4..9c78ab4e9978eb 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -538,8 +538,8 @@ relationPrimary | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery | tvfName=identifier LEFT_PAREN (properties=propertyItemList)? - RIGHT_PAREN tableAlias #tableValuedFunction - | LEFT_PAREN relations RIGHT_PAREN #relationList + RIGHT_PAREN tableAlias #tableValuedFunction + | LEFT_PAREN relations RIGHT_PAREN #relationList ; materializedViewName diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java index 45883fd6f46dbf..67155c8591ef08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java @@ -32,6 +32,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.proc.IndexSchemaProcNode; import org.apache.doris.common.proc.ProcNodeInterface; @@ -44,6 +45,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.lang3.StringUtils; @@ -55,6 +57,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; public class DescribeStmt extends ShowStmt { @@ -123,6 +126,25 @@ public boolean isAllTables() { @Override public void analyze(Analyzer analyzer) throws UserException { + // First handle meta table. + // It will convert this to corresponding table valued functions + // eg: DESC table$partitions -> partition_values(...) + if (dbTableName != null) { + dbTableName.analyze(analyzer); + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(dbTableName.getCtl()); + Pair sourceTableNameWithMetaName = catalog.getSourceTableNameWithMetaTableName( + dbTableName.getTbl()); + if (!Strings.isNullOrEmpty(sourceTableNameWithMetaName.second)) { + isTableValuedFunction = true; + Optional optTvfRef = catalog.getMetaTableFunctionRef( + dbTableName.getDb(), dbTableName.getTbl()); + if (!optTvfRef.isPresent()) { + throw new AnalysisException("meta table not found: " + sourceTableNameWithMetaName.second); + } + tableValuedFunctionRef = optTvfRef.get(); + } + } + if (!isAllTables && isTableValuedFunction) { tableValuedFunctionRef.analyze(analyzer); List columns = tableValuedFunctionRef.getTable().getBaseSchema(); @@ -148,8 +170,6 @@ public void analyze(Analyzer analyzer) throws UserException { } } - dbTableName.analyze(analyzer); - if (!Env.getCurrentEnv().getAccessManager() .checkTblPriv(ConnectContext.get(), dbTableName, PrivPredicate.SHOW)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "DESCRIBE", @@ -159,8 +179,7 @@ public void analyze(Analyzer analyzer) throws UserException { CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(dbTableName.getCtl()); DatabaseIf db = catalog.getDbOrAnalysisException(dbTableName.getDb()); - TableIf table = db.getTableOrAnalysisException(dbTableName.getTbl()); - + TableIf table = db.getTableOrDdlException(dbTableName.getTbl()); table.readLock(); try { if (!isAllTables) { @@ -387,3 +406,4 @@ private static List initEmptyRow() { return emptyRow; } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java index db66c260e5670c..88c9816209365a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos; import org.apache.doris.nereids.trees.expressions.functions.table.Numbers; +import org.apache.doris.nereids.trees.expressions.functions.table.PartitionValues; import org.apache.doris.nereids.trees.expressions.functions.table.Partitions; import org.apache.doris.nereids.trees.expressions.functions.table.Query; import org.apache.doris.nereids.trees.expressions.functions.table.S3; @@ -59,7 +60,8 @@ public class BuiltinTableValuedFunctions implements FunctionHelper { tableValued(Partitions.class, "partitions"), tableValued(Jobs.class, "jobs"), tableValued(Tasks.class, "tasks"), - tableValued(Query.class, "query") + tableValued(Query.class, "query"), + tableValued(PartitionValues.class, "partition_values") ); public static final BuiltinTableValuedFunctions INSTANCE = new BuiltinTableValuedFunctions(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index 1c03dad4883a98..d42a32ef8d2e1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -66,8 +66,6 @@ default boolean tryReadLock(long timeout, TimeUnit unit) { default void readUnlock() { } - ; - default void writeLock() { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java index 55b2ed1c58d972..e7066846c30919 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -340,4 +340,39 @@ private static String formatDateStr(String dateStr) { parts.length > 3 ? String.format(" %02d:%02d:%02d", Integer.parseInt(parts[3]), Integer.parseInt(parts[4]), Integer.parseInt(parts[5])) : ""); } + + + // Refer to be/src/vec/runtime/vdatetime_value.h + public static long convertToDateTimeV2( + int year, int month, int day, int hour, int minute, int second, int microsecond) { + return (long) microsecond | (long) second << 20 | (long) minute << 26 | (long) hour << 32 + | (long) day << 37 | (long) month << 42 | (long) year << 46; + } + + // Refer to be/src/vec/runtime/vdatetime_value.h + public static long convertToDateV2( + int year, int month, int day) { + return (long) day | (long) month << 5 | (long) year << 9; + } + + public static long convertStringToDateTimeV2(String dateTimeStr, int scale) { + String format = "yyyy-MM-dd HH:mm:ss"; + if (scale > 0) { + format += "."; + for (int i = 0; i < scale; i++) { + format += "S"; + } + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(format); + LocalDateTime dateTime = TimeUtils.formatDateTimeAndFullZero(dateTimeStr, formatter); + return convertToDateTimeV2(dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth(), + dateTime.getHour(), dateTime.getMinute(), dateTime.getSecond(), dateTime.getNano() / 1000); + } + + public static long convertStringToDateV2(String dateStr) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + LocalDateTime dateTime = TimeUtils.formatDateTimeAndFullZero(dateStr, formatter); + return convertToDateV2(dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth()); + } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index a79897c67df3eb..41cb44ef0b5fa3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.DropDbStmt; import org.apache.doris.analysis.DropTableStmt; import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.TableValuedFunctionRef; import org.apache.doris.analysis.TruncateTableStmt; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; @@ -30,7 +31,9 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -197,4 +200,24 @@ default CatalogLog constructEditLog() { void dropTable(DropTableStmt stmt) throws DdlException; void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlException; + + /** + * Try to parse meta table name from table name. + * Some catalog allow querying meta table like "table_name$partitions". + * Catalog can override this method to parse meta table name from table name. + * + * @param tableName table name like "table_name" or "table_name$partitions" + * @return pair of source table name and meta table name + */ + default Pair getSourceTableNameWithMetaTableName(String tableName) { + return Pair.of(tableName, ""); + } + + default Optional getMetaTableFunction(String dbName, String sourceNameWithMetaName) { + return Optional.empty(); + } + + default Optional getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) { + return Optional.empty(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 5faf1f2bb6e723..5173b414b8acc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -17,11 +17,13 @@ package org.apache.doris.datasource.hive; +import org.apache.doris.analysis.TableValuedFunctionRef; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.HdfsResource; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.security.authentication.AuthenticationConfig; import org.apache.doris.common.security.authentication.HadoopAuthenticator; @@ -39,10 +41,15 @@ import org.apache.doris.fs.FileSystemProvider; import org.apache.doris.fs.FileSystemProviderImpl; import org.apache.doris.fs.remote.dfs.DFSFileSystem; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.functions.table.PartitionValues; +import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.transaction.TransactionManagerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import lombok.Getter; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.hive.conf.HiveConf; @@ -52,6 +59,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ThreadPoolExecutor; /** @@ -277,6 +285,36 @@ public void setDefaultPropsIfMissing(boolean isReplay) { } } + @Override + public Pair getSourceTableNameWithMetaTableName(String tableName) { + for (MetaTableFunction metaFunction : MetaTableFunction.values()) { + if (metaFunction.containsMetaTable(tableName)) { + return Pair.of(metaFunction.getSourceTableName(tableName), metaFunction.name().toLowerCase()); + } + } + return Pair.of(tableName, ""); + } + + @Override + public Optional getMetaTableFunction(String dbName, String sourceNameWithMetaName) { + for (MetaTableFunction metaFunction : MetaTableFunction.values()) { + if (metaFunction.containsMetaTable(sourceNameWithMetaName)) { + return Optional.of(metaFunction.createFunction(name, dbName, sourceNameWithMetaName)); + } + } + return Optional.empty(); + } + + @Override + public Optional getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) { + for (MetaTableFunction metaFunction : MetaTableFunction.values()) { + if (metaFunction.containsMetaTable(sourceNameWithMetaName)) { + return Optional.of(metaFunction.createFunctionRef(name, dbName, sourceNameWithMetaName)); + } + } + return Optional.empty(); + } + public String getHiveMetastoreUris() { return catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, ""); } @@ -292,4 +330,58 @@ public int getHmsEventsBatchSizePerRpc() { public boolean isEnableHmsEventsIncrementalSync() { return enableHmsEventsIncrementalSync; } + + /** + * Enum for meta tables in hive catalog. + * eg: tbl$partitions + */ + private enum MetaTableFunction { + PARTITIONS("partition_values"); + + private final String suffix; + private final String tvfName; + + MetaTableFunction(String tvfName) { + this.suffix = "$" + name().toLowerCase(); + this.tvfName = tvfName; + } + + boolean containsMetaTable(String tableName) { + return tableName.endsWith(suffix) && (tableName.length() > suffix.length()); + } + + String getSourceTableName(String tableName) { + return tableName.substring(0, tableName.length() - suffix.length()); + } + + public TableValuedFunction createFunction(String ctlName, String dbName, String sourceNameWithMetaName) { + switch (this) { + case PARTITIONS: + List nameParts = Lists.newArrayList(ctlName, dbName, + getSourceTableName(sourceNameWithMetaName)); + return PartitionValues.create(nameParts); + default: + throw new AnalysisException("Unsupported meta function type: " + this); + } + } + + public TableValuedFunctionRef createFunctionRef(String ctlName, String dbName, String sourceNameWithMetaName) { + switch (this) { + case PARTITIONS: + Map params = Maps.newHashMap(); + params.put("catalog", ctlName); + params.put("database", dbName); + params.put("table", getSourceTableName(sourceNameWithMetaName)); + try { + return new TableValuedFunctionRef(tvfName, null, params); + } catch (org.apache.doris.common.AnalysisException e) { + LOG.warn("should not happen. {}.{}.{}", ctlName, dbName, sourceNameWithMetaName); + return null; + } + default: + throw new AnalysisException("Unsupported meta function type: " + this); + } + } + } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 7371f9766c02b5..a486e286c6ebd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -924,6 +924,6 @@ private List getFilesForPartitions( @Override public boolean isPartitionedTable() { makeSureInitialized(); - return remoteTable.getPartitionKeysSize() > 0; + return !isView() && remoteTable.getPartitionKeysSize() > 0; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index fae6ddb137b4d5..61521bd7681692 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1377,7 +1377,7 @@ protected LogicalPlan withTableAlias(LogicalPlan plan, TableAliasContext ctx) { @Override public LogicalPlan visitTableName(TableNameContext ctx) { - List tableId = visitMultipartIdentifier(ctx.multipartIdentifier()); + List nameParts = visitMultipartIdentifier(ctx.multipartIdentifier()); List partitionNames = new ArrayList<>(); boolean isTempPart = false; if (ctx.specifiedPartition() != null) { @@ -1425,8 +1425,9 @@ public LogicalPlan visitTableName(TableNameContext ctx) { TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample()); UnboundRelation relation = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), - tableId, partitionNames, isTempPart, tabletIdLists, relationHints, - Optional.ofNullable(tableSample), indexName, scanParams, Optional.ofNullable(tableSnapshot)); + nameParts, partitionNames, isTempPart, tabletIdLists, relationHints, + Optional.ofNullable(tableSample), indexName, scanParams, Optional.ofNullable(tableSnapshot)); + LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy(relation); LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias()); for (LateralViewContext lateralViewContext : ctx.lateralView()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 4e18039c4b12ae..b418a33bae0b56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -65,6 +65,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.functions.agg.QuantileUnion; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PreAggStatus; @@ -81,6 +82,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalTestScan; import org.apache.doris.nereids.trees.plans.logical.LogicalView; import org.apache.doris.nereids.util.RelationUtil; @@ -358,14 +360,32 @@ private Expression generateAggFunction(SlotReference slot, Column column) { } } + private Optional handleMetaTable(TableIf table, UnboundRelation unboundRelation, + List qualifiedTableName) { + Optional tvf = table.getDatabase().getCatalog().getMetaTableFunction( + qualifiedTableName.get(1), qualifiedTableName.get(2)); + if (tvf.isPresent()) { + return Optional.of(new LogicalTVFRelation(unboundRelation.getRelationId(), tvf.get())); + } + return Optional.empty(); + } + private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelation, - List qualifiedTableName, CascadesContext cascadesContext) { - // for create view stmt replace tablNname to ctl.db.tableName + List qualifiedTableName, CascadesContext cascadesContext) { + // for create view stmt replace tableName to ctl.db.tableName unboundRelation.getIndexInSqlString().ifPresent(pair -> { StatementContext statementContext = cascadesContext.getStatementContext(); statementContext.addIndexInSqlToString(pair, Utils.qualifiedNameWithBackquote(qualifiedTableName)); }); + + // Handle meta table like "table_name$partitions" + // qualifiedTableName should be like "ctl.db.tbl$partitions" + Optional logicalPlan = handleMetaTable(table, unboundRelation, qualifiedTableName); + if (logicalPlan.isPresent()) { + return logicalPlan.get(); + } + List qualifierWithoutTableName = Lists.newArrayList(); qualifierWithoutTableName.addAll(qualifiedTableName.subList(0, qualifiedTableName.size() - 1)); boolean isView = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/PartitionValues.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/PartitionValues.java new file mode 100644 index 00000000000000..0389bb626371aa --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/PartitionValues.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.table; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Properties; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.coercion.AnyDataType; +import org.apache.doris.tablefunction.PartitionValuesTableValuedFunction; +import org.apache.doris.tablefunction.TableValuedFunctionIf; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; + +/** + * partition_values + */ +public class PartitionValues extends TableValuedFunction { + public PartitionValues(Properties properties) { + super("partition_values", properties); + } + + /** + * Create PartitionValues function. + * @param qualifiedTableName ctl.db.tbl + * @return PartitionValues function + */ + public static TableValuedFunction create(List qualifiedTableName) { + Preconditions.checkArgument(qualifiedTableName != null && qualifiedTableName.size() == 3); + Map parameters = Maps.newHashMap(); + parameters.put(PartitionValuesTableValuedFunction.CATALOG, qualifiedTableName.get(0)); + parameters.put(PartitionValuesTableValuedFunction.DB, qualifiedTableName.get(1)); + parameters.put(PartitionValuesTableValuedFunction.TABLE, qualifiedTableName.get(2)); + return new PartitionValues(new Properties(parameters)); + } + + @Override + public FunctionSignature customSignature() { + return FunctionSignature.of(AnyDataType.INSTANCE_WITHOUT_INDEX, getArgumentsTypes()); + } + + @Override + protected TableValuedFunctionIf toCatalogFunction() { + try { + Map arguments = getTVFProperties().getMap(); + return new PartitionValuesTableValuedFunction(arguments); + } catch (Throwable t) { + throw new AnalysisException("Can not build PartitionsTableValuedFunction by " + + this + ": " + t.getMessage(), t); + } + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitPartitionValues(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java index ca14edd87def39..0b4b57e11dce16 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos; import org.apache.doris.nereids.trees.expressions.functions.table.Numbers; +import org.apache.doris.nereids.trees.expressions.functions.table.PartitionValues; import org.apache.doris.nereids.trees.expressions.functions.table.Partitions; import org.apache.doris.nereids.trees.expressions.functions.table.Query; import org.apache.doris.nereids.trees.expressions.functions.table.S3; @@ -59,6 +60,10 @@ default R visitPartitions(Partitions partitions, C context) { return visitTableValuedFunction(partitions, context); } + default R visitPartitionValues(PartitionValues partitionValues, C context) { + return visitTableValuedFunction(partitionValues, context); + } + default R visitJobs(Jobs jobs, C context) { return visitTableValuedFunction(jobs, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java index b145338ff81c48..b72498a2227a2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java @@ -100,8 +100,10 @@ public static Pair, TableIf> getDbAndTable(List qualifierN try { DatabaseIf db = catalog.getDb(dbName).orElseThrow(() -> new AnalysisException( "Database [" + dbName + "] does not exist.")); - TableIf table = db.getTable(tableName).orElseThrow(() -> new AnalysisException( - "Table [" + tableName + "] does not exist in database [" + dbName + "].")); + Pair sourceTblNameWithMetaTblName = catalog.getSourceTableNameWithMetaTableName(tableName); + String sourceTableName = sourceTblNameWithMetaTblName.first; + TableIf table = db.getTable(sourceTableName).orElseThrow(() -> new AnalysisException( + "Table [" + sourceTableName + "] does not exist in database [" + dbName + "].")); return Pair.of(db, table); } catch (Throwable e) { throw new AnalysisException(e.getMessage(), e.getCause()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 746519082aed73..ac0443bbf5283b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -30,10 +30,13 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.SchemaTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.TableProperty; +import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ClientPool; import org.apache.doris.common.Pair; @@ -42,11 +45,14 @@ import org.apache.doris.common.proc.PartitionsProcDir; import org.apache.doris.common.util.NetUtils; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveMetaStoreCache; import org.apache.doris.datasource.hudi.source.HudiCachedPartitionProcessor; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; @@ -78,6 +84,7 @@ import org.apache.doris.thrift.TMetadataTableRequestParams; import org.apache.doris.thrift.TMetadataType; import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPartitionValuesMetadataParams; import org.apache.doris.thrift.TPartitionsMetadataParams; import org.apache.doris.thrift.TPipelineWorkloadGroup; import org.apache.doris.thrift.TRow; @@ -204,7 +211,8 @@ public static TFetchSchemaTableDataResult getMetadataTable(TFetchSchemaTableData } TFetchSchemaTableDataResult result; TMetadataTableRequestParams params = request.getMetadaTableParams(); - switch (request.getMetadaTableParams().getMetadataType()) { + TMetadataType metadataType = request.getMetadaTableParams().getMetadataType(); + switch (metadataType) { case ICEBERG: result = icebergMetadataResult(params); break; @@ -232,11 +240,17 @@ public static TFetchSchemaTableDataResult getMetadataTable(TFetchSchemaTableData case TASKS: result = taskMetadataResult(params); break; + case PARTITION_VALUES: + result = partitionValuesMetadataResult(params); + break; default: return errorResult("Metadata table params is not set."); } if (result.getStatus().getStatusCode() == TStatusCode.OK) { - filterColumns(result, params.getColumnsName(), params.getMetadataType(), params); + if (metadataType != TMetadataType.PARTITION_VALUES) { + // partition_values' result already sorted by column names + filterColumns(result, params.getColumnsName(), params.getMetadataType(), params); + } } if (LOG.isDebugEnabled()) { LOG.debug("getMetadataTable() end."); @@ -329,7 +343,8 @@ private static TFetchSchemaTableDataResult icebergMetadataResult(TMetadataTableR TRow trow = new TRow(); LocalDateTime committedAt = LocalDateTime.ofInstant(Instant.ofEpochMilli( snapshot.timestampMillis()), TimeUtils.getTimeZone().toZoneId()); - long encodedDatetime = convertToDateTimeV2(committedAt.getYear(), committedAt.getMonthValue(), + long encodedDatetime = TimeUtils.convertToDateTimeV2(committedAt.getYear(), + committedAt.getMonthValue(), committedAt.getDayOfMonth(), committedAt.getHour(), committedAt.getMinute(), committedAt.getSecond(), committedAt.getNano() / 1000); @@ -785,12 +800,6 @@ private static void filterColumns(TFetchSchemaTableDataResult result, result.setDataBatch(filterColumnsRows); } - private static long convertToDateTimeV2( - int year, int month, int day, int hour, int minute, int second, int microsecond) { - return (long) microsecond | (long) second << 20 | (long) minute << 26 | (long) hour << 32 - | (long) day << 37 | (long) month << 42 | (long) year << 46; - } - private static TFetchSchemaTableDataResult mtmvMetadataResult(TMetadataTableRequestParams params) { if (LOG.isDebugEnabled()) { LOG.debug("mtmvMetadataResult() start"); @@ -1477,4 +1486,120 @@ private static void fillBatch(List dataBatch, Map dataBatch; + try { + TableIf table = PartitionValuesTableValuedFunction.analyzeAndGetTable(ctlName, dbName, tblName, false); + TableType tableType = table.getType(); + switch (tableType) { + case HMS_EXTERNAL_TABLE: + dataBatch = partitionValuesMetadataResultForHmsTable((HMSExternalTable) table, + params.getColumnsName()); + break; + default: + return errorResult("not support table type " + tableType.name()); + } + TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); + result.setDataBatch(dataBatch); + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } catch (Throwable t) { + LOG.warn("error when get partition values metadata. {}.{}.{}", ctlName, dbName, tblName, t); + return errorResult("error when get partition values metadata: " + Util.getRootCauseMessage(t)); + } + } + + private static List partitionValuesMetadataResultForHmsTable(HMSExternalTable tbl, List colNames) + throws AnalysisException { + List partitionCols = tbl.getPartitionColumns(); + List colIdxs = Lists.newArrayList(); + List types = Lists.newArrayList(); + for (String colName : colNames) { + for (int i = 0; i < partitionCols.size(); ++i) { + if (partitionCols.get(i).getName().equalsIgnoreCase(colName)) { + colIdxs.add(i); + types.add(partitionCols.get(i).getType()); + } + } + } + if (colIdxs.size() != colNames.size()) { + throw new AnalysisException( + "column " + colNames + " does not match partition columns of table " + tbl.getName()); + } + + HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .getMetaStoreCache((HMSExternalCatalog) tbl.getCatalog()); + HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( + tbl.getDbName(), tbl.getName(), tbl.getPartitionColumnTypes()); + Map> valuesMap = hivePartitionValues.getPartitionValuesMap(); + List dataBatch = Lists.newArrayList(); + for (Map.Entry> entry : valuesMap.entrySet()) { + TRow trow = new TRow(); + List values = entry.getValue(); + if (values.size() != partitionCols.size()) { + continue; + } + + for (int i = 0; i < colIdxs.size(); ++i) { + int idx = colIdxs.get(i); + String partitionValue = values.get(idx); + if (partitionValue == null || partitionValue.equals(TablePartitionValues.HIVE_DEFAULT_PARTITION)) { + trow.addToColumnValue(new TCell().setIsNull(true)); + } else { + Type type = types.get(i); + switch (type.getPrimitiveType()) { + case BOOLEAN: + trow.addToColumnValue(new TCell().setBoolVal(Boolean.valueOf(partitionValue))); + break; + case TINYINT: + case SMALLINT: + case INT: + trow.addToColumnValue(new TCell().setIntVal(Integer.valueOf(partitionValue))); + break; + case BIGINT: + trow.addToColumnValue(new TCell().setLongVal(Long.valueOf(partitionValue))); + break; + case FLOAT: + trow.addToColumnValue(new TCell().setDoubleVal(Float.valueOf(partitionValue))); + break; + case DOUBLE: + trow.addToColumnValue(new TCell().setDoubleVal(Double.valueOf(partitionValue))); + break; + case VARCHAR: + case CHAR: + case STRING: + trow.addToColumnValue(new TCell().setStringVal(partitionValue)); + break; + case DATE: + case DATEV2: + trow.addToColumnValue( + new TCell().setLongVal(TimeUtils.convertStringToDateV2(partitionValue))); + break; + case DATETIME: + case DATETIMEV2: + trow.addToColumnValue( + new TCell().setLongVal(TimeUtils.convertStringToDateTimeV2(partitionValue, + ((ScalarType) type).getScalarScale()))); + break; + default: + throw new AnalysisException( + "Unsupported partition column type for $partitions sys table " + type); + } + } + } + dataBatch.add(trow); + } + return dataBatch; + } + } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionValuesTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionValuesTableValuedFunction.java new file mode 100644 index 00000000000000..59efe9fcefde21 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionValuesTableValuedFunction.java @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TMetaScanRange; +import org.apache.doris.thrift.TMetadataType; +import org.apache.doris.thrift.TPartitionValuesMetadataParams; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The Implement of table valued function + * partition_values("catalog"="ctl1", "database" = "db1","table" = "table1"). + */ +public class PartitionValuesTableValuedFunction extends MetadataTableValuedFunction { + private static final Logger LOG = LogManager.getLogger(PartitionValuesTableValuedFunction.class); + + public static final String NAME = "partition_values"; + + public static final String CATALOG = "catalog"; + public static final String DB = "database"; + public static final String TABLE = "table"; + + private static final ImmutableSet PROPERTIES_SET = ImmutableSet.of(CATALOG, DB, TABLE); + + private final String catalogName; + private final String databaseName; + private final String tableName; + private TableIf table; + private List schema; + + public PartitionValuesTableValuedFunction(Map params) throws AnalysisException { + Map validParams = Maps.newHashMap(); + for (String key : params.keySet()) { + if (!PROPERTIES_SET.contains(key.toLowerCase())) { + throw new AnalysisException("'" + key + "' is invalid property"); + } + // check ctl, db, tbl + validParams.put(key.toLowerCase(), params.get(key)); + } + String catalogName = validParams.get(CATALOG); + String dbName = validParams.get(DB); + String tableName = validParams.get(TABLE); + if (StringUtils.isEmpty(catalogName) || StringUtils.isEmpty(dbName) || StringUtils.isEmpty(tableName)) { + throw new AnalysisException("catalog, database and table are required"); + } + this.table = analyzeAndGetTable(catalogName, dbName, tableName, true); + this.catalogName = catalogName; + this.databaseName = dbName; + this.tableName = tableName; + if (LOG.isDebugEnabled()) { + LOG.debug("PartitionsTableValuedFunction() end"); + } + } + + public static TableIf analyzeAndGetTable(String catalogName, String dbName, String tableName, boolean checkAuth) { + if (checkAuth) { + // This method will be called at 2 places: + // One is when planing the query, which should check the privilege of the user. + // the other is when BE call FE to fetch partition values, which should not check the privilege. + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ConnectContext.get(), catalogName, dbName, + tableName, PrivPredicate.SHOW)) { + String message = ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("SHOW PARTITIONS", + ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), + catalogName + ": " + dbName + ": " + tableName); + throw new AnalysisException(message); + } + } + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName); + if (catalog == null) { + throw new AnalysisException("can not find catalog: " + catalogName); + } + // disallow unsupported catalog + if (!(catalog.isInternalCatalog() || catalog instanceof HMSExternalCatalog + || catalog instanceof MaxComputeExternalCatalog)) { + throw new AnalysisException(String.format("Catalog of type '%s' is not allowed in ShowPartitionsStmt", + catalog.getType())); + } + + Optional db = catalog.getDb(dbName); + if (!db.isPresent()) { + throw new AnalysisException("can not find database: " + dbName); + } + TableIf table; + try { + table = db.get().getTableOrMetaException(tableName, TableType.OLAP, + TableType.HMS_EXTERNAL_TABLE, TableType.MAX_COMPUTE_EXTERNAL_TABLE); + } catch (MetaNotFoundException e) { + throw new AnalysisException(e.getMessage(), e); + } + + if (!(table instanceof HMSExternalTable)) { + throw new AnalysisException("Currently only support hive table's partition values meta table"); + } + HMSExternalTable hmsTable = (HMSExternalTable) table; + if (!hmsTable.isPartitionedTable()) { + throw new AnalysisException("Table " + tableName + " is not a partitioned table"); + } + return table; + } + + @Override + public TMetadataType getMetadataType() { + return TMetadataType.PARTITION_VALUES; + } + + @Override + public TMetaScanRange getMetaScanRange() { + if (LOG.isDebugEnabled()) { + LOG.debug("getMetaScanRange() start"); + } + TMetaScanRange metaScanRange = new TMetaScanRange(); + metaScanRange.setMetadataType(TMetadataType.PARTITION_VALUES); + TPartitionValuesMetadataParams partitionParam = new TPartitionValuesMetadataParams(); + partitionParam.setCatalog(catalogName); + partitionParam.setDatabase(databaseName); + partitionParam.setTable(tableName); + metaScanRange.setPartitionValuesParams(partitionParam); + return metaScanRange; + } + + @Override + public String getTableName() { + return "PartitionsValuesTableValuedFunction"; + } + + @Override + public List getTableColumns() throws AnalysisException { + Preconditions.checkNotNull(table); + // TODO: support other type of sys tables + if (schema == null) { + List partitionColumns = ((HMSExternalTable) table).getPartitionColumns(); + schema = Lists.newArrayList(); + for (Column column : partitionColumns) { + schema.add(new Column(column)); + } + } + return schema; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java index 1ceddeb89cf4a8..00169ad555f93e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java @@ -55,7 +55,7 @@ /** * The Implement of table valued function - * partitions("database" = "db1","table" = "table1"). + * partitions("catalog"="ctl1", "database" = "db1","table" = "table1"). */ public class PartitionsTableValuedFunction extends MetadataTableValuedFunction { private static final Logger LOG = LogManager.getLogger(PartitionsTableValuedFunction.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java index 6b6fda088a8a04..d4faa46019541c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java @@ -77,6 +77,8 @@ public static TableValuedFunctionIf getTableFunction(String funcName, Map TimeUtils.convertStringToDateV2("2021-1-1")); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateV2("1900-01-1")); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateV2("20210101")); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateV2("")); + ExceptionChecker.expectThrows(NullPointerException.class, () -> TimeUtils.convertStringToDateV2(null)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateV2("2024:12:31")); + } + + @Test + public void testConvertToBEDatetimeV2Type() { + long result = TimeUtils.convertStringToDateTimeV2("2021-01-01 10:10:10", 0); + Assert.assertEquals(142219811099770880L, result); + result = TimeUtils.convertStringToDateTimeV2("1900-01-01 00:00:00.12", 2); + Assert.assertEquals(133705149423146176L, result); + result = TimeUtils.convertStringToDateTimeV2("1899-12-31 23:59:59.000", 3); + Assert.assertEquals(133687385164611584L, result); + result = TimeUtils.convertStringToDateTimeV2("9999-12-31 23:59:59.123456", 6); + Assert.assertEquals(703674213003812984L, result); + + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2021-1-1", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("1900-01-1", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("20210101", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, () -> TimeUtils.convertStringToDateTimeV2("", 0)); + ExceptionChecker.expectThrows(NullPointerException.class, () -> TimeUtils.convertStringToDateTimeV2(null, 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2024-10-10", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2024-10-10 10", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2024:12:31", 0)); + ExceptionChecker.expectThrows(DateTimeParseException.class, + () -> TimeUtils.convertStringToDateTimeV2("2024-10-10 11:11:11", 6)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java index 67115e676871b4..369a57017cba28 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java @@ -18,6 +18,8 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PartitionInfo; @@ -98,6 +100,8 @@ public void bindExternalRelation() { new Column("name", Type.VARCHAR) ); + Database externalDatabase = new Database(10000, DEFAULT_CLUSTER_PREFIX + DB1); + OlapTable externalOlapTable = new OlapTable(1, tableName, externalTableColumns, KeysType.DUP_KEYS, new PartitionInfo(), new RandomDistributionInfo(10)) { @Override @@ -109,6 +113,11 @@ public List getBaseSchema(boolean full) { public boolean hasDeleteSign() { return false; } + + @Override + public DatabaseIf getDatabase() { + return externalDatabase; + } }; CustomTableResolver customTableResolver = qualifiedTable -> { diff --git a/gensrc/thrift/Data.thrift b/gensrc/thrift/Data.thrift index d1163821076d8a..dc1190c6e42559 100644 --- a/gensrc/thrift/Data.thrift +++ b/gensrc/thrift/Data.thrift @@ -54,6 +54,7 @@ struct TCell { 3: optional i64 longVal 4: optional double doubleVal 5: optional string stringVal + 6: optional bool isNull // add type: date datetime } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index dac90824fa2fac..ad3828e392e438 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -981,6 +981,7 @@ struct TMetadataTableRequestParams { 10: optional PlanNodes.TTasksMetadataParams tasks_metadata_params 11: optional PlanNodes.TPartitionsMetadataParams partitions_metadata_params 12: optional PlanNodes.TMetaCacheStatsParams meta_cache_stats_params + 13: optional PlanNodes.TPartitionValuesMetadataParams partition_values_metadata_params } struct TSchemaTableRequestParams { diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index d6690fe77a12f7..6e328a93e157d3 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -509,6 +509,12 @@ struct TPartitionsMetadataParams { 3: optional string table } +struct TPartitionValuesMetadataParams { + 1: optional string catalog + 2: optional string database + 3: optional string table +} + struct TJobsMetadataParams { 1: optional string type 2: optional Types.TUserIdentity current_user_ident @@ -526,6 +532,7 @@ struct TQueriesMetadataParams { 4: optional TJobsMetadataParams jobs_params 5: optional TTasksMetadataParams tasks_params 6: optional TPartitionsMetadataParams partitions_params + 7: optional TPartitionValuesMetadataParams partition_values_params } struct TMetaCacheStatsParams { @@ -542,6 +549,7 @@ struct TMetaScanRange { 8: optional TTasksMetadataParams tasks_params 9: optional TPartitionsMetadataParams partitions_params 10: optional TMetaCacheStatsParams meta_cache_stats_params + 11: optional TPartitionValuesMetadataParams partition_values_params } // Specification of an individual data range which is held in its entirety diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 9290768e0dbe65..fd37adb86549cc 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -712,7 +712,8 @@ enum TMetadataType { JOBS, TASKS, WORKLOAD_SCHED_POLICY, - PARTITIONS; + PARTITIONS, + PARTITION_VALUES; } enum TIcebergQueryType { diff --git a/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out b/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out new file mode 100644 index 00000000000000..bc69c716277ccd --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out @@ -0,0 +1,120 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql01 -- +\N 0.1 test1 +\N 0.2 test2 +100 0.3 test3 + +-- !sql02 -- +\N 0.1 test1 +\N 0.2 test2 +100 0.3 test3 + +-- !sql03 -- +\N 0.1 test1 +\N 0.2 test2 +100 0.3 test3 + +-- !sql11 -- +\N 0.1 +\N 0.2 +100 0.3 + +-- !sql12 -- +0.1 \N +0.2 \N +0.3 100 + +-- !sql13 -- +test1 \N +test2 \N +test3 100 + +-- !sql21 -- +test3 100 0.3 + +-- !sql22 -- +test1 +test2 +test3 + +-- !sql22 -- +3 + +-- !sql22 -- +3 + +-- !sql31 -- +0.1 \N +0.2 \N +0.3 100 + +-- !sql41 -- +100 + +-- !sql42 -- +test3 + +-- !sql51 -- +test3 + +-- !sql61 -- + +-- !sql71 -- +\N 0.2 test2 +100 0.3 test3 +\N 0.1 test1 + +-- !sql72 -- + +-- !sql73 -- +test3 + +-- !sql81 -- +100 0.3 test3 100 0.3 test3 + +-- !sql91 -- +t_int int Yes true \N NONE +t_float float Yes true \N NONE +t_string varchar(65533) Yes true \N NONE + +-- !sql92 -- +t_int int Yes true \N NONE +t_float float Yes true \N NONE +t_string varchar(65533) Yes true \N NONE + +-- !sql93 -- +t_int int Yes true \N NONE + +-- !sql94 -- +t_int int Yes true \N NONE + +-- !sql95 -- +\N 0.1 test1 +\N 0.2 test2 +100 0.3 test3 + +-- !sql101 -- +k1 int Yes true \N + +-- !sql102 -- + +-- !sql111 -- +p1 boolean Yes true \N NONE +p2 tinyint Yes true \N NONE +p3 smallint Yes true \N NONE +p4 int Yes true \N NONE +p5 bigint Yes true \N NONE +p6 date Yes true \N NONE +p7 datetime(6) Yes true \N NONE +p8 varchar(65533) Yes true \N NONE + +-- !sql112 -- +1 test1 true -128 -32768 -2147483648 -9223372036854775808 1900-01-01 1899-01-01T23:59:59 \N +2 \N false 127 32767 2147483647 9223372036854775807 9999-12-31 0001-01-01T00:00:01.321 boston +3 \N \N \N \N \N \N \N \N + +-- !sql113 -- +\N \N \N \N \N \N \N \N +false -128 -32768 -2147483648 -9223372036854775808 1900-01-01 1899-01-01T23:59:59 \N +false 127 32767 2147483647 9223372036854775807 9999-12-31 0001-01-01T00:00:01.321 boston + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy b/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy new file mode 100644 index 00000000000000..7cb764d0165797 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_partition_values_tvf", "p0,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Hive test.") + return; + } + for (String hivePrefix : ["hive3"]) { + String extHiveHmsHost = context.config.otherConfigs.get("externalEnvIp") + String extHiveHmsPort = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = "${hivePrefix}_test_external_catalog_hive_partition" + + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + + // 1. test qualifier + qt_sql01 """ select * from ${catalog_name}.multi_catalog.orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + sql """ switch ${catalog_name} """ + qt_sql02 """ select * from multi_catalog.orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + sql """ use multi_catalog""" + qt_sql03 """ select * from orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + + // 2. test select order + qt_sql11 """ select * except(t_string) from orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + qt_sql12 """ select t_float, t_int from orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + qt_sql13 """ select t_string, t_int from orc_partitioned_columns\$partitions order by t_int, t_float, t_string""" + + // 3. test agg + qt_sql21 """ select max(t_string), max(t_int), max(t_float) from orc_partitioned_columns\$partitions""" + qt_sql22 """ select max(t_string) from orc_partitioned_columns\$partitions group by t_int, t_float order by t_int, t_float""" + qt_sql22 """ select count(*) from orc_partitioned_columns\$partitions;""" + qt_sql22 """ select count(1) from orc_partitioned_columns\$partitions;""" + + // 4. test alias + qt_sql31 """ select pv.t_float, pv.t_int from orc_partitioned_columns\$partitions as pv group by t_int, t_float order by t_int, t_float""" + + // 5. test CTE + qt_sql41 """ with v1 as (select t_string, t_int from orc_partitioned_columns\$partitions order by t_int, t_float, t_string) select max(t_int) from v1; """ + qt_sql42 """ with v1 as (select t_string, t_int from orc_partitioned_columns\$partitions order by t_int, t_float, t_string) select c1 from (select max(t_string) as c1 from v1) x; """ + + // 6. test subquery + qt_sql51 """select c1 from (select max(t_string) as c1 from (select * from multi_catalog.orc_partitioned_columns\$partitions)x)y;""" + + // 7. test where + qt_sql61 """select * from orc_partitioned_columns\$partitions where t_int != "__HIVE_DEFAULT_PARTITION__" order by t_int, t_float, t_string; """ + + // 8. test view + sql """drop database if exists internal.partition_values_db""" + sql """create database if not exists internal.partition_values_db""" + sql """create view internal.partition_values_db.v1 as select * from ${catalog_name}.multi_catalog.orc_partitioned_columns\$partitions""" + qt_sql71 """select * from internal.partition_values_db.v1""" + qt_sql72 """select t_string, t_int from internal.partition_values_db.v1 where t_int != "__HIVE_DEFAULT_PARTITION__"""" + qt_sql73 """with v1 as (select t_string, t_int from internal.partition_values_db.v1 order by t_int, t_float, t_string) select c1 from (select max(t_string) as c1 from v1) x;""" + + // 9. test join + qt_sql81 """select * from orc_partitioned_columns\$partitions p1 join orc_partitioned_columns\$partitions p2 on p1.t_int = p2.t_int order by p1.t_int, p1.t_float""" + + // 10. test desc + qt_sql91 """desc orc_partitioned_columns\$partitions""" + qt_sql92 """desc function partition_values("catalog" = "${catalog_name}", "database" = "multi_catalog", "table" = "orc_partitioned_columns");""" + qt_sql93 """desc orc_partitioned_one_column\$partitions""" + qt_sql94 """desc function partition_values("catalog" = "${catalog_name}", "database" = "multi_catalog", "table" = "orc_partitioned_one_column");""" + qt_sql95 """select * from partition_values("catalog" = "${catalog_name}", "database" = "multi_catalog", "table" = "orc_partitioned_columns") order by t_int, t_float""" + + // 11. test non partition table + test { + sql """select * from hive_text_complex_type\$partitions""" + exception "is not a partitioned table" + } + test { + sql """desc hive_text_complex_type\$partitions""" + exception "is not a partitioned table" + } + + // 12. test inner table + sql """create table internal.partition_values_db.pv_inner1 (k1 int) distributed by hash (k1) buckets 1 properties("replication_num" = "1")""" + qt_sql101 """desc internal.partition_values_db.pv_inner1""" + qt_sql102 """select * from internal.partition_values_db.pv_inner1""" + test { + sql """desc internal.partition_values_db.pv_inner1\$partitions""" + exception """Unknown table 'pv_inner1\$partitions'""" + } + + test { + sql """select * from internal.partition_values_db.pv_inner1\$partitions""" + exception """Table [pv_inner1\$partitions] does not exist in database [partition_values_db]""" + } + + // 13. test all types of partition columns + sql """switch ${catalog_name}""" + sql """drop database if exists partition_values_db"""; + sql """create database partition_values_db""" + sql """use partition_values_db""" + + sql """create table partition_values_all_types ( + k1 int, + k2 string, + p1 boolean, + p2 tinyint, + p3 smallint, + p4 int, + p5 bigint, + p6 date, + p7 datetime, + p8 string + ) partition by list(p1, p2, p3, p4, p5, p6, p7, p8)(); + """ + + qt_sql111 """desc partition_values_all_types\$partitions;""" + + sql """insert into partition_values_all_types values + (1, "test1", true, -128, -32768, -2147483648, -9223372036854775808, "1900-01-01", "1899-01-01 23:59:59", ""), + (2, null, false, 127, 32767, 2147483647, 9223372036854775807, "9999-12-31", "0001-01-01 00:00:01.321", "boston"), + (3, "", null, null, null, null, null, null, null, null); + """ + + qt_sql112 """select * from partition_values_all_types order by k1;""" + qt_sql113 """select * from partition_values_all_types\$partitions order by p1,p2,p3;""" + } +} +