From a07cd2652eb178d8bf44632ee9527336cb38a86e Mon Sep 17 00:00:00 2001 From: Vallish Date: Sat, 24 Aug 2024 12:30:25 +0000 Subject: [PATCH] [Enhancement] support information_schema.partitions --- .../schema_partitions_scanner.cpp | 134 +++++++++++-- .../schema_partitions_scanner.h | 15 +- .../schema_scanner/schema_scanner_helper.cpp | 45 +++++ .../schema_scanner/schema_scanner_helper.h | 19 ++ .../tablefunction/MetadataGenerator.java | 122 ++++++++++++ gensrc/thrift/FrontendService.thrift | 1 + .../system/test_partitions_schema.out | 31 +++ .../system/test_partitions_schema.groovy | 179 ++++++++++++++++++ 8 files changed, 525 insertions(+), 21 deletions(-) create mode 100644 be/src/exec/schema_scanner/schema_scanner_helper.cpp create mode 100644 be/src/exec/schema_scanner/schema_scanner_helper.h create mode 100644 regression-test/data/query_p0/system/test_partitions_schema.out create mode 100644 regression-test/suites/query_p0/system/test_partitions_schema.groovy diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp index ea7394e15e12d2f..46b76306651df51 100644 --- a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp @@ -22,10 +22,14 @@ #include #include "exec/schema_scanner/schema_helper.h" -#include "runtime/decimalv2_value.h" -#include "runtime/define_primitive_type.h" -#include "util/runtime_profile.h" +#include "runtime/client_cache.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "schema_scanner_helper.h" +#include "util/thrift_rpc_helper.h" #include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/data_types/data_type_factory.hpp" namespace doris { class RuntimeState; @@ -63,9 +67,7 @@ std::vector SchemaPartitionsScanner::_s_tbls_columns }; SchemaPartitionsScanner::SchemaPartitionsScanner() - : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_PARTITIONS), - _db_index(0), - _table_index(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_PARTITIONS) {} SchemaPartitionsScanner::~SchemaPartitionsScanner() {} @@ -75,21 +77,14 @@ Status SchemaPartitionsScanner::start(RuntimeState* state) { } SCOPED_TIMER(_get_db_timer); TGetDbsParams db_params; - if (nullptr != _param->common_param->db) { + if (_param->common_param->db) { db_params.__set_pattern(*(_param->common_param->db)); } - if (nullptr != _param->common_param->catalog) { + if (_param->common_param->catalog) { db_params.__set_catalog(*(_param->common_param->catalog)); } - if (nullptr != _param->common_param->current_user_ident) { + if (_param->common_param->current_user_ident) { db_params.__set_current_user_ident(*(_param->common_param->current_user_ident)); - } else { - if (nullptr != _param->common_param->user) { - db_params.__set_user(*(_param->common_param->user)); - } - if (nullptr != _param->common_param->user_ip) { - db_params.__set_user_ip(*(_param->common_param->user_ip)); - } } if (nullptr != _param->common_param->ip && 0 != _param->common_param->port) { @@ -98,17 +93,122 @@ Status SchemaPartitionsScanner::start(RuntimeState* state) { } else { return Status::InternalError("IP or port doesn't exists"); } + _block_rows_limit = state->batch_size(); + _rpc_timeout_ms = state->execution_timeout() * 1000; + return Status::OK(); +} + +Status SchemaPartitionsScanner::get_onedb_info_from_fe(int64_t dbId) { + TNetworkAddress master_addr = ExecEnv::GetInstance()->master_info()->network_address; + + TSchemaTableRequestParams schema_table_request_params; + for (int i = 0; i < _s_tbls_columns.size(); i++) { + schema_table_request_params.__isset.columns_name = true; + schema_table_request_params.columns_name.emplace_back(_s_tbls_columns[i].name); + } + schema_table_request_params.__set_current_user_ident(*_param->common_param->current_user_ident); + schema_table_request_params.__set_catalog(*_param->common_param->catalog); + schema_table_request_params.__set_dbId(dbId); + + TFetchSchemaTableDataRequest request; + request.__set_schema_table_name(TSchemaTableName::PARTITIONS); + request.__set_schema_table_params(schema_table_request_params); + + TFetchSchemaTableDataResult result; + + RETURN_IF_ERROR(ThriftRpcHelper::rpc( + master_addr.hostname, master_addr.port, + [&request, &result](FrontendServiceConnection& client) { + client->fetchSchemaTableData(result, request); + }, + _rpc_timeout_ms)); + + Status status(Status::create(result.status)); + if (!status.ok()) { + LOG(WARNING) << "fetch table options from FE failed, errmsg=" << status; + return status; + } + std::vector result_data = result.data_batch; + + _partitions_block = vectorized::Block::create_unique(); + for (int i = 0; i < _s_tbls_columns.size(); ++i) { + TypeDescriptor descriptor(_s_tbls_columns[i].type); + auto data_type = vectorized::DataTypeFactory::instance().create_data_type(descriptor, true); + _partitions_block->insert(vectorized::ColumnWithTypeAndName( + data_type->create_column(), data_type, _s_tbls_columns[i].name)); + } + _partitions_block->reserve(_block_rows_limit); + if (result_data.size() > 0) { + int col_size = result_data[0].column_value.size(); + if (col_size != _s_tbls_columns.size()) { + return Status::InternalError("table options schema is not match for FE and BE"); + } + } + + for (int i = 0; i < result_data.size(); i++) { + TRow row = result_data[i]; + + for (int j = 0; j < _s_tbls_columns.size(); j++) { + if ((_s_tbls_columns[j].type == TYPE_BIGINT) || _s_tbls_columns[j].type == TYPE_INT) { + SchemaScannerHelper::insert_int_value(j, row.column_value[j].longVal, + _partitions_block.get()); + } else if (_s_tbls_columns[j].type == TYPE_DATETIME) { + std::vector datas(1); + VecDateTimeValue src[1]; + src[0].from_date_str(row.column_value[j].stringVal.data(), + row.column_value[j].stringVal.size()); + datas[0] = src; + SchemaScannerHelper::insert_datetime_value(j, datas, _partitions_block.get()); + } else { + SchemaScannerHelper::insert_string_value(j, row.column_value[j].stringVal, + _partitions_block.get()); + } + } + } return Status::OK(); } +bool SchemaPartitionsScanner::check_and_mark_eos(bool* eos) const { + if (_row_idx == _total_rows) { + *eos = true; + if (_db_index < _db_result.db_ids.size()) { + *eos = false; + } + return true; + } + return false; +} + Status SchemaPartitionsScanner::get_next_block_internal(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - *eos = true; + + if ((_partitions_block == nullptr) || (_row_idx == _total_rows)) { + if (_db_index < _db_result.db_ids.size()) { + RETURN_IF_ERROR(get_onedb_info_from_fe(_db_result.db_ids[_db_index])); + _row_idx = 0; // reset row index so that it start filling for next block. + _total_rows = _partitions_block->rows(); + _db_index++; + } + } + + if (check_and_mark_eos(eos)) { + return Status::OK(); + } + + int current_batch_rows = std::min(_block_rows_limit, _total_rows - _row_idx); + vectorized::MutableBlock mblock = vectorized::MutableBlock::build_mutable_block(block); + RETURN_IF_ERROR(mblock.add_rows(_partitions_block.get(), _row_idx, current_batch_rows)); + _row_idx += current_batch_rows; + + if (!check_and_mark_eos(eos)) { + *eos = false; + } return Status::OK(); } diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.h b/be/src/exec/schema_scanner/schema_partitions_scanner.h index 87e55db984a3dee..3c246f36eecb931 100644 --- a/be/src/exec/schema_scanner/schema_partitions_scanner.h +++ b/be/src/exec/schema_scanner/schema_partitions_scanner.h @@ -40,11 +40,18 @@ class SchemaPartitionsScanner : public SchemaScanner { Status start(RuntimeState* state) override; Status get_next_block_internal(vectorized::Block* block, bool* eos) override; - int _db_index; - int _table_index; - TGetDbsResult _db_result; - TListTableStatusResult _table_result; static std::vector _s_tbls_columns; + +private: + Status get_onedb_info_from_fe(int64_t dbId); + bool check_and_mark_eos(bool* eos) const; + int _block_rows_limit = 4096; + int _db_index = 0; + TGetDbsResult _db_result; + int _row_idx = 0; + int _total_rows = 0; + std::unique_ptr _partitions_block = nullptr; + int _rpc_timeout_ms = 3000; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_scanner_helper.cpp b/be/src/exec/schema_scanner/schema_scanner_helper.cpp new file mode 100644 index 000000000000000..1012155fa61c275 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_scanner_helper.cpp @@ -0,0 +1,45 @@ +#include "schema_scanner_helper.h" + +#include "runtime/client_cache.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "util/thrift_rpc_helper.h" +#include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/data_types/data_type_factory.hpp" + +namespace doris { + +void SchemaScannerHelper::insert_string_value(int col_index, std::string str_val, + vectorized::Block* block) { + vectorized::MutableColumnPtr mutable_col_ptr; + mutable_col_ptr = std::move(*block->get_by_position(col_index).column).assume_mutable(); + auto* nullable_column = reinterpret_cast(mutable_col_ptr.get()); + vectorized::IColumn* col_ptr = &nullable_column->get_nested_column(); + reinterpret_cast(col_ptr)->insert_data(str_val.data(), + str_val.size()); + nullable_column->get_null_map_data().emplace_back(0); +} + +void SchemaScannerHelper::insert_datetime_value(int col_index, const std::vector& datas, + vectorized::Block* block) { + vectorized::MutableColumnPtr mutable_col_ptr; + mutable_col_ptr = std::move(*block->get_by_position(col_index).column).assume_mutable(); + auto* nullable_column = reinterpret_cast(mutable_col_ptr.get()); + vectorized::IColumn* col_ptr = &nullable_column->get_nested_column(); + auto data = datas[0]; + reinterpret_cast*>(col_ptr)->insert_data( + reinterpret_cast(data), 0); + nullable_column->get_null_map_data().emplace_back(0); +} + +void SchemaScannerHelper::insert_int_value(int col_index, int64_t int_val, + vectorized::Block* block) { + vectorized::MutableColumnPtr mutable_col_ptr; + mutable_col_ptr = std::move(*block->get_by_position(col_index).column).assume_mutable(); + auto* nullable_column = reinterpret_cast(mutable_col_ptr.get()); + vectorized::IColumn* col_ptr = &nullable_column->get_nested_column(); + reinterpret_cast*>(col_ptr)->insert_value(int_val); + nullable_column->get_null_map_data().emplace_back(0); +} +} // namespace doris diff --git a/be/src/exec/schema_scanner/schema_scanner_helper.h b/be/src/exec/schema_scanner/schema_scanner_helper.h new file mode 100644 index 000000000000000..1be1eb3095cd77a --- /dev/null +++ b/be/src/exec/schema_scanner/schema_scanner_helper.h @@ -0,0 +1,19 @@ +#ifndef _SCHEMA_SCANNER_HELPER_H_ +// this is a util class which can be used by all shema scanner +// all common functions are added in this class. +namespace doris { + +namespace vectorized { +class Block; +} // namespace vectorized +class SchemaScannerHelper { +public: + static void insert_string_value(int col_index, std::string str_val, vectorized::Block* block); + static void insert_datetime_value(int col_index, const std::vector& datas, + vectorized::Block* block); + + static void insert_int_value(int col_index, int64_t int_val, vectorized::Block* block); +}; + +} // namespace doris +#endif diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 6a122d6640b4b80..6b4075f95cd61c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -17,6 +17,7 @@ package org.apache.doris.tablefunction; +import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; @@ -27,6 +28,7 @@ import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.SchemaTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; @@ -95,6 +97,7 @@ import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.List; import java.util.Map; @@ -117,6 +120,8 @@ public class MetadataGenerator { private static final ImmutableMap TABLE_PROPERTIES_COLUMN_TO_INDEX; + private static final ImmutableMap PARTITIONS_COLUMN_TO_INDEX; + static { ImmutableMap.Builder activeQueriesbuilder = new ImmutableMap.Builder(); List activeQueriesColList = SchemaTable.TABLE_MAP.get("active_queries").getFullSchema(); @@ -164,6 +169,13 @@ public class MetadataGenerator { propertiesBuilder.put(propertiesColList.get(i).getName().toLowerCase(), i); } TABLE_PROPERTIES_COLUMN_TO_INDEX = propertiesBuilder.build(); + + ImmutableMap.Builder partitionsBuilder = new ImmutableMap.Builder(); + List partitionsColList = SchemaTable.TABLE_MAP.get("partitions").getFullSchema(); + for (int i = 0; i < partitionsColList.size(); i++) { + partitionsBuilder.put(partitionsColList.get(i).getName().toLowerCase(), i); + } + PARTITIONS_COLUMN_TO_INDEX = partitionsBuilder.build(); } public static TFetchSchemaTableDataResult getMetadataTable(TFetchSchemaTableDataRequest request) throws TException { @@ -255,6 +267,10 @@ public static TFetchSchemaTableDataResult getSchemaTableData(TFetchSchemaTableDa result = tablePropertiesMetadataResult(schemaTableParams); columnIndex = TABLE_PROPERTIES_COLUMN_TO_INDEX; break; + case PARTITIONS: + result = partitionsMetadataResult(schemaTableParams); + columnIndex = PARTITIONS_COLUMN_TO_INDEX; + break; default: return errorResult("invalid schema table name."); } @@ -1250,4 +1266,110 @@ private static TFetchSchemaTableDataResult tablePropertiesMetadataResult(TSchema result.setStatus(new TStatus(TStatusCode.OK)); return result; } + + private static void partitionsForInternalCatalog(UserIdentity currentUserIdentity, + CatalogIf catalog, DatabaseIf database, List tables, List dataBatch) { + for (TableIf table : tables) { + if (!(table instanceof OlapTable)) { + continue; + } + if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(currentUserIdentity, catalog.getName(), + database.getFullName(), table.getName(), PrivPredicate.SHOW)) { + continue; + } + + OlapTable olapTable = (OlapTable) table; + Collection allPartitions = olapTable.getAllPartitions(); + + for (Partition partition : allPartitions) { + TRow trow = new TRow(); + trow.addToColumnValue(new TCell().setStringVal(catalog.getName())); // TABLE_CATALOG + trow.addToColumnValue(new TCell().setStringVal(database.getFullName())); // TABLE_SCHEMA + trow.addToColumnValue(new TCell().setStringVal(table.getName())); // TABLE_NAME + trow.addToColumnValue(new TCell().setStringVal(partition.getName())); // PARTITION_NAME + trow.addToColumnValue(new TCell().setStringVal("")); // SUBPARTITION_NAME (not available) + + trow.addToColumnValue(new TCell().setIntVal(0)); //PARTITION_ORDINAL_POSITION (not available) + trow.addToColumnValue(new TCell().setIntVal(0)); //SUBPARTITION_ORDINAL_POSITION (not available) + trow.addToColumnValue(new TCell().setStringVal( + olapTable.getPartitionInfo().getType().toString())); // PARTITION_METHOD + trow.addToColumnValue(new TCell().setStringVal("")); // SUBPARTITION_METHOD + ArrayList expr = olapTable.getPartitionInfo().getPartitionExprs(); + String partExpr = (expr != null) ? olapTable.getPartitionInfo().getPartitionExprs().toString() : ""; + + trow.addToColumnValue(new TCell().setStringVal(partExpr)); // PARTITION_EXPRESSION (not available) + trow.addToColumnValue(new TCell().setStringVal("")); // SUBPARTITION_EXPRESSION (not available) + trow.addToColumnValue(new TCell().setStringVal( + olapTable.getPartitionInfo().getPartitionRangeString(partition.getId()))); + trow.addToColumnValue(new TCell().setIntVal(0)); //TABLE_ROWS (not available) + trow.addToColumnValue(new TCell().setIntVal(0)); //AVG_ROW_LENGTH (not available) + trow.addToColumnValue(new TCell().setIntVal(0)); //DATA_LENGTH (not available) + trow.addToColumnValue(new TCell().setIntVal(0)); //MAX_DATA_LENGTH (not available) + trow.addToColumnValue(new TCell().setIntVal(0)); //INDEX_LENGTH (not available) + trow.addToColumnValue(new TCell().setIntVal(0)); //DATA_FREE (not available) + trow.addToColumnValue(new TCell().setStringVal( + TimeUtils.longToTimeString(partition.getVisibleVersionTime()))); //CREATE_TIME + trow.addToColumnValue(new TCell().setStringVal("")); // UPDATE_TIME (not available) + trow.addToColumnValue(new TCell().setStringVal("")); // CHECK_TIME (not available) + trow.addToColumnValue(new TCell().setIntVal(0)); //CHECKSUM (not available) + trow.addToColumnValue(new TCell().setStringVal("")); // PARTITION_COMMENT (not available) + trow.addToColumnValue(new TCell().setStringVal("")); // NODEGROUP (not available) + trow.addToColumnValue(new TCell().setStringVal("")); // TABLESPACE_NAME (not available) + dataBatch.add(trow); + } + } // for table + } + + private static void partitionsForExternalCatalog(UserIdentity currentUserIdentity, + CatalogIf catalog, DatabaseIf database, List tables, List dataBatch) { + for (TableIf table : tables) { + if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(currentUserIdentity, catalog.getName(), + database.getFullName(), table.getName(), PrivPredicate.SHOW)) { + continue; + } + // TODO + } // for table + } + + private static TFetchSchemaTableDataResult partitionsMetadataResult(TSchemaTableRequestParams params) { + if (!params.isSetCurrentUserIdent()) { + return errorResult("current user ident is not set."); + } + + if (!params.isSetDbId()) { + return errorResult("current db id is not set."); + } + + if (!params.isSetCatalog()) { + return errorResult("current catalog is not set."); + } + + TUserIdentity tcurrentUserIdentity = params.getCurrentUserIdent(); + UserIdentity currentUserIdentity = UserIdentity.fromThrift(tcurrentUserIdentity); + TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); + Long dbId = params.getDbId(); + String clg = params.getCatalog(); + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(clg); + List dataBatch = Lists.newArrayList(); + DatabaseIf database = catalog.getDbNullable(dbId); + if (database == null) { + // BE gets the database id list from FE and then invokes this interface + // per database. there is a chance that in between database can be dropped. + // so need to handle database not exist case and return ok so that BE continue the + // loop with next database. + result.setDataBatch(dataBatch); + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } + List tables = database.getTables(); + if (catalog instanceof InternalCatalog) { + // only olap tables + partitionsForInternalCatalog(currentUserIdentity, catalog, database, tables, dataBatch); + } else if (catalog instanceof ExternalCatalog) { + partitionsForExternalCatalog(currentUserIdentity, catalog, database, tables, dataBatch); + } + result.setDataBatch(dataBatch); + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index b6e4aacf65614ae..0a8ad3e981b18c8 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1005,6 +1005,7 @@ enum TSchemaTableName { TABLE_OPTIONS = 6, WORKLOAD_GROUP_PRIVILEGES = 7, TABLE_PROPERTIES = 8, + PARTITIONS, } struct TMetadataTableRequestParams { diff --git a/regression-test/data/query_p0/system/test_partitions_schema.out b/regression-test/data/query_p0/system/test_partitions_schema.out new file mode 100644 index 000000000000000..38b247e7b6d0bc5 --- /dev/null +++ b/regression-test/data/query_p0/system/test_partitions_schema.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_check_1 -- +internal test_partitions_schema_db duplicate_table duplicate_table 0 0 UNPARTITIONED [] 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db listtable p_cn 0 0 LIST [`city`] partitionKeys: [types: [VARCHAR]; keys: [Beijing]; types: [VARCHAR]; keys: [Shanghai]; types: [VARCHAR]; keys: [Hong Kong]; ]; 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db listtable p_jp 0 0 LIST [`city`] partitionKeys: [types: [VARCHAR]; keys: [Tokyo]; ]; 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db listtable p_usa 0 0 LIST [`city`] partitionKeys: [types: [VARCHAR]; keys: [New York]; types: [VARCHAR]; keys: [San Francisco]; ]; 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db randomtable randomtable 0 0 UNPARTITIONED [] 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db rangetable p201701 0 0 RANGE [`date`] [types: [DATEV2]; keys: [0000-01-01]; ..types: [DATEV2]; keys: [2017-02-01]; ) 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db rangetable p201702 0 0 RANGE [`date`] [types: [DATEV2]; keys: [2017-02-01]; ..types: [DATEV2]; keys: [2017-03-01]; ) 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db rangetable p201703 0 0 RANGE [`date`] [types: [DATEV2]; keys: [2017-03-01]; ..types: [DATEV2]; keys: [2017-04-01]; ) 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db test_row_column_page_size1 test_row_column_page_size1 0 0 UNPARTITIONED [] 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db test_row_column_page_size2 test_row_column_page_size2 0 0 UNPARTITIONED [] 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 + +-- !select_check_2 -- +internal test_partitions_schema_db duplicate_table duplicate_table 0 0 UNPARTITIONED [] 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db listtable p_cn 0 0 LIST [`city`] partitionKeys: [types: [VARCHAR]; keys: [Beijing]; types: [VARCHAR]; keys: [Shanghai]; types: [VARCHAR]; keys: [Hong Kong]; ]; 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db listtable p_jp 0 0 LIST [`city`] partitionKeys: [types: [VARCHAR]; keys: [Tokyo]; ]; 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db listtable p_usa 0 0 LIST [`city`] partitionKeys: [types: [VARCHAR]; keys: [New York]; types: [VARCHAR]; keys: [San Francisco]; ]; 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db randomtable randomtable 0 0 UNPARTITIONED [] 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db rangetable p201701 0 0 RANGE [`date`] [types: [DATEV2]; keys: [0000-01-01]; ..types: [DATEV2]; keys: [2017-02-01]; ) 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db rangetable p201702 0 0 RANGE [`date`] [types: [DATEV2]; keys: [2017-02-01]; ..types: [DATEV2]; keys: [2017-03-01]; ) 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db rangetable p201703 0 0 RANGE [`date`] [types: [DATEV2]; keys: [2017-03-01]; ..types: [DATEV2]; keys: [2017-04-01]; ) 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 +internal test_partitions_schema_db test_row_column_page_size1 test_row_column_page_size1 0 0 UNPARTITIONED [] 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 + +-- !select_check_3 -- + +-- !select_check_4 -- +internal test_partitions_schema_db duplicate_table duplicate_table 0 0 UNPARTITIONED [] 0 0 0 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 0 + +-- !select_check_5 -- + diff --git a/regression-test/suites/query_p0/system/test_partitions_schema.groovy b/regression-test/suites/query_p0/system/test_partitions_schema.groovy new file mode 100644 index 000000000000000..1b488b9514a4401 --- /dev/null +++ b/regression-test/suites/query_p0/system/test_partitions_schema.groovy @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_partitions_schema") { + def dbName = "test_partitions_schema_db" + sql "drop database if exists ${dbName}" + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "use ${dbName}" + + sql """ + CREATE TABLE IF NOT EXISTS rangetable + ( + `user_id` LARGEINT NOT NULL COMMENT "User id", + `date` DATE NOT NULL COMMENT "Data fill in date time", + `timestamp` DATETIME NOT NULL COMMENT "Timestamp of data being poured", + `city` VARCHAR(20) COMMENT "The city where the user is located", + `age` SMALLINT COMMENT "User age", + `sex` TINYINT COMMENT "User gender", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "User last visit time", + `cost` BIGINT SUM DEFAULT "0" COMMENT "Total user consumption", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "User maximum dwell time", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "User minimum dwell time" + ) + ENGINE=olap + AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) + PARTITION BY RANGE(`date`) + ( + PARTITION `p201701` VALUES LESS THAN ("2017-02-01"), + PARTITION `p201702` VALUES LESS THAN ("2017-03-01"), + PARTITION `p201703` VALUES LESS THAN ("2017-04-01") + ) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 8 + PROPERTIES + ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE IF NOT EXISTS listtable + ( + `user_id` LARGEINT NOT NULL COMMENT "User id", + `date` DATE NOT NULL COMMENT "Data fill in date time", + `timestamp` DATETIME NOT NULL COMMENT "Timestamp of data being poured", + `city` VARCHAR(20) COMMENT "The city where the user is located", + `age` SMALLINT COMMENT "User Age", + `sex` TINYINT COMMENT "User gender", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "User last visit time", + `cost` BIGINT SUM DEFAULT "0" COMMENT "Total user consumption", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "User maximum dwell time", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "User minimum dwell time" + ) + ENGINE=olap + AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) + PARTITION BY LIST(`city`) + ( + PARTITION `p_cn` VALUES IN ("Beijing", "Shanghai", "Hong Kong"), + PARTITION `p_usa` VALUES IN ("New York", "San Francisco"), + PARTITION `p_jp` VALUES IN ("Tokyo") + ) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 + PROPERTIES + ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE IF NOT EXISTS randomtable + ( + `user_id` LARGEINT NOT NULL COMMENT "User id", + `date` DATE NOT NULL COMMENT "Data fill in date time", + `timestamp` DATETIME NOT NULL COMMENT "Timestamp of data being poured", + `city` VARCHAR(20) COMMENT "The city where the user is located", + `age` SMALLINT COMMENT "User Age", + `sex` TINYINT COMMENT "User gender" + ) + ENGINE=olap + DISTRIBUTED BY RANDOM BUCKETS 16 + PROPERTIES + ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE IF NOT EXISTS duplicate_table + ( + `timestamp` DATETIME NOT NULL COMMENT "Log time", + `type` INT NOT NULL COMMENT "Log type", + `error_code` INT COMMENT "Error code", + `error_msg` VARCHAR(1024) COMMENT "Error detail message", + `op_id` BIGINT COMMENT "Operator ID", + `op_time` DATETIME COMMENT "Operation time" + ) + DISTRIBUTED BY HASH(`type`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + // test row column page size + sql """ + CREATE TABLE IF NOT EXISTS test_row_column_page_size1 ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "store_row_column" = "true" + ); + """ + + sql """ + CREATE TABLE IF NOT EXISTS test_row_column_page_size2 ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "store_row_column" = "true", + "row_store_page_size" = "8190" + ); + """ + + qt_select_check_1 """select * from information_schema.partitions where table_schema=\"${dbName}\" order by TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,PARTITION_NAME""" + sql """ + drop table test_row_column_page_size2; + """ + qt_select_check_2 """select * from information_schema.partitions where table_schema=\"${dbName}\" order by TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,PARTITION_NAME """ + + def user = "partitions_user" + sql "DROP USER IF EXISTS ${user}" + sql "CREATE USER ${user} IDENTIFIED BY '123abc!@#'" + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + } + + sql "GRANT SELECT_PRIV ON information_schema.partitions TO ${user}" + + def tokens = context.config.jdbcUrl.split('/') + def url=tokens[0] + "//" + tokens[2] + "/" + "information_schema" + "?" + + connect(user=user, password='123abc!@#', url=url) { + qt_select_check_3 """select * from information_schema.partitions ORDER BY TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,PARTITION_NAME; """ + } + + sql "GRANT SELECT_PRIV ON ${dbName}.duplicate_table TO ${user}" + connect(user=user, password='123abc!@#', url=url) { + qt_select_check_4 """select * from information_schema.partitions ORDER BY TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,PARTITION_NAME; """ + } + + sql "REVOKE SELECT_PRIV ON ${dbName}.duplicate_table FROM ${user}" + connect(user=user, password='123abc!@#', url=url) { + qt_select_check_5 """select * from information_schema.partitions ORDER BY TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,PARTITION_NAME; """ + } + +}