From 0c5eebcb5283ac02df160e203bd0362a6dd1e9b3 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Thu, 10 Oct 2024 10:55:09 +0800 Subject: [PATCH] [Refactor](exec) refactor schema scanner del unless code --- be/src/exec/schema_scanner.cpp | 23 +-------- be/src/exec/schema_scanner.h | 12 ++--- .../schema_statistics_scanner.cpp | 50 ------------------- .../schema_statistics_scanner.h | 35 ------------- .../vec/exec/parquet/parquet_thrift_test.cpp | 1 - 5 files changed, 6 insertions(+), 115 deletions(-) delete mode 100644 be/src/exec/schema_scanner/schema_statistics_scanner.cpp delete mode 100644 be/src/exec/schema_scanner/schema_statistics_scanner.h diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 90140e748f5d6b..dead83e3d86d85 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -77,9 +77,6 @@ namespace doris { class ObjectPool; -SchemaScanner::SchemaScanner(const std::vector& columns) - : _is_init(false), _columns(columns), _schema_table_type(TSchemaTableType::SCH_INVALID) {} - SchemaScanner::SchemaScanner(const std::vector& columns, TSchemaTableType::type type) : _is_init(false), _columns(columns), _schema_table_type(type) {} @@ -125,7 +122,6 @@ Status SchemaScanner::get_next_block_async(RuntimeState* state) { return; } SCOPED_ATTACH_TASK(state); - _dependency->block(); _async_thread_running = true; _finish_dependency->block(); if (!_opened) { @@ -150,19 +146,6 @@ Status SchemaScanner::get_next_block_async(RuntimeState* state) { return Status::OK(); } -Status SchemaScanner::get_next_block_internal(vectorized::Block* block, bool* eos) { - if (!_is_init) { - return Status::InternalError("used before initialized."); - } - - if (nullptr == block || nullptr == eos) { - return Status::InternalError("input pointer is nullptr."); - } - - *eos = true; - return Status::OK(); -} - Status SchemaScanner::init(SchemaScannerParam* param, ObjectPool* pool) { if (_is_init) { return Status::OK(); @@ -426,21 +409,18 @@ Status SchemaScanner::insert_block_column(TCell cell, int col_index, vectorized: case TYPE_BIGINT: { reinterpret_cast*>(col_ptr)->insert_value( cell.longVal); - nullable_column->get_null_map_data().emplace_back(0); break; } case TYPE_INT: { reinterpret_cast*>(col_ptr)->insert_value( cell.intVal); - nullable_column->get_null_map_data().emplace_back(0); break; } case TYPE_BOOLEAN: { reinterpret_cast*>(col_ptr)->insert_value( cell.boolVal); - nullable_column->get_null_map_data().emplace_back(0); break; } @@ -449,7 +429,6 @@ Status SchemaScanner::insert_block_column(TCell cell, int col_index, vectorized: case TYPE_CHAR: { reinterpret_cast(col_ptr)->insert_data(cell.stringVal.data(), cell.stringVal.size()); - nullable_column->get_null_map_data().emplace_back(0); break; } @@ -461,7 +440,6 @@ Status SchemaScanner::insert_block_column(TCell cell, int col_index, vectorized: auto data = datas[0]; reinterpret_cast*>(col_ptr)->insert_data( reinterpret_cast(data), 0); - nullable_column->get_null_map_data().emplace_back(0); break; } default: { @@ -470,6 +448,7 @@ Status SchemaScanner::insert_block_column(TCell cell, int col_index, vectorized: return Status::InternalError(ss.str()); } } + nullable_column->get_null_map_data().emplace_back(0); return Status::OK(); } diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index da61d58b943fc4..440912bff1d729 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -19,10 +19,10 @@ #include #include -#include -#include #include +#include +#include #include #include #include @@ -82,8 +82,6 @@ struct SchemaScannerParam { // virtual scanner for all schema table class SchemaScanner { - ENABLE_FACTORY_CREATOR(SchemaScanner); - public: struct ColumnDesc { const char* name = nullptr; @@ -94,8 +92,8 @@ class SchemaScanner { int precision = -1; int scale = -1; }; - SchemaScanner(const std::vector& columns); - SchemaScanner(const std::vector& columns, TSchemaTableType::type type); + SchemaScanner(const std::vector& columns, + TSchemaTableType::type type = TSchemaTableType::SCH_INVALID); virtual ~SchemaScanner(); // init object need information, schema etc. @@ -103,7 +101,7 @@ class SchemaScanner { Status get_next_block(RuntimeState* state, vectorized::Block* block, bool* eos); // Start to work virtual Status start(RuntimeState* state); - virtual Status get_next_block_internal(vectorized::Block* block, bool* eos); + virtual Status get_next_block_internal(vectorized::Block* block, bool* eos) = 0; const std::vector& get_column_desc() const { return _columns; } // factory function static std::unique_ptr create(TSchemaTableType::type type); diff --git a/be/src/exec/schema_scanner/schema_statistics_scanner.cpp b/be/src/exec/schema_scanner/schema_statistics_scanner.cpp deleted file mode 100644 index f4f3d5dba83271..00000000000000 --- a/be/src/exec/schema_scanner/schema_statistics_scanner.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "exec/schema_scanner/schema_statistics_scanner.h" - -#include - -#include "runtime/define_primitive_type.h" -#include "vec/common/string_ref.h" - -namespace doris { - -std::vector SchemaStatisticsScanner::_s_cols_statistics = { - // name, type, size, is_null - {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, - {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), false}, - {"TABLE_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, - {"NON_UNIQUE", TYPE_BIGINT, sizeof(int64_t), false}, - {"INDEX_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), false}, - {"INDEX_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, - {"SEQ_IN_INDEX", TYPE_BIGINT, sizeof(int64_t), false}, - {"COLUMN_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, - {"COLLATION", TYPE_VARCHAR, sizeof(StringRef), true}, - {"CARDINALITY", TYPE_BIGINT, sizeof(int64_t), true}, - {"SUB_PART", TYPE_BIGINT, sizeof(int64_t), true}, - {"PACKED", TYPE_VARCHAR, sizeof(StringRef), true}, - {"NULLABLE", TYPE_VARCHAR, sizeof(StringRef), false}, - {"INDEX_TYPE", TYPE_VARCHAR, sizeof(StringRef), false}, - {"COMMENT", TYPE_VARCHAR, sizeof(StringRef), true}, -}; - -SchemaStatisticsScanner::SchemaStatisticsScanner() : SchemaScanner(_s_cols_statistics) {} - -SchemaStatisticsScanner::~SchemaStatisticsScanner() {} - -} // namespace doris diff --git a/be/src/exec/schema_scanner/schema_statistics_scanner.h b/be/src/exec/schema_scanner/schema_statistics_scanner.h deleted file mode 100644 index 1a756512abf18a..00000000000000 --- a/be/src/exec/schema_scanner/schema_statistics_scanner.h +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "exec/schema_scanner.h" - -namespace doris { -class SchemaStatisticsScanner : public SchemaScanner { - ENABLE_FACTORY_CREATOR(SchemaStatisticsScanner); - -public: - SchemaStatisticsScanner(); - ~SchemaStatisticsScanner() override; - -private: - static std::vector _s_cols_statistics; -}; -} // namespace doris diff --git a/be/test/vec/exec/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/parquet/parquet_thrift_test.cpp index 132de072127e14..fe2221bf8d3725 100644 --- a/be/test/vec/exec/parquet/parquet_thrift_test.cpp +++ b/be/test/vec/exec/parquet/parquet_thrift_test.cpp @@ -361,7 +361,6 @@ static void create_block(std::unique_ptr& block) { {"date_col", TYPE_DATEV2, sizeof(uint32_t), true}, {"date_v2_col", TYPE_DATEV2, sizeof(uint32_t), true}, {"timestamp_v2_col", TYPE_DATETIMEV2, sizeof(int128_t), true, 18, 0}}; - SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; doris::TupleDescriptor* tuple_desc = create_tuple_desc(&object_pool, column_descs); auto tuple_slots = tuple_desc->slots();