diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h index cbf8f1eca65ae2a..dd1e9528fa375c8 100644 --- a/be/src/olap/iterators.h +++ b/be/src/olap/iterators.h @@ -117,7 +117,7 @@ class StorageReadOptions { Version version; int64_t tablet_id = 0; // slots that cast may be eliminated in storage layer - std::map target_cast_type_for_variants; + std::map target_cast_type_for_variants; RowRanges row_ranges; size_t topn_limit = 0; }; diff --git a/be/src/olap/rowset/rowset_reader_context.h b/be/src/olap/rowset/rowset_reader_context.h index 59abf85fb720952..2de76df80401859 100644 --- a/be/src/olap/rowset/rowset_reader_context.h +++ b/be/src/olap/rowset/rowset_reader_context.h @@ -81,7 +81,7 @@ struct RowsetReaderContext { const std::set* output_columns = nullptr; RowsetId rowset_id; // slots that cast may be eliminated in storage layer - std::map target_cast_type_for_variants; + std::map target_cast_type_for_variants; int64_t ttl_seconds = 0; size_t topn_limit = 0; }; diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 2cec6f48f6bb697..07b56540a6544f4 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -1884,7 +1884,7 @@ bool SegmentIterator::_can_evaluated_by_vectorized(ColumnPredicate* predicate) { if (field_type == FieldType::OLAP_FIELD_TYPE_VARIANT) { // Use variant cast dst type field_type = TabletColumn::get_field_type_by_type( - _opts.target_cast_type_for_variants[_schema->column(cid)->name()]); + _opts.target_cast_type_for_variants[_schema->column(cid)->name()].type); } switch (predicate->type()) { case PredicateType::EQ: diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp index 631a041379a9afc..d158e6a5ac526ef 100644 --- a/be/src/olap/tablet_reader.cpp +++ b/be/src/olap/tablet_reader.cpp @@ -273,7 +273,8 @@ TabletColumn TabletReader::materialize_column(const TabletColumn& orig) { } TabletColumn column_with_cast_type = orig; auto cast_type = _reader_context.target_cast_type_for_variants.at(orig.name()); - column_with_cast_type.set_type(TabletColumn::get_field_type_by_type(cast_type)); + FieldType filed_type = TabletColumn::get_field_type_by_type(cast_type.type); + column_with_cast_type.set_type(filed_type); return column_with_cast_type; } diff --git a/be/src/olap/tablet_reader.h b/be/src/olap/tablet_reader.h index 06c3daa653a33be..18ebb9653ccde73 100644 --- a/be/src/olap/tablet_reader.h +++ b/be/src/olap/tablet_reader.h @@ -136,7 +136,7 @@ class TabletReader { std::vector function_filters; std::vector delete_predicates; // slots that cast may be eliminated in storage layer - std::map target_cast_type_for_variants; + std::map target_cast_type_for_variants; std::vector rs_splits; // For unique key table with merge-on-write diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp index 14b5ab2acd5ad64..623032507ecf526 100644 --- a/be/src/pipeline/exec/scan_operator.cpp +++ b/be/src/pipeline/exec/scan_operator.cpp @@ -30,6 +30,7 @@ #include "pipeline/exec/meta_scan_operator.h" #include "pipeline/exec/olap_scan_operator.h" #include "pipeline/exec/operator.h" +#include "runtime/types.h" #include "util/runtime_profile.h" #include "vec/exprs/vcast_expr.h" #include "vec/exprs/vcompound_pred.h" @@ -125,14 +126,14 @@ Status ScanLocalState::_normalize_conjuncts(RuntimeState* state) { // The conjuncts is always on output tuple, so use _output_tuple_desc; std::vector slots = p._output_tuple_desc->slots(); - auto init_value_range = [&](SlotDescriptor* slot, PrimitiveType type) { - switch (type) { -#define M(NAME) \ - case TYPE_##NAME: { \ - ColumnValueRange range(slot->col_name(), slot->is_nullable(), \ - slot->type().precision, slot->type().scale); \ - _slot_id_to_value_range[slot->id()] = std::pair {slot, range}; \ - break; \ + auto init_value_range = [&](SlotDescriptor* slot, const TypeDescriptor& type_desc) { + switch (type_desc.type) { +#define M(NAME) \ + case TYPE_##NAME: { \ + ColumnValueRange range(slot->col_name(), slot->is_nullable(), \ + type_desc.precision, type_desc.scale); \ + _slot_id_to_value_range[slot->id()] = std::pair {slot, range}; \ + break; \ } #define APPLY_FOR_PRIMITIVE_TYPE(M) \ M(TINYINT) \ @@ -173,7 +174,7 @@ Status ScanLocalState::_normalize_conjuncts(RuntimeState* state) { continue; } } - init_value_range(slot, slot->type().type); + init_value_range(slot, slot->type()); } get_cast_types_for_variants(); @@ -586,7 +587,7 @@ Status ScanLocalState::_normalize_in_and_eq_predicate(vectorized::VExpr ColumnValueRange& range, PushDownType* pdt) { auto temp_range = ColumnValueRange::create_empty_column_value_range( - slot->is_nullable(), slot->type().precision, slot->type().scale); + slot->is_nullable(), range.precision(), range.scale()); // 1. Normalize in conjuncts like 'where col in (v1, v2, v3)' if (TExprNodeType::IN_PRED == expr->node_type()) { HybridSetBase::IteratorBase* iter = nullptr; @@ -737,7 +738,7 @@ Status ScanLocalState::_normalize_not_in_and_not_eq_predicate( ColumnValueRange& range, PushDownType* pdt) { bool is_fixed_range = range.is_fixed_value_range(); auto not_in_range = ColumnValueRange::create_empty_column_value_range( - range.column_name(), slot->is_nullable(), slot->type().precision, slot->type().scale); + range.column_name(), slot->is_nullable(), range.precision(), range.scale()); PushDownType temp_pdt = PushDownType::UNACCEPTABLE; // 1. Normalize in conjuncts like 'where col in (v1, v2, v3)' if (TExprNodeType::IN_PRED == expr->node_type()) { @@ -917,14 +918,14 @@ Status ScanLocalState::_normalize_is_null_predicate(vectorized::VExpr* if (reinterpret_cast(expr)->fn().name.function_name == "is_null_pred") { auto temp_range = ColumnValueRange::create_empty_column_value_range( - slot->is_nullable(), slot->type().precision, slot->type().scale); + slot->is_nullable(), range.precision(), range.scale()); temp_range.set_contain_null(true); range.intersection(temp_range); *pdt = temp_pdt; } else if (reinterpret_cast(expr)->fn().name.function_name == "is_not_null_pred") { auto temp_range = ColumnValueRange::create_empty_column_value_range( - slot->is_nullable(), slot->type().precision, slot->type().scale); + slot->is_nullable(), range.precision(), range.scale()); temp_range.set_contain_null(false); range.intersection(temp_range); *pdt = temp_pdt; @@ -1161,7 +1162,7 @@ Status ScanLocalState::_normalize_match_predicate(vectorized::VExpr* ex // create empty range as temp range, temp range should do intersection on range auto temp_range = ColumnValueRange::create_empty_column_value_range( - slot->is_nullable(), slot->type().precision, slot->type().scale); + slot->is_nullable(), range.precision(), range.scale()); // Normalize match conjuncts like 'where col match value' auto match_checker = [](const std::string& fn_name) { return is_match_condition(fn_name); }; @@ -1324,7 +1325,7 @@ Status ScanLocalState::_get_topn_filters(RuntimeState* state) { template void ScanLocalState::_filter_and_collect_cast_type_for_variant( const vectorized::VExpr* expr, - phmap::flat_hash_map>& colname_to_cast_types) { + std::unordered_map>& colname_to_cast_types) { auto& p = _parent->cast(); const auto* cast_expr = dynamic_cast(expr); if (cast_expr != nullptr) { @@ -1337,10 +1338,9 @@ void ScanLocalState::_filter_and_collect_cast_type_for_variant( } std::vector slots = output_tuple_desc()->slots(); SlotDescriptor* src_slot_desc = p._slot_id_to_slot_desc[src_slot->slot_id()]; - PrimitiveType cast_dst_type = - cast_expr->get_target_type()->get_type_as_type_descriptor().type; + TypeDescriptor type_desc = cast_expr->get_target_type()->get_type_as_type_descriptor(); if (src_slot_desc->type().is_variant_type()) { - colname_to_cast_types[src_slot_desc->col_name()].push_back(cast_dst_type); + colname_to_cast_types[src_slot_desc->col_name()].push_back(type_desc); } } for (const auto& child : expr->children()) { @@ -1350,7 +1350,7 @@ void ScanLocalState::_filter_and_collect_cast_type_for_variant( template void ScanLocalState::get_cast_types_for_variants() { - phmap::flat_hash_map> colname_to_cast_types; + std::unordered_map> colname_to_cast_types; for (auto it = _conjuncts.begin(); it != _conjuncts.end();) { auto& conjunct = *it; if (conjunct->root()) { diff --git a/be/src/pipeline/exec/scan_operator.h b/be/src/pipeline/exec/scan_operator.h index 45b9a382e9b7d94..7502ffff9ab19bd 100644 --- a/be/src/pipeline/exec/scan_operator.h +++ b/be/src/pipeline/exec/scan_operator.h @@ -28,6 +28,7 @@ #include "pipeline/common/runtime_filter_consumer.h" #include "pipeline/dependency.h" #include "runtime/descriptors.h" +#include "runtime/types.h" #include "vec/exec/scan/vscan_node.h" #include "vec/exprs/vectorized_fn_call.h" #include "vec/exprs/vin_predicate.h" @@ -340,7 +341,7 @@ class ScanLocalState : public ScanLocalStateBase { void get_cast_types_for_variants(); void _filter_and_collect_cast_type_for_variant( const vectorized::VExpr* expr, - phmap::flat_hash_map>& colname_to_cast_types); + std::unordered_map>& colname_to_cast_types); Status _get_topn_filters(RuntimeState* state); @@ -357,7 +358,7 @@ class ScanLocalState : public ScanLocalStateBase { std::vector _push_down_functions; // colname -> cast dst type - std::map _cast_types_for_variants; + std::map _cast_types_for_variants; // slot id -> ColumnValueRange // Parsed from conjuncts diff --git a/regression-test/suites/variant_p0/sql/implicit_cast.sql b/regression-test/suites/variant_p0/sql/implicit_cast.sql index 45acfd3851313e7..416e2616ea07346 100644 --- a/regression-test/suites/variant_p0/sql/implicit_cast.sql +++ b/regression-test/suites/variant_p0/sql/implicit_cast.sql @@ -12,4 +12,6 @@ SELECT v["payload"]["member"]["id"] FROM ghdata where v["payload"]["member"]["id select k, json_extract(v, '$.repo') from ghdata WHERE v["type"] = 'WatchEvent' order by k limit 10; -- SELECT v["payload"]["member"]["id"], count() FROM ghdata where v["payload"]["member"]["id"] is not null group by v["payload"]["member"]["id"] order by 1, 2 desc LIMIT 10; select k, v["id"], v["type"], v["repo"]["name"] from ghdata WHERE v["type"] = 'WatchEvent' order by k limit 10; -SELECT v["payload"]["pusher_type"] FROM ghdata where v["payload"]["pusher_type"] is not null ORDER BY k LIMIT 10; \ No newline at end of file +SELECT v["payload"]["pusher_type"] FROM ghdata where v["payload"]["pusher_type"] is not null ORDER BY k LIMIT 10; +-- implicit cast to decimal type +SELECT v["id"] FROM ghdata where v["id"] not in (7273, 10.118626, -69352) order by cast(v["id"] as decimal) limit 10; \ No newline at end of file