Skip to content

Commit

Permalink
[Fix](Variant) check storage type before apply get_row_ranges_by_dict
Browse files Browse the repository at this point in the history
    1. storage type maybe missmatch with predicate type eg. cast(v['a'] as date) = '2020-01-01'. v['a'] in storage is string type, but predicate type is date type

    2. serialization object with empty subcolumns may lost num_rows, so need to record num_rows and set back num_rows in serdes
  • Loading branch information
eldenmoon committed Jul 26, 2024
1 parent 13cfe3d commit a9a9222
Show file tree
Hide file tree
Showing 6 changed files with 134 additions and 1 deletion.
5 changes: 5 additions & 0 deletions be/src/olap/rowset/segment_v2/segment_iterator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -699,6 +699,11 @@ Status SegmentIterator::_get_row_ranges_from_conditions(RowRanges* condition_row
if (_opts.io_ctx.reader_type == ReaderType::READER_QUERY) {
RowRanges dict_row_ranges = RowRanges::create_single(num_rows());
for (auto cid : cids) {
if (!_segment->can_apply_predicate_safely(cid,
_opts.col_id_to_predicates.at(cid).get(),
*_schema, _opts.io_ctx.reader_type)) {
continue;
}
RowRanges tmp_row_ranges = RowRanges::create_single(num_rows());
DCHECK(_opts.col_id_to_predicates.count(cid) > 0);
RETURN_IF_ERROR(_column_iterators[cid]->get_row_ranges_by_dict(
Expand Down
19 changes: 18 additions & 1 deletion be/src/vec/data_types/data_type_object.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,13 @@ int64_t DataTypeObject::get_uncompressed_serialized_bytes(const IColumn& column,
size_t size = 0;

size += sizeof(uint32_t);
size_t num_of_columns = 0;
for (const auto& entry : subcolumns) {
auto type = entry->data.get_least_common_type();
if (is_nothing(type)) {
continue;
}

++num_of_columns;
PColumnMeta column_meta_pb;
column_meta_pb.set_name(entry->path.get_path());
type->to_pb_column_meta(&column_meta_pb);
Expand All @@ -78,6 +79,10 @@ int64_t DataTypeObject::get_uncompressed_serialized_bytes(const IColumn& column,
size += type->get_uncompressed_serialized_bytes(entry->data.get_finalized_column(),
be_exec_version);
}
if (num_of_columns == 0) {
// Make sure the num_of_rows in ColumnObject not lost
size += sizeof(uint32_t);
}

return size;
}
Expand Down Expand Up @@ -121,6 +126,11 @@ char* DataTypeObject::serialize(const IColumn& column, char* buf, int be_exec_ve
}
// serialize num of subcolumns
*reinterpret_cast<uint32_t*>(size_pos) = num_of_columns;
if (num_of_columns == 0) {
// Make sure the num_of_rows in ColumnObject not lost
*reinterpret_cast<uint32_t*>(buf) = column_object.rows();
buf += sizeof(uint32_t);
}

return buf;
}
Expand Down Expand Up @@ -155,6 +165,13 @@ const char* DataTypeObject::deserialize(const char* buf, IColumn* column,
}
column_object->add_sub_column(key, std::move(sub_column), type);
}
size_t num_rows = 0;
if (num_subcolumns == 0) {
// Make sure the num_of_rows in ColumnObject not lost
num_rows = *reinterpret_cast<const uint32_t*>(buf);
column_object->set_num_rows(num_rows);
buf += sizeof(uint32_t);
}

column_object->finalize();
#ifndef NDEBUG
Expand Down
16 changes: 16 additions & 0 deletions regression-test/data/variant_p0/rqg/rqg4.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !rqg4 --
0

-- !rqg4_2 --
500

-- !rqg4_3 --
500

-- !rqg4_4 --
70

-- !rqg4_5 --
70

67 changes: 67 additions & 0 deletions regression-test/data/variant_p0/rqg/rqg5.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !rqg5 --
0

-- !rqg5_2 --
50

-- !rqg5_3 --
50

-- !rqg5_4 --
50

-- !rqg5_5 --
50

-- !rqg5_6 --
50

-- !rqg5_7 --
0

-- !rqg5_8 --
0

-- !rqg5_9 --
50

-- !rqg5_10 --
50

-- !rqg5_11 --
50

-- !rqg5_12 --
50

-- !rqg5_13 --
50

-- !rqg5_14 --
50

-- !rqg5_15 --
0

-- !rqg5_16 --
100

-- !rqg5_17 --
100

-- !rqg5_18 --
100

-- !rqg5_19 --
100

-- !rqg5_20 --
100

-- !rqg5_21 --
100

-- !rqg5_22 --
0

6 changes: 6 additions & 0 deletions regression-test/suites/variant_p0/rqg/rqg4.sql

Large diffs are not rendered by default.

22 changes: 22 additions & 0 deletions regression-test/suites/variant_p0/rqg/rqg5.sql

Large diffs are not rendered by default.

0 comments on commit a9a9222

Please sign in to comment.