Skip to content

Commit

Permalink
1
Browse files Browse the repository at this point in the history
  • Loading branch information
Yukang-Lian committed Sep 6, 2024
1 parent 895dddf commit 7d0d303
Show file tree
Hide file tree
Showing 10 changed files with 204 additions and 7 deletions.
6 changes: 6 additions & 0 deletions be/src/olap/partial_update_info.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include "olap/rowset/rowset_writer_context.h"
#include "olap/tablet_schema.h"
#include "olap/utils.h"
#include "util/bitmap_value.h"
#include "vec/common/assert_cast.h"
#include "vec/core/block.h"

Expand Down Expand Up @@ -144,6 +145,11 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids(
DateV2Value<DateV2ValueType> dv;
dv.from_unixtime(timestamp_ms / 1000, timezone);
default_value = dv.debug_string();
} else if (UNLIKELY(column.type() == FieldType::OLAP_FIELD_TYPE_OBJECT &&
to_lower(column.default_value()).find(to_lower("BITMAP_EMPTY")) !=
std::string::npos)) {
BitmapValue v = BitmapValue {};
default_value = v.to_string();
} else {
default_value = column.default_value();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ BINARY: 'BINARY';
BINLOG: 'BINLOG';
BITAND: 'BITAND';
BITMAP: 'BITMAP';
BITMAP_EMPTY: 'BITMAP_EMPTY';
BITMAP_UNION: 'BITMAP_UNION';
BITOR: 'BITOR';
BITXOR: 'BITXOR';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1194,7 +1194,7 @@ columnDef
((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)?
((NOT)? nullable=NULL)?
(AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)?
(DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | PI | stringValue=STRING_LITERAL
(DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | PI | BITMAP_EMPTY | stringValue=STRING_LITERAL
| CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))?
(ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)?
(COMMENT comment=STRING_LITERAL)?
Expand Down Expand Up @@ -1656,6 +1656,7 @@ nonReserved
| BIN
| BITAND
| BITMAP
| BITMAP_EMPTY
| BITMAP_UNION
| BITOR
| BITXOR
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -422,8 +422,10 @@ public void analyze(boolean isOlap) throws AnalysisException {
}

if (type.getPrimitiveType() == PrimitiveType.BITMAP) {
if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE) {
throw new AnalysisException("Bitmap type column can not set default value");
if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE
&& !defaultValue.value.equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value)) {
throw new AnalysisException("Bitmap type column default value only support null or "
+ DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value);
}
defaultValue = DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2780,6 +2780,8 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) {
defaultValue = Optional.of(DefaultValue.CURRENT_DATE_DEFAULT_VALUE);
} else if (ctx.PI() != null) {
defaultValue = Optional.of(DefaultValue.PI_DEFAULT_VALUE);
} else if (ctx.BITMAP_EMPTY() != null) {
defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
}
}
if (ctx.UPDATE() != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -318,13 +318,15 @@ public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKey
}
defaultValue = Optional.of(DefaultValue.HLL_EMPTY_DEFAULT_VALUE);
} else if (type.isBitmapType()) {
if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) {
throw new AnalysisException("Bitmap type column can not set default value");
if (defaultValue.isPresent() && isOlap && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE
&& !defaultValue.get().getValue().equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.getValue())) {
throw new AnalysisException("Bitmap type column default value only support "
+ DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
}
defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
} else if (type.isArrayType() && defaultValue.isPresent() && isOlap
&& defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE && !defaultValue.get()
.getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) {
.getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) {
throw new AnalysisException("Array type column default value only support null or "
+ DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE);
} else if (type.isMapType()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ public class DefaultValue {
public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP";
public static String NOW = "now";
public static String HLL_EMPTY = "HLL_EMPTY";
public static String BITMAP_EMPTY = "BITMAP_EMPTY";
public static DefaultValue CURRENT_DATE_DEFAULT_VALUE = new DefaultValue(CURRENT_DATE, CURRENT_DATE.toLowerCase());
public static DefaultValue CURRENT_TIMESTAMP_DEFAULT_VALUE = new DefaultValue(CURRENT_TIMESTAMP, NOW);
// default null
Expand All @@ -38,7 +39,7 @@ public class DefaultValue {
// default "value", "0" means empty hll
public static DefaultValue HLL_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO, HLL_EMPTY);
// default "value", "0" means empty bitmap
public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO);
public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO, BITMAP_EMPTY);
// default "value", "[]" means empty array
public static DefaultValue ARRAY_EMPTY_DEFAULT_VALUE = new DefaultValue("[]");
// default "value", "3.14159265358979323846" means pi, refer to M_PI in <math.h>.
Expand Down
43 changes: 43 additions & 0 deletions regression-test/data/correctness_p0/test_default_bitmap_empty.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !insert_into1 --
0
0
0
0

-- !stream_load_csv1 --
0
0
0
0
0
0

-- !select_2 --
0
0
0
0

-- !stream_load_csv2 --
0
0
0
0
0
0

-- !insert_into3 --
0
0
0
0

-- !stream_load_csv3 --
0
0
0
0
0
0

Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
5,5
6,6
137 changes: 137 additions & 0 deletions regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("test_default_bitmap_empty") {
sql "SET enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
def tableName = "test_default_bitmap_empty"

sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE IF NOT EXISTS ${tableName}
(
k TINYINT,
v1 bitmap NOT NULL DEFAULT bitmap_empty,
v2 INT
)
UNIQUE KEY(K)
DISTRIBUTED BY HASH(k)
PROPERTIES("replication_num" = "1");
"""

// test insert into.
sql " insert into ${tableName} (k, v2) values (1, 1); "
sql " insert into ${tableName} (k, v2) values (2, 2); "
sql " insert into ${tableName} (k, v2) values (3, 3); "
sql " insert into ${tableName} (k, v2) values (4, 4); "
sql "sync"
qt_insert_into1 """ select bitmap_count(v1) from ${tableName}; """

// test csv stream load.
streamLoad {
table "${tableName}"

set 'column_separator', ','
set 'columns', 'k, v1=bitmap_empty(), v2'

file 'test_default_bitmap_empty_streamload.csv'

time 10000 // limit inflight 10s
}

sql "sync"

qt_stream_load_csv1 """ select bitmap_count(v1) from ${tableName}; """

// test partial update
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE IF NOT EXISTS ${tableName}
(
k TINYINT,
v1 bitmap NOT NULL DEFAULT bitmap_empty,
v2 INT
)
UNIQUE KEY(K)
DISTRIBUTED BY HASH(k)
PROPERTIES("replication_num" = "1");
"""

sql "set enable_unique_key_partial_update=true;"
sql "set enable_insert_strict=false;"

sql " insert into ${tableName} (k, v2) values (1, 1); "
sql " insert into ${tableName} (k, v2) values (2, 2); "
sql " insert into ${tableName} (k, v2) values (3, 3); "
sql " insert into ${tableName} (k, v2) values (4, 4); "
sql "sync"

qt_select_2 "select bitmap_count(v1) from ${tableName};"

streamLoad {
table "${tableName}"

set 'partial_columns', 'true'
set 'column_separator', ','
set 'columns', 'k, v2'

file 'test_default_bitmap_empty_streamload.csv'

time 10000 // limit inflight 10s
}

sql "sync"

qt_stream_load_csv2 """ select bitmap_count(v1) from ${tableName}; """

sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE IF NOT EXISTS ${tableName}
(
k TINYINT,
v1 bitmap BITMAP_UNION default BITMAP_EMPTY,
v2 INT replace_if_not_null
)
aggregate KEY(K)
DISTRIBUTED BY HASH(k)
PROPERTIES("replication_num" = "1");
"""

// test insert into.
sql " insert into ${tableName} (k, v2) values (1, 1); "
sql " insert into ${tableName} (k, v2) values (2, 2); "
sql " insert into ${tableName} (k, v2) values (3, 3); "
sql " insert into ${tableName} (k, v2) values (4, 4); "
sql "sync"
qt_insert_into3 """ select bitmap_count(v1) from ${tableName}; """

// test csv stream load.
streamLoad {
table "${tableName}"

set 'column_separator', ','
set 'columns', 'k, v1=bitmap_empty(), v2'

file 'test_default_bitmap_empty_streamload.csv'

time 10000 // limit inflight 10s
}

sql "sync"

qt_stream_load_csv3 """ select bitmap_count(v1) from ${tableName}; """
}

0 comments on commit 7d0d303

Please sign in to comment.