Skip to content

Commit

Permalink
[FIX](serde)fix datetimev2 serde parse from string with scale (#27965)
Browse files Browse the repository at this point in the history
  • Loading branch information
@amorynan @zhiqiang-hhhh
amorynan authored and zhiqiang-hhhh committed Mar 19, 2024
1 parent 1ff92ca commit 04935bb
Show file tree
Hide file tree
Showing 6 changed files with 147 additions and 49 deletions.
6 changes: 3 additions & 3 deletions be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,9 @@ Status DataTypeDateTimeV2SerDe::deserialize_one_cell_from_json(IColumn& column,
}

} else if (ReadBuffer rb(slice.data, slice.size);
!read_datetime_v2_text_impl<UInt64>(val, rb)) {
return Status::InvalidDataFormat("parse date fail, string: '{}'",
std::string(rb.position(), rb.count()).c_str());
!read_datetime_v2_text_impl<UInt64>(val, rb, scale)) {
return Status::InvalidArgument("parse date fail, string: '{}'",
std::string(rb.position(), rb.count()).c_str());
}
column_data.insert_value(val);
return Status::OK();
Expand Down
10 changes: 5 additions & 5 deletions ...ion-test/data/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.out
View file

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions ...on-test/data/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.out
View file

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
3|"2022-12-01 22:23:24.999999"|22.6789|33.6789|["2022-12-01 22:23:24.999999","2022-12-01 23:23:24.999999"]|[22.6789,33.6789]|[22.6789,33.6789]
4|"2022-12-02 22:23:24.999999"|23.6789|34.6789|["2022-12-02 22:23:24.999999","2022-12-02 23:23:24.999999"]|[23.6789,34.6789]|[22.6789,34.6789]
146 changes: 110 additions & 36 deletions regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@
2022-12-02T22:23:24.999

-- !select --
2022-12-01T22:23:24.999
2022-12-02T22:23:24.999
2022-12-01T22:23:25
2022-12-02T22:23:25
2022-12-01T22:23:25
2022-12-02T22:23:25

-- !select --
2022-12-02T22:23:24.999999
Expand All @@ -16,8 +18,10 @@
2022-12-02T22:23:24.999

-- !select --
2022-12-01T23:23:24.999
2022-12-02T23:23:24.999
2022-12-01T23:23:25
2022-12-02T23:23:25
2022-12-01T23:23:25
2022-12-02T23:23:25

-- !select --
23
Expand All @@ -31,10 +35,14 @@
-- !select --
22.679
23.679
22.679
23.679

-- !select --
22.679
22.679
22.679
22.679

-- !select --
23
Expand All @@ -48,139 +56,203 @@
-- !select --
33.679
34.679
33.679
34.679

-- !select --
33.679
34.679
33.679
34.679

-- !select --
[22.679]
[23.679]
[22.679]
[23.679]

-- !select --
[24.990, 25.990]
[24.990, 25.990]
[24.990, 25.990]
[24.990, 25.990]

-- !select --
[24.990, 25.990]

-- !select --
[33.679]
[34.679]
[33.679]
[34.679]

-- !select --
[24.990, 25.990]
[24.990, 25.990]
[24.990, 25.990]
[24.990, 25.990]

-- !select --
[24.990, 25.990]

-- !select --
["2022-12-01 22:23:24.999"]
["2022-12-02 22:23:24.999"]
[null]
[null]
["2022-12-01 22:23:25.000"]
["2022-12-02 22:23:25.000"]

-- !select --
["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"]
["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"]
["2022-12-02 22:23:25.000", "2022-12-02 22:23:23.998"]
["2022-12-02 22:23:25.000", "2022-12-02 22:23:23.998"]
["2022-12-02 22:23:25.000", "2022-12-02 22:23:23.998"]
["2022-12-02 22:23:25.000", "2022-12-02 22:23:23.998"]

-- !select --
[2022-12-02 22:23:24.999, 2022-12-02 22:23:23.997]

-- !select --
[]
[]
[]
[]

-- !select --
["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"]
["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"]
["2022-12-01 22:23:25.000000", "2022-12-01 23:23:25.000000"]
["2022-12-02 22:23:25.000000", "2022-12-02 23:23:25.000000"]
["2022-12-01 22:23:25.000000", "2022-12-01 23:23:25.000000"]
["2022-12-02 22:23:25.000000", "2022-12-02 23:23:25.000000"]

-- !select --
\N
\N
\N
\N

-- !select --
[22.679]
[]
[22.679]
[]

-- !select --
[22.679, 33.679]
[23.679, 34.679]
[22.679, 33.679]
[23.679, 34.679]

-- !select --
\N
\N
\N
\N

-- !select --
["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"]
["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"]
["2022-12-02 22:23:24.123", "2022-12-02 22:23:23.123"]
["2022-12-02 22:23:24.123", "2022-12-02 22:23:23.123"]
["2022-12-02 22:23:24.123", "2022-12-02 22:23:23.123"]
["2022-12-02 22:23:24.123", "2022-12-02 22:23:23.123"]

-- !select --
["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"]
["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"]
["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"]
["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"]
["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"]
["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"]

-- !select --
["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999", "2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"]
["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999", "2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"]
["2022-12-01 22:23:25.000000", "2022-12-01 23:23:25.000000", "2022-12-02 22:23:24.123000", "2022-12-02 22:23:23.123000"]
["2022-12-02 22:23:25.000000", "2022-12-02 23:23:25.000000", "2022-12-02 22:23:24.123000", "2022-12-02 22:23:23.123000"]
["2022-12-01 22:23:25.000000", "2022-12-01 23:23:25.000000", "2022-12-02 22:23:24.123000", "2022-12-02 22:23:23.123000"]
["2022-12-02 22:23:25.000000", "2022-12-02 23:23:25.000000", "2022-12-02 22:23:24.123000", "2022-12-02 22:23:23.123000"]

-- !select --
[22.679, 33.679, 22.679, 33.679, 22.679, 33.679]
[23.679, 34.679, 23.679, 34.679, 23.679, 34.679]
[22.679, 33.679, 22.679, 33.679, 22.679, 33.679]
[23.679, 34.679, 23.679, 34.679, 23.679, 34.679]

-- !select --
[{"col": 22.679, "col": 22.679, "col": "2022-12-01 22:23:24.999", "col": 22.679}, {"col": 33.679, "col": 33.679, "col": "2022-12-01 23:23:24.999", "col": 33.679}]
[{"col": 23.679, "col": 23.679, "col": "2022-12-02 22:23:24.999", "col": 23.679}, {"col": 34.679, "col": 34.679, "col": "2022-12-02 23:23:24.999", "col": 34.679}]
[{"1": 22.679, "2": 22.679, "3": "2022-12-01 22:23:25.000", "4": 22.679}, {"1": 33.679, "2": 33.679, "3": "2022-12-01 23:23:25.000", "4": 33.679}]
[{"1": 23.679, "2": 23.679, "3": "2022-12-02 22:23:25.000", "4": 23.679}, {"1": 34.679, "2": 34.679, "3": "2022-12-02 23:23:25.000", "4": 34.679}]
[{"1": 22.679, "2": 22.679, "3": "2022-12-01 22:23:25.000", "4": 22.679}, {"1": 33.679, "2": 33.679, "3": "2022-12-01 23:23:25.000", "4": 33.679}]
[{"1": 23.679, "2": 23.679, "3": "2022-12-02 22:23:25.000", "4": 23.679}, {"1": 34.679, "2": 34.679, "3": "2022-12-02 23:23:25.000", "4": 34.679}]

-- !select --
[{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 22:23:23.997"}]
[{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 22:23:23.997"}]
[{"1": "2022-12-02 22:23:24.123"}, {"1": "2022-12-02 22:23:23.123"}]
[{"1": "2022-12-02 22:23:24.123"}, {"1": "2022-12-02 22:23:23.123"}]
[{"1": "2022-12-02 22:23:24.123"}, {"1": "2022-12-02 22:23:23.123"}]
[{"1": "2022-12-02 22:23:24.123"}, {"1": "2022-12-02 22:23:23.123"}]

-- !select --
[{"col": "2022-12-01 22:23:24.999"}, {"col": "2022-12-01 23:23:24.999"}]
[{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 23:23:24.999"}]
[{"1": "2022-12-01 22:23:25.000"}, {"1": "2022-12-01 23:23:25.000"}]
[{"1": "2022-12-02 22:23:25.000"}, {"1": "2022-12-02 23:23:25.000"}]
[{"1": "2022-12-01 22:23:25.000"}, {"1": "2022-12-01 23:23:25.000"}]
[{"1": "2022-12-02 22:23:25.000"}, {"1": "2022-12-02 23:23:25.000"}]

-- !select --
[{"col": "2022-12-01 22:23:24.999", "col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-01 23:23:24.999", "col": "2022-12-02 22:23:23.997"}]
[{"col": "2022-12-02 22:23:24.999", "col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 23:23:24.999", "col": "2022-12-02 22:23:23.997"}]
[{"1": "2022-12-01 22:23:25.000", "2": "2022-12-02 22:23:24.123"}, {"1": "2022-12-01 23:23:25.000", "2": "2022-12-02 22:23:23.123"}]
[{"1": "2022-12-02 22:23:25.000", "2": "2022-12-02 22:23:24.123"}, {"1": "2022-12-02 23:23:25.000", "2": "2022-12-02 22:23:23.123"}]
[{"1": "2022-12-01 22:23:25.000", "2": "2022-12-02 22:23:24.123"}, {"1": "2022-12-01 23:23:25.000", "2": "2022-12-02 22:23:23.123"}]
[{"1": "2022-12-02 22:23:25.000", "2": "2022-12-02 22:23:24.123"}, {"1": "2022-12-02 23:23:25.000", "2": "2022-12-02 22:23:23.123"}]

-- !select --
["2022-12-02 22:23:23.997", "2022-12-02 22:23:24.999"]
["2022-12-02 22:23:23.997", "2022-12-02 22:23:24.999"]
["2022-12-02 22:23:23.123", "2022-12-02 22:23:24.123"]
["2022-12-02 22:23:23.123", "2022-12-02 22:23:24.123"]
["2022-12-02 22:23:23.123", "2022-12-02 22:23:24.123"]
["2022-12-02 22:23:23.123", "2022-12-02 22:23:24.123"]

-- !select --
["2023-03-08 23:23:23.997", "2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"]
["2023-03-08 23:23:23.997", "2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"]
["2023-03-08 23:23:23.123", "2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"]
["2023-03-08 23:23:23.123", "2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"]
["2023-03-08 23:23:23.123", "2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"]
["2023-03-08 23:23:23.123", "2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"]

-- !select --
2022-12-01T22:23:24.999 ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2022-12-01 22:23:24.999", "2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"]
2022-12-02T22:23:24.999 ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] ["2022-12-02 22:23:24.999", "2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"]
\N ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] [null, "2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"]
\N ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] [null, "2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"]
2022-12-01T22:23:25 ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] ["2022-12-01 22:23:25.000", "2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"]
2022-12-02T22:23:25 ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] ["2022-12-02 22:23:25.000", "2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"]

-- !select --
[25.990, 22.679, 33.679]
[25.990, 23.679, 34.679]
[25.990, 22.679, 33.679]
[25.990, 23.679, 34.679]

-- !select --
22.679 [22.679, 33.679] [22.679, 22.679, 33.679]
23.679 [23.679, 34.679] [23.679, 23.679, 34.679]
22.679 [22.679, 33.679] [22.679, 22.679, 33.679]
23.679 [23.679, 34.679] [23.679, 23.679, 34.679]

-- !select --
["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"]
["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"]
["2022-12-02 22:23:24.123", "2022-12-02 22:23:23.123"]
["2022-12-02 22:23:24.123", "2022-12-02 22:23:23.123"]
["2022-12-02 22:23:24.123", "2022-12-02 22:23:23.123"]
["2022-12-02 22:23:24.123", "2022-12-02 22:23:23.123"]

-- !select --
["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999", "2023-03-08 23:23:23.997"]
["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999", "2023-03-08 23:23:23.997"]
["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000", "2023-03-08 23:23:23.123"]
["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000", "2023-03-08 23:23:23.123"]
["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000", "2023-03-08 23:23:23.123"]
["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000", "2023-03-08 23:23:23.123"]

-- !select --
2022-12-01T22:23:24.999 ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999", "2022-12-01 22:23:24.999"]
2022-12-02T22:23:24.999 ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999", "2022-12-02 22:23:24.999"]
\N ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000", null]
\N ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000", null]
2022-12-01T22:23:25 ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000", "2022-12-01 22:23:25.000"]
2022-12-02T22:23:25 ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000", "2022-12-02 22:23:25.000"]

-- !select --
[22.679, 33.679, 25.990]
[23.679, 34.679, 25.990]
[22.679, 33.679, 25.990]
[23.679, 34.679, 25.990]

-- !select --
22.679 [22.679, 33.679] [22.679, 33.679, 22.679]
23.679 [23.679, 34.679] [23.679, 34.679, 23.679]
22.679 [22.679, 33.679] [22.679, 33.679, 22.679]
23.679 [23.679, 34.679] [23.679, 34.679, 23.679]

-- !select --
[23, 11]
Expand All @@ -194,4 +266,6 @@
-- !select --
[22.679, 56.358]
[23.679, 58.358]
[22.679, 56.358]
[23.679, 58.358]

22 changes: 22 additions & 0 deletions ...sion-test/suites/query_p0/sql_functions/array_functions/test_array_with_scale_type.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,28 @@ suite("test_array_with_scale_type") {
)
"""

// load with same insert into data
streamLoad {
table "${tableName}"

set 'column_separator', '|'

file 'test_array_with_scale_type.csv'
time 10000 // limit inflight 10s

check { result, exception, startTime, endTime ->
if (exception != null) {
throw exception
}
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("success", json.Status.toLowerCase())
assertEquals(2, json.NumberTotalRows)
assertEquals(2, json.NumberLoadedRows)
assertEquals(0, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
sql """INSERT INTO ${tableName} values
(1,"2022-12-01 22:23:24.999999",22.6789,33.6789,["2022-12-01 22:23:24.999999","2022-12-01 23:23:24.999999"],[22.6789,33.6789],[22.6789,33.6789]),
(2,"2022-12-02 22:23:24.999999",23.6789,34.6789,["2022-12-02 22:23:24.999999","2022-12-02 23:23:24.999999"],[23.6789,34.6789],[22.6789,34.6789])
Expand Down

0 comments on commit 04935bb

Please sign in to comment.