From 725f07f68149f3d36140afdbe0598509e8bd7c6d Mon Sep 17 00:00:00 2001 From: amory Date: Thu, 21 Mar 2024 13:32:17 +0800 Subject: [PATCH 01/14] [FIX](serde )pick 27965 for decimalv2 (#32533) * fix scale * fixed --- .../serde/data_type_datetimev2_serde.cpp | 6 +- .../test_array_with_scale_type.csv | 2 + .../test_array_with_scale_type.out | 74 +++++++++++++++++++ .../test_array_with_scale_type.groovy | 22 ++++++ 4 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.csv diff --git a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp index 10f5f0e79f71463..7dc7dbf17c65dc8 100644 --- a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp @@ -79,9 +79,9 @@ Status DataTypeDateTimeV2SerDe::deserialize_one_cell_from_json(IColumn& column, } } else if (ReadBuffer rb(slice.data, slice.size); - !read_datetime_v2_text_impl(val, rb)) { - return Status::InvalidDataFormat("parse date fail, string: '{}'", - std::string(rb.position(), rb.count()).c_str()); + !read_datetime_v2_text_impl(val, rb, scale)) { + return Status::InvalidArgument("parse date fail, string: '{}'", + std::string(rb.position(), rb.count()).c_str()); } column_data.insert_value(val); return Status::OK(); diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.csv b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.csv new file mode 100644 index 000000000000000..a14f2b3fd6eb42e --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.csv @@ -0,0 +1,2 @@ +3|"2022-12-01 22:23:24.999999"|22.6789|33.6789|["2022-12-01 22:23:24.999999","2022-12-01 23:23:24.999999"]|[22.6789,33.6789]|[22.6789,33.6789] +4|"2022-12-02 22:23:24.999999"|23.6789|34.6789|["2022-12-02 22:23:24.999999","2022-12-02 23:23:24.999999"]|[23.6789,34.6789]|[22.6789,34.6789] diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out index 8758907842bb2bf..278e084c87d635c 100644 --- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out @@ -6,6 +6,8 @@ 2022-12-02T22:23:24.999 -- !select -- +2022-12-01T22:23:25 +2022-12-02T22:23:25 2022-12-01T22:23:24.999 2022-12-02T22:23:24.999 @@ -16,6 +18,8 @@ 2022-12-02T22:23:24.999 -- !select -- +2022-12-01T23:23:25 +2022-12-02T23:23:25 2022-12-01T23:23:24.999 2022-12-02T23:23:24.999 @@ -31,10 +35,14 @@ -- !select -- 22.679 23.679 +22.679 +23.679 -- !select -- 22.679 22.679 +22.679 +22.679 -- !select -- 23 @@ -48,18 +56,26 @@ -- !select -- 33.679 34.679 +33.679 +34.679 -- !select -- 33.679 34.679 +33.679 +34.679 -- !select -- [22.679] [23.679] +[22.679] +[23.679] -- !select -- [24.990, 25.990] [24.990, 25.990] +[24.990, 25.990] +[24.990, 25.990] -- !select -- [24.990, 25.990] @@ -67,21 +83,29 @@ -- !select -- [33.679] [34.679] +[33.679] +[34.679] -- !select -- [24.990, 25.990] [24.990, 25.990] +[24.990, 25.990] +[24.990, 25.990] -- !select -- [24.990, 25.990] -- !select -- +[null] +[null] ["2022-12-01 22:23:24.999"] ["2022-12-02 22:23:24.999"] -- !select -- ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] -- !select -- [2022-12-02 22:23:24.999, 2022-12-02 22:23:23.997] @@ -89,98 +113,146 @@ -- !select -- [] [] +[] +[] -- !select -- +["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] +["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] -- !select -- \N \N +\N +\N -- !select -- [22.679] [] +[22.679] +[] -- !select -- [22.679, 33.679] [23.679, 34.679] +[22.679, 33.679] +[23.679, 34.679] -- !select -- \N \N +\N +\N -- !select -- ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] -- !select -- +["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] +["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] -- !select -- +["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000", "2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000", "2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999", "2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999", "2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] -- !select -- [22.679, 33.679, 22.679, 33.679, 22.679, 33.679] [23.679, 34.679, 23.679, 34.679, 23.679, 34.679] +[22.679, 33.679, 22.679, 33.679, 22.679, 33.679] +[23.679, 34.679, 23.679, 34.679, 23.679, 34.679] -- !select -- +[{"col": 22.679, "col": 22.679, "col": "2022-12-01 22:23:25.000", "col": 22.679}, {"col": 33.679, "col": 33.679, "col": "2022-12-01 23:23:25.000", "col": 33.679}] +[{"col": 23.679, "col": 23.679, "col": "2022-12-02 22:23:25.000", "col": 23.679}, {"col": 34.679, "col": 34.679, "col": "2022-12-02 23:23:25.000", "col": 34.679}] [{"col": 22.679, "col": 22.679, "col": "2022-12-01 22:23:24.999", "col": 22.679}, {"col": 33.679, "col": 33.679, "col": "2022-12-01 23:23:24.999", "col": 33.679}] [{"col": 23.679, "col": 23.679, "col": "2022-12-02 22:23:24.999", "col": 23.679}, {"col": 34.679, "col": 34.679, "col": "2022-12-02 23:23:24.999", "col": 34.679}] -- !select -- [{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 22:23:23.997"}] [{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 22:23:23.997"}] +[{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 22:23:23.997"}] +[{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 22:23:23.997"}] -- !select -- +[{"col": "2022-12-01 22:23:25.000"}, {"col": "2022-12-01 23:23:25.000"}] +[{"col": "2022-12-02 22:23:25.000"}, {"col": "2022-12-02 23:23:25.000"}] [{"col": "2022-12-01 22:23:24.999"}, {"col": "2022-12-01 23:23:24.999"}] [{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 23:23:24.999"}] -- !select -- +[{"col": "2022-12-01 22:23:25.000", "col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-01 23:23:25.000", "col": "2022-12-02 22:23:23.997"}] +[{"col": "2022-12-02 22:23:25.000", "col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 23:23:25.000", "col": "2022-12-02 22:23:23.997"}] [{"col": "2022-12-01 22:23:24.999", "col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-01 23:23:24.999", "col": "2022-12-02 22:23:23.997"}] [{"col": "2022-12-02 22:23:24.999", "col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 23:23:24.999", "col": "2022-12-02 22:23:23.997"}] -- !select -- ["2022-12-02 22:23:23.997", "2022-12-02 22:23:24.999"] ["2022-12-02 22:23:23.997", "2022-12-02 22:23:24.999"] +["2022-12-02 22:23:23.997", "2022-12-02 22:23:24.999"] +["2022-12-02 22:23:23.997", "2022-12-02 22:23:24.999"] -- !select -- +["2023-03-08 23:23:23.997", "2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] +["2023-03-08 23:23:23.997", "2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] ["2023-03-08 23:23:23.997", "2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2023-03-08 23:23:23.997", "2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] -- !select -- +\N ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] [null, "2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] +\N ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] [null, "2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] 2022-12-01T22:23:24.999 ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2022-12-01 22:23:24.999", "2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] 2022-12-02T22:23:24.999 ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] ["2022-12-02 22:23:24.999", "2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] -- !select -- [25.990, 22.679, 33.679] [25.990, 23.679, 34.679] +[25.990, 22.679, 33.679] +[25.990, 23.679, 34.679] -- !select -- 22.679 [22.679, 33.679] [22.679, 22.679, 33.679] 23.679 [23.679, 34.679] [23.679, 23.679, 34.679] +22.679 [22.679, 33.679] [22.679, 22.679, 33.679] +23.679 [23.679, 34.679] [23.679, 23.679, 34.679] -- !select -- ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] -- !select -- +["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000", "2023-03-08 23:23:23.997"] +["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000", "2023-03-08 23:23:23.997"] ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999", "2023-03-08 23:23:23.997"] ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999", "2023-03-08 23:23:23.997"] -- !select -- +\N ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000", null] +\N ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000", null] 2022-12-01T22:23:24.999 ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999", "2022-12-01 22:23:24.999"] 2022-12-02T22:23:24.999 ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999", "2022-12-02 22:23:24.999"] -- !select -- [22.679, 33.679, 25.990] [23.679, 34.679, 25.990] +[22.679, 33.679, 25.990] +[23.679, 34.679, 25.990] -- !select -- 22.679 [22.679, 33.679] [22.679, 33.679, 22.679] 23.679 [23.679, 34.679] [23.679, 34.679, 23.679] +22.679 [22.679, 33.679] [22.679, 33.679, 22.679] +23.679 [23.679, 34.679] [23.679, 34.679, 23.679] -- !select -- [23, 11] @@ -194,4 +266,6 @@ -- !select -- [22.679, 56.358] [23.679, 58.358] +[22.679, 56.358] +[23.679, 58.358] diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_with_scale_type.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_with_scale_type.groovy index 53a37100f0ea1f2..96a616bcac28810 100644 --- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_with_scale_type.groovy +++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_with_scale_type.groovy @@ -36,6 +36,28 @@ suite("test_array_with_scale_type") { "storage_format" = "V2" ) """ + // load with same insert into data + streamLoad { + table "${tableName}" + + set 'column_separator', '|' + + file 'test_array_with_scale_type.csv' + time 10000 // limit inflight 10s + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(2, json.NumberTotalRows) + assertEquals(2, json.NumberLoadedRows) + assertEquals(0, json.NumberFilteredRows) + assertEquals(0, json.NumberUnselectedRows) + } + } sql """INSERT INTO ${tableName} values (1,"2022-12-01 22:23:24.999999",22.6789,33.6789,["2022-12-01 22:23:24.999999","2022-12-01 23:23:24.999999"],[22.6789,33.6789],[22.6789,33.6789]), From dc28f65a3d0cc80834b58534155dfb460a4d25fe Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Thu, 21 Mar 2024 14:05:50 +0800 Subject: [PATCH 02/14] [pick-2.0][Enhancement](jdbc catalog) Add a property to test the connection when creating a Jdbc catalog (#32581) --- be/src/service/internal_service.cpp | 59 ++++++++++ be/src/service/internal_service.h | 5 + be/src/vec/exec/vjdbc_connector.cpp | 27 ++++- be/src/vec/exec/vjdbc_connector.h | 5 + .../org/apache/doris/jdbc/JdbcExecutor.java | 21 ++++ .../apache/doris/catalog/JdbcResource.java | 9 +- .../doris/datasource/CatalogFactory.java | 2 +- .../datasource/jdbc/JdbcExternalCatalog.java | 111 +++++++++++++++++- .../datasource/jdbc/client/JdbcClient.java | 18 +++ .../jdbc/client/JdbcOracleClient.java | 5 + .../jdbc/client/JdbcSapHanaClient.java | 5 + .../doris/rpc/BackendServiceClient.java | 5 + .../apache/doris/rpc/BackendServiceProxy.java | 12 ++ .../jdbc/JdbcExternalCatalogTest.java | 2 +- gensrc/proto/internal_service.proto | 11 ++ .../jdbc/test_clickhouse_jdbc_catalog.out | Bin 5002 -> 5437 bytes .../jdbc/test_clickhouse_jdbc_catalog.groovy | 30 ++++- 17 files changed, 320 insertions(+), 7 deletions(-) diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index acc2e29a3ef55eb..2fd4310c4bc7abd 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -683,6 +684,64 @@ void PInternalServiceImpl::tablet_fetch_data(google::protobuf::RpcController* co } } +void PInternalServiceImpl::test_jdbc_connection(google::protobuf::RpcController* controller, + const PJdbcTestConnectionRequest* request, + PJdbcTestConnectionResult* result, + google::protobuf::Closure* done) { + bool ret = _heavy_work_pool.try_offer([request, result, done]() { + VLOG_RPC << "test jdbc connection"; + brpc::ClosureGuard closure_guard(done); + TTableDescriptor table_desc; + vectorized::JdbcConnectorParam jdbc_param; + Status st = Status::OK(); + { + const uint8_t* buf = (const uint8_t*)request->jdbc_table().data(); + uint32_t len = request->jdbc_table().size(); + st = deserialize_thrift_msg(buf, &len, false, &table_desc); + if (!st.ok()) { + LOG(WARNING) << "test jdbc connection failed, errmsg=" << st; + st.to_protobuf(result->mutable_status()); + return; + } + } + TJdbcTable jdbc_table = (table_desc.jdbcTable); + jdbc_param.catalog_id = jdbc_table.catalog_id; + jdbc_param.driver_class = jdbc_table.jdbc_driver_class; + jdbc_param.driver_path = jdbc_table.jdbc_driver_url; + jdbc_param.driver_checksum = jdbc_table.jdbc_driver_checksum; + jdbc_param.jdbc_url = jdbc_table.jdbc_url; + jdbc_param.user = jdbc_table.jdbc_user; + jdbc_param.passwd = jdbc_table.jdbc_password; + jdbc_param.query_string = request->query_str(); + jdbc_param.table_type = static_cast(request->jdbc_table_type()); + jdbc_param.connection_pool_min_size = jdbc_table.connection_pool_min_size; + jdbc_param.connection_pool_max_size = jdbc_table.connection_pool_max_size; + jdbc_param.connection_pool_max_life_time = jdbc_table.connection_pool_max_life_time; + jdbc_param.connection_pool_max_wait_time = jdbc_table.connection_pool_max_wait_time; + jdbc_param.connection_pool_keep_alive = jdbc_table.connection_pool_keep_alive; + + std::unique_ptr jdbc_connector; + jdbc_connector.reset(new (std::nothrow) vectorized::JdbcConnector(jdbc_param)); + + st = jdbc_connector->test_connection(); + st.to_protobuf(result->mutable_status()); + + Status clean_st = jdbc_connector->clean_datasource(); + if (!clean_st.ok()) { + LOG(WARNING) << "Failed to clean JDBC datasource: " << clean_st.msg(); + } + Status close_st = jdbc_connector->close(); + if (!close_st.ok()) { + LOG(WARNING) << "Failed to close JDBC connector: " << close_st.msg(); + } + }); + + if (!ret) { + offer_failed(result, done, _heavy_work_pool); + return; + } +} + void PInternalServiceImpl::get_column_ids_by_tablet_ids(google::protobuf::RpcController* controller, const PFetchColIdsRequest* request, PFetchColIdsResponse* response, diff --git a/be/src/service/internal_service.h b/be/src/service/internal_service.h index 47762cf7e521c24..259a41c2e335c66 100644 --- a/be/src/service/internal_service.h +++ b/be/src/service/internal_service.h @@ -184,6 +184,11 @@ class PInternalServiceImpl : public PBackendService { void glob(google::protobuf::RpcController* controller, const PGlobRequest* request, PGlobResponse* response, google::protobuf::Closure* done) override; + void test_jdbc_connection(google::protobuf::RpcController* controller, + const PJdbcTestConnectionRequest* request, + PJdbcTestConnectionResult* result, + google::protobuf::Closure* done) override; + private: void _exec_plan_fragment_in_pthread(google::protobuf::RpcController* controller, const PExecPlanFragmentRequest* request, diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp index 9b98627db9415cc..9344faad01deabc 100644 --- a/be/src/vec/exec/vjdbc_connector.cpp +++ b/be/src/vec/exec/vjdbc_connector.cpp @@ -156,7 +156,11 @@ Status JdbcConnector::open(RuntimeState* state, bool read) { ctor_params.__set_jdbc_password(_conn_param.passwd); ctor_params.__set_jdbc_driver_class(_conn_param.driver_class); ctor_params.__set_driver_path(local_location); - ctor_params.__set_batch_size(read ? state->batch_size() : 0); + if (state == nullptr) { + ctor_params.__set_batch_size(read ? 1 : 0); + } else { + ctor_params.__set_batch_size(read ? state->batch_size() : 0); + } ctor_params.__set_op(read ? TJdbcOperation::READ : TJdbcOperation::WRITE); ctor_params.__set_table_type(_conn_param.table_type); ctor_params.__set_connection_pool_min_size(_conn_param.connection_pool_min_size); @@ -185,6 +189,23 @@ Status JdbcConnector::open(RuntimeState* state, bool read) { return Status::OK(); } +Status JdbcConnector::test_connection() { + RETURN_IF_ERROR(open(nullptr, true)); + + JNIEnv* env = nullptr; + RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); + + env->CallNonvirtualVoidMethod(_executor_obj, _executor_clazz, _executor_test_connection_id); + return JniUtil::GetJniExceptionMsg(env); +} + +Status JdbcConnector::clean_datasource() { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); + env->CallNonvirtualVoidMethod(_executor_obj, _executor_clazz, _executor_clean_datasource_id); + return JniUtil::GetJniExceptionMsg(env); +} + Status JdbcConnector::query() { if (!_is_open) { return Status::InternalError("Query before open of JdbcConnector."); @@ -840,6 +861,10 @@ Status JdbcConnector::_register_func_id(JNIEnv* env) { JDBC_EXECUTOR_TRANSACTION_SIGNATURE, _executor_abort_trans_id)); RETURN_IF_ERROR(register_id(_executor_clazz, "getResultColumnTypeNames", JDBC_EXECUTOR_GET_TYPES_SIGNATURE, _executor_get_types_id)); + RETURN_IF_ERROR( + register_id(_executor_clazz, "testConnection", "()V", _executor_test_connection_id)); + RETURN_IF_ERROR( + register_id(_executor_clazz, "cleanDataSource", "()V", _executor_clean_datasource_id)); return Status::OK(); } diff --git a/be/src/vec/exec/vjdbc_connector.h b/be/src/vec/exec/vjdbc_connector.h index 55b39f0aec20a84..32dc5aba47b1459 100644 --- a/be/src/vec/exec/vjdbc_connector.h +++ b/be/src/vec/exec/vjdbc_connector.h @@ -103,6 +103,9 @@ class JdbcConnector : public TableConnector { Status close() override; + Status test_connection(); + Status clean_datasource(); + private: Status _register_func_id(JNIEnv* env); Status _check_column_type(); @@ -165,6 +168,8 @@ class JdbcConnector : public TableConnector { jmethodID _executor_begin_trans_id; jmethodID _executor_finish_trans_id; jmethodID _executor_abort_trans_id; + jmethodID _executor_test_connection_id; + jmethodID _executor_clean_datasource_id; std::map _map_column_idx_to_cast_idx; std::vector _input_array_string_types; std::vector diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java index 3011c5d82f7b85f..6f15600eddc5b37 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java @@ -180,6 +180,27 @@ public boolean abortReadConnection(Connection connection, ResultSet resultSet, T return false; } + + public void cleanDataSource() { + if (druidDataSource != null) { + druidDataSource.close(); + JdbcDataSource.getDataSource().getSourcesMap().remove(config.createCacheKey()); + druidDataSource = null; + } + } + + public void testConnection() throws UdfRuntimeException { + try { + resultSet = ((PreparedStatement) stmt).executeQuery(); + if (!resultSet.next()) { + throw new UdfRuntimeException( + "Failed to test connection in BE: query executed but returned no results."); + } + } catch (SQLException e) { + throw new UdfRuntimeException("Failed to test connection in BE: ", e); + } + } + public int read() throws UdfRuntimeException { try { resultSet = ((PreparedStatement) stmt).executeQuery(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index 3291b64cbb05904..1faf27e1040e184 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -104,6 +104,8 @@ public class JdbcResource extends Resource { public static final String CONNECTION_POOL_KEEP_ALIVE = "connection_pool_keep_alive"; public static final String CHECK_SUM = "checksum"; public static final String CREATE_TIME = "create_time"; + public static final String TEST_CONNECTION = "test_connection"; + private static final ImmutableList ALL_PROPERTIES = new ImmutableList.Builder().add( JDBC_URL, USER, @@ -120,7 +122,8 @@ public class JdbcResource extends Resource { CONNECTION_POOL_MAX_SIZE, CONNECTION_POOL_MAX_LIFE_TIME, CONNECTION_POOL_MAX_WAIT_TIME, - CONNECTION_POOL_KEEP_ALIVE + CONNECTION_POOL_KEEP_ALIVE, + TEST_CONNECTION ).build(); private static final ImmutableList OPTIONAL_PROPERTIES = new ImmutableList.Builder().add( ONLY_SPECIFIED_DATABASE, @@ -131,7 +134,8 @@ public class JdbcResource extends Resource { CONNECTION_POOL_MAX_SIZE, CONNECTION_POOL_MAX_LIFE_TIME, CONNECTION_POOL_MAX_WAIT_TIME, - CONNECTION_POOL_KEEP_ALIVE + CONNECTION_POOL_KEEP_ALIVE, + TEST_CONNECTION ).build(); // The default value of optional properties @@ -148,6 +152,7 @@ public class JdbcResource extends Resource { OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_LIFE_TIME, "1800000"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_WAIT_TIME, "5000"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_KEEP_ALIVE, "false"); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(TEST_CONNECTION, "true"); } // timeout for both connection and read. 10 seconds is long enough. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java index 09ad69ec8b661d1..3bf674a8abdac4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java @@ -119,7 +119,7 @@ private static CatalogIf createCatalog(long catalogId, String name, String resou catalog = new EsExternalCatalog(catalogId, name, resource, props, comment); break; case "jdbc": - catalog = new JdbcExternalCatalog(catalogId, name, resource, props, comment); + catalog = new JdbcExternalCatalog(catalogId, name, resource, props, comment, isReplay); break; case "iceberg": catalog = IcebergExternalCatalogFactory.createCatalog(catalogId, name, resource, props, comment); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index ea9d5d749997a3a..774ccd2dd25b1ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -17,10 +17,14 @@ package org.apache.doris.datasource.jdbc; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.JdbcResource; +import org.apache.doris.catalog.JdbcTable; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.external.JdbcExternalDatabase; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; @@ -28,19 +32,37 @@ import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.jdbc.client.JdbcClient; import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; +import org.apache.doris.datasource.jdbc.client.JdbcClientException; +import org.apache.doris.proto.InternalService; +import org.apache.doris.proto.InternalService.PJdbcTestConnectionRequest; +import org.apache.doris.proto.InternalService.PJdbcTestConnectionResult; +import org.apache.doris.rpc.BackendServiceProxy; +import org.apache.doris.rpc.RpcException; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TStatusCode; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; import lombok.Getter; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; @Getter public class JdbcExternalCatalog extends ExternalCatalog { + private static final Logger LOG = LogManager.getLogger(JdbcExternalCatalog.class); + private static final List REQUIRED_PROPERTIES = ImmutableList.of( JdbcResource.JDBC_URL, JdbcResource.DRIVER_URL, @@ -52,10 +74,11 @@ public class JdbcExternalCatalog extends ExternalCatalog { private transient JdbcClient jdbcClient; public JdbcExternalCatalog(long catalogId, String name, String resource, Map props, - String comment) + String comment, boolean isReplay) throws DdlException { super(catalogId, name, InitCatalogLog.Type.JDBC, comment); this.catalogProperty = new CatalogProperty(resource, processCompatibleProperties(props)); + testJdbcConnection(isReplay); } @Override @@ -73,6 +96,9 @@ public void checkProperties() throws DdlException { JdbcResource.checkBooleanProperty(JdbcResource.ONLY_SPECIFIED_DATABASE, getOnlySpecifiedDatabase()); JdbcResource.checkBooleanProperty(JdbcResource.LOWER_CASE_TABLE_NAMES, getLowerCaseTableNames()); + JdbcResource.checkBooleanProperty(JdbcResource.CONNECTION_POOL_KEEP_ALIVE, + String.valueOf(isConnectionPoolKeepAlive())); + JdbcResource.checkBooleanProperty(JdbcResource.TEST_CONNECTION, String.valueOf(isTestConnection())); JdbcResource.checkDatabaseListProperties(getOnlySpecifiedDatabase(), getIncludeDatabaseMap(), getExcludeDatabaseMap()); JdbcResource.checkConnectionPoolProperties(getConnectionPoolMinSize(), getConnectionPoolMaxSize(), @@ -177,6 +203,11 @@ public boolean isConnectionPoolKeepAlive() { .getDefaultPropertyValue(JdbcResource.CONNECTION_POOL_KEEP_ALIVE))); } + public boolean isTestConnection() { + return Boolean.parseBoolean(catalogProperty.getOrDefault(JdbcResource.TEST_CONNECTION, JdbcResource + .getDefaultPropertyValue(JdbcResource.TEST_CONNECTION))); + } + @Override protected void initLocalObjectsImpl() { JdbcClientConfig jdbcClientConfig = new JdbcClientConfig() @@ -244,4 +275,82 @@ public void setDefaultPropsWhenCreating(boolean isReplay) throws DdlException { } } } + + private void testJdbcConnection(boolean isReplay) throws DdlException { + if (FeConstants.runningUnitTest) { + // skip test connection in unit test + return; + } + if (!isReplay) { + if (isTestConnection()) { + try { + initLocalObjectsImpl(); + testFeToJdbcConnection(); + testBeToJdbcConnection(); + } finally { + jdbcClient.closeClient(); + jdbcClient = null; + } + } + } + } + + private void testFeToJdbcConnection() throws DdlException { + try { + jdbcClient.testConnection(); + } catch (JdbcClientException e) { + String errorMessage = "Test FE Connection to JDBC Failed: " + e.getMessage(); + LOG.error(errorMessage, e); + throw new DdlException(errorMessage, e); + } + } + + private void testBeToJdbcConnection() throws DdlException { + Backend aliveBe = null; + for (Backend be : Env.getCurrentSystemInfo().getIdToBackend().values()) { + if (be.isAlive()) { + aliveBe = be; + } + } + if (aliveBe == null) { + throw new DdlException("Test BE Connection to JDBC Failed: No Alive backends"); + } + TNetworkAddress address = new TNetworkAddress(aliveBe.getHost(), aliveBe.getBrpcPort()); + try { + JdbcTable jdbcTable = getTestConnectionJdbcTable(); + PJdbcTestConnectionRequest request = InternalService.PJdbcTestConnectionRequest.newBuilder() + .setJdbcTable(ByteString.copyFrom(new TSerializer().serialize(jdbcTable.toThrift()))) + .setJdbcTableType(jdbcTable.getJdbcTableType().getValue()) + .setQueryStr(jdbcClient.getTestQuery()).build(); + InternalService.PJdbcTestConnectionResult result = null; + Future future = BackendServiceProxy.getInstance() + .testJdbcConnection(address, request); + result = future.get(); + TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode()); + if (code != TStatusCode.OK) { + throw new DdlException("Test BE Connection to JDBC Failed: " + result.getStatus().getErrorMsgs(0)); + } + } catch (TException | RpcException | ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private JdbcTable getTestConnectionJdbcTable() throws DdlException { + JdbcTable jdbcTable = new JdbcTable(0, "test_jdbc_connection", Lists.newArrayList(), + TableType.JDBC_EXTERNAL_TABLE); + jdbcTable.setCatalogId(this.getId()); + jdbcTable.setJdbcTypeName(this.getDatabaseTypeName()); + jdbcTable.setJdbcUrl(this.getJdbcUrl()); + jdbcTable.setJdbcUser(this.getJdbcUser()); + jdbcTable.setJdbcPasswd(this.getJdbcPasswd()); + jdbcTable.setDriverClass(this.getDriverClass()); + jdbcTable.setDriverUrl(this.getDriverUrl()); + jdbcTable.setCheckSum(JdbcResource.computeObjectChecksum(this.getDriverUrl())); + jdbcTable.setConnectionPoolMinSize(this.getConnectionPoolMinSize()); + jdbcTable.setConnectionPoolMaxSize(this.getConnectionPoolMaxSize()); + jdbcTable.setConnectionPoolMaxLifeTime(this.getConnectionPoolMaxLifeTime()); + jdbcTable.setConnectionPoolMaxWaitTime(this.getConnectionPoolMaxWaitTime()); + jdbcTable.setConnectionPoolKeepAlive(this.isConnectionPoolKeepAlive()); + return jdbcTable; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index 7840e4cf0aee75d..1db27ef6eaf0176 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -478,4 +478,22 @@ protected Type createDecimalOrStringType(int precision, int scale) { } return ScalarType.createStringType(); } + + public void testConnection() { + String testQuery = getTestQuery(); + try (Connection conn = getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(testQuery)) { + if (!rs.next()) { + throw new JdbcClientException( + "Failed to test connection in FE: query executed but returned no results."); + } + } catch (SQLException e) { + throw new JdbcClientException("Failed to test connection in FE: " + e.getMessage(), e); + } + } + + public String getTestQuery() { + return "select 1"; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index d0a9f2c3de7ab3f..37fd1b6c72c2b07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -45,6 +45,11 @@ protected String getCatalogName(Connection conn) throws SQLException { return conn.getCatalog(); } + @Override + public String getTestQuery() { + return "SELECT 1 FROM dual"; + } + @Override public List getDatabaseNameList() { Connection conn = getConnection(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java index eb8742c6e809984..2df36b4cbaafe72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java @@ -36,6 +36,11 @@ protected String[] getTableTypes() { return new String[] {"TABLE", "VIEW", "OLAP VIEW", "JOIN VIEW", "HIERARCHY VIEW", "CALC VIEW"}; } + @Override + public String getTestQuery() { + return "SELECT 1 FROM DUMMY"; + } + @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { String hanaType = fieldSchema.getDataTypeName(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java index e4ece51146b4a1a..87321efb85b78e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java @@ -114,6 +114,11 @@ public Future fetchTableStructureAsync( return stub.fetchTableSchema(request); } + public Future testJdbcConnection( + InternalService.PJdbcTestConnectionRequest request) { + return stub.testJdbcConnection(request); + } + public Future updateCache(InternalService.PUpdateCacheRequest request) { return stub.updateCache(request); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java index 9b5c491df69b985..72afa75ffcc63d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java @@ -284,6 +284,18 @@ public Future fetchTableStructureAsync( } } + public Future testJdbcConnection( + TNetworkAddress address, InternalService.PJdbcTestConnectionRequest request) throws RpcException { + try { + final BackendServiceClient client = getProxy(address); + return client.testJdbcConnection(request); + } catch (Throwable e) { + LOG.warn("test jdbc connection catch a exception, address={}:{}", + address.getHostname(), address.getPort(), e); + throw new RpcException(address.hostname, e.getMessage()); + } + } + public Future updateCache( TNetworkAddress address, InternalService.PUpdateCacheRequest request) throws RpcException { try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java index 8394daf0682b6ca..4ef950ef9819a72 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java @@ -41,7 +41,7 @@ public void setUp() throws DdlException { properties.put(JdbcResource.DRIVER_URL, "ojdbc8.jar"); properties.put(JdbcResource.JDBC_URL, "jdbc:oracle:thin:@127.0.0.1:1521:XE"); properties.put(JdbcResource.DRIVER_CLASS, "oracle.jdbc.driver.OracleDriver"); - jdbcExternalCatalog = new JdbcExternalCatalog(1L, "testCatalog", null, properties, "testComment"); + jdbcExternalCatalog = new JdbcExternalCatalog(1L, "testCatalog", null, properties, "testComment", false); } @Test diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 1ef97ec2df387f5..a1fd0e42a70b199 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -614,6 +614,16 @@ message PFetchTableSchemaResult { repeated PTypeDesc column_types = 4; } +message PJdbcTestConnectionRequest { + optional bytes jdbc_table = 1; + optional int32 jdbc_table_type = 2; + optional string query_str = 3; +} + +message PJdbcTestConnectionResult { + optional PStatus status = 1; +} + message PRowLocation { optional int64 tablet_id = 1; optional string rowset_id = 2; @@ -727,5 +737,6 @@ service PBackendService { rpc get_column_ids_by_tablet_ids(PFetchColIdsRequest) returns (PFetchColIdsResponse); rpc get_tablet_rowset_versions(PGetTabletVersionsRequest) returns (PGetTabletVersionsResponse); rpc glob(PGlobRequest) returns (PGlobResponse); + rpc test_jdbc_connection(PJdbcTestConnectionRequest) returns (PJdbcTestConnectionResult); }; diff --git a/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.out index 32c74765aabc1edb15711d03fc09f0884f77e1aa..65b532901165b0ac304c624cb5d6c98115a17089 100644 GIT binary patch delta 33 ncmeBD->bEuOITD_S3$8PwYVfcIX^ECNE8*NCYNkXpT`IQ$@dIv delta 7 Ocmdn1)uq0nOBetQvjXM- diff --git a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy index 9948c49d24a55b4..ff89e1b9a300ac2 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy @@ -101,6 +101,32 @@ suite("test_clickhouse_jdbc_catalog", "p0,external,clickhouse,external_docker,ex order_qt_dt_with_tz """ select * from dt_with_tz order by id; """ + sql """create catalog if not exists clickhouse_catalog_test_conn_correct properties( + "type"="jdbc", + "user"="default", + "password"="123456", + "jdbc_url" = "jdbc:clickhouse://${externalEnvIp}:${clickhouse_port}/doris_test", + "driver_url" = "${driver_url}", + "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver", + "test_connection" = "true" + ); + """ + order_qt_test_conn_correct """ select * from clickhouse_catalog_test_conn_correct.doris_test.type; """ + + test { + sql """create catalog if not exists clickhouse_catalog_test_conn_mistake properties( + "type"="jdbc", + "user"="default", + "password"="1234567", + "jdbc_url" = "jdbc:clickhouse://${externalEnvIp}:${clickhouse_port}/doris_test", + "driver_url" = "${driver_url}", + "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver", + "test_connection" = "true" + ); + """ + exception "Test FE Connection to JDBC Failed: Can not connect to jdbc due to error: Code: 516. DB::Exception: default: Authentication failed: password is incorrect, or there is no user with such name." + } + }finally { res_dbs_log = sql "show databases;" for(int i = 0;i < res_dbs_log.size();i++) { @@ -108,7 +134,9 @@ suite("test_clickhouse_jdbc_catalog", "p0,external,clickhouse,external_docker,ex log.info( "database = ${res_dbs_log[i][0]} => tables = "+tbs.toString()) } } - + sql """ drop catalog if exists ${catalog_name} """ + sql """ drop catalog if exists clickhouse_catalog_test_conn_correct """ + sql """ drop catalog if exists clickhouse_catalog_test_conn_mistake """ } } From d3b4193e96891f667c423a1dab02451d8eb9dce0 Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:09:44 +0800 Subject: [PATCH 03/14] [fix](nereids)str_to_date function's signature for folding constant is wrong #32474 (#32478) --- .../executable/DateTimeExtractAndTransform.java | 11 ++++++++--- .../partition_prune/test_date_function_prune.groovy | 5 +++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java index 8b51ceeb177626c..e458b975475bf43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java @@ -570,10 +570,15 @@ public static Expression makeDate(IntegerLiteral year, IntegerLiteral dayOfYear) /** * date transformation function: str_to_date */ - @ExecFunction(name = "str_to_date", argTypes = {"VARCHAR, VARCHAR"}, returnType = "DATETIME") + @ExecFunction(name = "str_to_date", argTypes = {"VARCHAR", "VARCHAR"}, returnType = "DATETIMEV2") public static Expression strToDate(VarcharLiteral str, VarcharLiteral format) { - return DateTimeLiteral.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue()) - .toFormatter(), str.getValue())); + if (org.apache.doris.analysis.DateLiteral.hasTimePart(format.getStringValue())) { + return DateTimeV2Literal.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue()) + .toFormatter(), str.getValue())); + } else { + return DateV2Literal.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue()) + .toFormatter(), str.getValue())); + } } @ExecFunction(name = "timestamp", argTypes = {"DATETIME"}, returnType = "DATETIME") diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_function_prune.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_function_prune.groovy index 1523bbb662f2290..cd2a509fa072c9a 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_function_prune.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_function_prune.groovy @@ -74,4 +74,9 @@ suite("test_date_function_prune") { sql "select * from dp where Date(date_time) in ('2020-01-01', '2020-01-03')" contains("partitions=2/3 (p1,p3)") } + + explain { + sql "select * from dp where date_time > str_to_date('2020-01-02','%Y-%m-%d')" + contains("partitions=2/3 (p2,p3)") + } } \ No newline at end of file From a4eb078cbaf62c184a858009613f6dc99edbc2ea Mon Sep 17 00:00:00 2001 From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:35:57 +0800 Subject: [PATCH 04/14] [branch-2.0](routine-load) enhance auto resume to keep routine load stable (#32590) --- .../apache/doris/common/InternalErrorCode.java | 3 ++- .../doris/load/routineload/RoutineLoadJob.java | 6 +++--- .../doris/load/routineload/ScheduleRule.java | 15 ++++++++++----- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/InternalErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/InternalErrorCode.java index 2bbd5c58efa02b8..b871fd198cbd715 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/InternalErrorCode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/InternalErrorCode.java @@ -34,7 +34,8 @@ public enum InternalErrorCode { MANUAL_STOP_ERR(101), TOO_MANY_FAILURE_ROWS_ERR(102), CREATE_TASKS_ERR(103), - TASKS_ABORT_ERR(104); + TASKS_ABORT_ERR(104), + CANNOT_RESUME_ERR(105); private long errCode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 0d9ae5163515978..a55f0a02124c970 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -1133,7 +1133,7 @@ public void afterAborted(TransactionState txnState, boolean txnOperated, String + " with reason: " + txnStatusChangeReasonString + " please check the jsonpaths"; updateState(JobState.PAUSED, - new ErrorReason(InternalErrorCode.TASKS_ABORT_ERR, msg), + new ErrorReason(InternalErrorCode.CANNOT_RESUME_ERR, msg), false /* not replay */); return; case OFFSET_OUT_OF_RANGE: @@ -1146,14 +1146,14 @@ public void afterAborted(TransactionState txnState, boolean txnOperated, String + " using the Alter ROUTINE LOAD command to modify it," + " and resume the job"; updateState(JobState.PAUSED, - new ErrorReason(InternalErrorCode.TASKS_ABORT_ERR, msg), + new ErrorReason(InternalErrorCode.CANNOT_RESUME_ERR, msg), false /* not replay */); return; case PAUSE: msg = "be " + taskBeId + " abort task " + "with reason: " + txnStatusChangeReasonString; updateState(JobState.PAUSED, - new ErrorReason(InternalErrorCode.TASKS_ABORT_ERR, msg), + new ErrorReason(InternalErrorCode.CANNOT_RESUME_ERR, msg), false /* not replay */); return; default: diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/ScheduleRule.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/ScheduleRule.java index a0aab5c1cb91e40..052f22bf3de3fd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/ScheduleRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/ScheduleRule.java @@ -55,11 +55,16 @@ public static boolean isNeedAutoSchedule(RoutineLoadJob jobRoutine) { /* * Handle all backends are down. */ - LOG.debug("try to auto reschedule routine load {}, firstResumeTimestamp: {}, autoResumeCount: {}, " - + "pause reason: {}", - jobRoutine.id, jobRoutine.firstResumeTimestamp, jobRoutine.autoResumeCount, - jobRoutine.pauseReason == null ? "null" : jobRoutine.pauseReason.getCode().name()); - if (jobRoutine.pauseReason != null && jobRoutine.pauseReason.getCode() == InternalErrorCode.REPLICA_FEW_ERR) { + if (LOG.isDebugEnabled()) { + LOG.debug("try to auto reschedule routine load {}, firstResumeTimestamp: {}, autoResumeCount: {}, " + + "pause reason: {}", + jobRoutine.id, jobRoutine.firstResumeTimestamp, jobRoutine.autoResumeCount, + jobRoutine.pauseReason == null ? "null" : jobRoutine.pauseReason.getCode().name()); + } + if (jobRoutine.pauseReason != null + && jobRoutine.pauseReason.getCode() != InternalErrorCode.MANUAL_PAUSE_ERR + && jobRoutine.pauseReason.getCode() != InternalErrorCode.TOO_MANY_FAILURE_ROWS_ERR + && jobRoutine.pauseReason.getCode() != InternalErrorCode.CANNOT_RESUME_ERR) { int dead = deadBeCount(jobRoutine.clusterName); if (dead > Config.max_tolerable_backend_down_num) { LOG.debug("dead backend num {} is larger than config {}, " From e247a801aae9852e5123a5c7e4209cabf946bba6 Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:45:36 +0800 Subject: [PATCH 05/14] Improve analyze stats case, avoid cluster delay caused failure. (#32507) (#32605) --- .../suites/statistics/analyze_stats.groovy | 55 +++++++++---------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy index 1fa86e52b02e233..9773a841e66cba4 100644 --- a/regression-test/suites/statistics/analyze_stats.groovy +++ b/regression-test/suites/statistics/analyze_stats.groovy @@ -2722,25 +2722,36 @@ PARTITION `p599` VALUES IN (599) assertEquals("0.0", result_sample[1][2]) assertEquals("SAMPLE", result_sample[1][9]) - // Test show task - result_sample = sql """analyze table trigger_test with sample percent 10""" - String jobId = result_sample[0][0] - result_sample = sql """show analyze task status ${jobId}""" - assertEquals(2, result_sample.size()) - Thread.sleep(1000) - sql """drop stats trigger_test""" - // Test trigger type + sql """drop stats trigger_test""" sql """insert into trigger_test values(1,'name1') """ sql """insert into trigger_test values(2,'name2') """ sql """insert into trigger_test values(3,'name3') """ sql """insert into trigger_test values(4,'name4') """ - sql """analyze database trigger PROPERTIES("use.auto.analyzer"="true")""" + // Test analyze default full. + sql """analyze table trigger_test with sync""" + def result = sql """show column stats trigger_test""" + assertEquals(2, result.size()) + assertEquals("4.0", result[0][2]) + assertEquals("FULL", result[0][9]) + assertEquals("4.0", result[1][2]) + assertEquals("FULL", result[1][9]) + + // Test manual analyze ignore health value + sql """insert into trigger_test values(5,'name5') """ + sql """analyze table trigger_test with sync""" + result = sql """show column stats trigger_test""" + assertEquals(2, result.size()) + assertEquals("5.0", result[0][2]) + assertEquals("5.0", result[1][2]) + // Test auto analyze with job type SYSTEM + sql """drop stats trigger_test""" + sql """analyze database trigger PROPERTIES("use.auto.analyzer"="true")""" int i = 0; for (0; i < 10; i++) { - def result = sql """show column stats trigger_test""" + result = sql """show column stats trigger_test""" if (result.size() != 2) { Thread.sleep(1000) continue; @@ -2751,29 +2762,17 @@ PARTITION `p599` VALUES IN (599) } if (i < 10) { sql """analyze table trigger_test with sync""" - def result = sql """show column stats trigger_test""" + result = sql """show column stats trigger_test""" assertEquals(result.size(), 2) assertEquals(result[0][11], "MANUAL") assertEquals(result[1][11], "MANUAL") } - // Test analyze default full. - sql """analyze table trigger_test with sync""" - def result = sql """show column stats trigger_test""" - assertEquals(2, result.size()) - assertEquals("4.0", result[0][2]) - assertEquals("FULL", result[0][9]) - assertEquals("4.0", result[1][2]) - assertEquals("FULL", result[1][9]) - - // Test analyze hive health value - sql """insert into trigger_test values(5,'name5') """ - sql """analyze table trigger_test with sync""" - result = sql """show column stats trigger_test""" - assertEquals(2, result.size()) - assertEquals("5.0", result[0][2]) - assertEquals("5.0", result[1][2]) - + // Test show task + result_sample = sql """analyze table trigger_test with sample percent 10""" + String jobId = result_sample[0][0] + result_sample = sql """show analyze task status ${jobId}""" + assertEquals(2, result_sample.size()) sql """DROP DATABASE IF EXISTS trigger""" } From a29fb5a6074d4ab3855248453fac296e475729d2 Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Thu, 21 Mar 2024 17:25:15 +0800 Subject: [PATCH 06/14] [fix](jdbc catalog) Fix query errors without jdbc pool default value on only BE upgrade (#32619) --- .../org/apache/doris/jdbc/JdbcDataSourceConfig.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSourceConfig.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSourceConfig.java index dcf576986fef538..5fdbc211ab01613 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSourceConfig.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSourceConfig.java @@ -30,11 +30,11 @@ public class JdbcDataSourceConfig { private int batchSize; private TJdbcOperation op; private TOdbcTableType tableType; - private int connectionPoolMinSize; - private int connectionPoolMaxSize; - private int connectionPoolMaxWaitTime; - private int connectionPoolMaxLifeTime; - private boolean connectionPoolKeepAlive; + private int connectionPoolMinSize = 1; + private int connectionPoolMaxSize = 10; + private int connectionPoolMaxWaitTime = 5000; + private int connectionPoolMaxLifeTime = 1800000; + private boolean connectionPoolKeepAlive = false; public String createCacheKey() { return catalogId + jdbcUrl + jdbcUser + jdbcPassword + jdbcDriverUrl + jdbcDriverClass From f2ad149e899bbed4e152bc8034823b03d9b5fc24 Mon Sep 17 00:00:00 2001 From: Kang Date: Thu, 21 Mar 2024 17:45:37 +0800 Subject: [PATCH 07/14] [fix](inverted index) skip read index column data only for DUP and MOW table #32594 (#32600) --- be/src/olap/iterators.h | 1 + be/src/olap/rowset/beta_rowset_reader.cpp | 2 + .../rowset/segment_v2/segment_iterator.cpp | 6 + .../test_index_skip_read_data.out | 83 +++++++++++ .../test_index_skip_read_data.groovy | 131 ++++++++++++++++++ 5 files changed, 223 insertions(+) create mode 100644 regression-test/data/inverted_index_p0/test_index_skip_read_data.out create mode 100644 regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h index f1b195f8f98d26b..3b9d205e83d468f 100644 --- a/be/src/olap/iterators.h +++ b/be/src/olap/iterators.h @@ -92,6 +92,7 @@ class StorageReadOptions { int block_row_max = 4096 - 32; // see https://github.com/apache/doris/pull/11816 TabletSchemaSPtr tablet_schema = nullptr; + bool enable_unique_key_merge_on_write = false; bool record_rowids = false; // flag for enable topn opt bool use_topn_opt = false; diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index 0f8ee1715621dba..bb11347990c5cb2 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -215,6 +215,8 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context } _read_options.use_page_cache = _read_context->use_page_cache; _read_options.tablet_schema = _read_context->tablet_schema; + _read_options.enable_unique_key_merge_on_write = + _read_context->enable_unique_key_merge_on_write; _read_options.record_rowids = _read_context->record_rowids; _read_options.use_topn_opt = _read_context->use_topn_opt; _read_options.read_orderby_key_reverse = _read_context->read_orderby_key_reverse; diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 984d118ddf5ea5d..80fb6f7856c6355 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -1044,6 +1044,12 @@ Status SegmentIterator::_apply_inverted_index_on_block_column_predicate( } bool SegmentIterator::_need_read_data(ColumnId cid) { + // only support DUP_KEYS and UNIQUE_KEYS with MOW + if (!((_opts.tablet_schema->keys_type() == KeysType::DUP_KEYS || + (_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS && + _opts.enable_unique_key_merge_on_write)))) { + return true; + } // if there is delete predicate, we always need to read data if (_opts.delete_condition_predicates->num_of_column_predicate() > 0) { return true; diff --git a/regression-test/data/inverted_index_p0/test_index_skip_read_data.out b/regression-test/data/inverted_index_p0/test_index_skip_read_data.out new file mode 100644 index 000000000000000..bb79d437b2a4929 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_index_skip_read_data.out @@ -0,0 +1,83 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql10 -- +1 20 300 +1 20 400 + +-- !sql11 -- +1 20 300 +1 20 400 + +-- !sql12 -- +1 20 300 +1 20 400 + +-- !sql13 -- +1 20 300 + +-- !sql14 -- +1 20 400 + +-- !sql15 -- +20 +20 + +-- !sql16 -- +1 +1 + +-- !sql17 -- +1 20 + +-- !sql18 -- +1 20 + +-- !sql20 -- +1 20 400 + +-- !sql21 -- +1 20 400 + +-- !sql22 -- +1 20 400 + +-- !sql23 -- + +-- !sql24 -- +1 20 400 + +-- !sql25 -- +20 + +-- !sql26 -- +1 + +-- !sql27 -- + +-- !sql28 -- +1 20 + +-- !sql30 -- +1 20 400 + +-- !sql31 -- +1 20 400 + +-- !sql32 -- +1 20 400 + +-- !sql33 -- + +-- !sql34 -- +1 20 400 + +-- !sql35 -- +20 + +-- !sql36 -- +1 + +-- !sql37 -- + +-- !sql38 -- +1 20 + diff --git a/regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy b/regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy new file mode 100644 index 000000000000000..70213910934d3ae --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_index_skip_read_data", "p0"){ + def indexTbName1 = "test_index_skip_read_data_dup" + def indexTbName2 = "test_index_skip_read_data_mow" + def indexTbName3 = "test_index_skip_read_data_mor" + + + // dup + sql "DROP TABLE IF EXISTS ${indexTbName1}" + + sql """ + CREATE TABLE ${indexTbName1} ( + `k1` int(11) NULL COMMENT "", + `k2` varchar(20) NULL COMMENT "", + `data` text NULL COMMENT "", + INDEX idx_k1 (`k1`) USING INVERTED COMMENT '', + INDEX idx_k2 (`k2`) USING BITMAP COMMENT '', + INDEX idx_data (`data`) USING INVERTED COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ); + """ + + sql """ INSERT INTO ${indexTbName1} VALUES (1, 20, 300); """ + sql """ INSERT INTO ${indexTbName1} VALUES (1, 20, 400); """ + + qt_sql10 """ SELECT * FROM ${indexTbName1} ORDER BY k1,k2,data; """ + qt_sql11 """ SELECT * FROM ${indexTbName1} WHERE k1 = 1 ORDER BY k1,k2,data; """ + qt_sql12 """ SELECT * FROM ${indexTbName1} WHERE k2 = 20 ORDER BY k1,k2,data; """ + qt_sql13 """ SELECT * FROM ${indexTbName1} WHERE data = 300 ORDER BY k1,k2,data; """ + qt_sql14 """ SELECT * FROM ${indexTbName1} WHERE data = 400 ORDER BY k1,k2,data; """ + qt_sql15 """ SELECT k2 FROM ${indexTbName1} WHERE k1 = 1 ORDER BY k1,k2,data; """ + qt_sql16 """ SELECT k1 FROM ${indexTbName1} WHERE k2 = 20 ORDER BY k1,k2,data; """ + qt_sql17 """ SELECT k1, k2 FROM ${indexTbName1} WHERE data = 300 ORDER BY k1,k2,data; """ + qt_sql18 """ SELECT k1, k2 FROM ${indexTbName1} WHERE data = 400 ORDER BY k1,k2,data; """ + + + + // mow + sql "DROP TABLE IF EXISTS ${indexTbName2}" + + sql """ + CREATE TABLE ${indexTbName2} ( + `k1` int(11) NULL COMMENT "", + `k2` varchar(20) NULL COMMENT "", + `data` text NULL COMMENT "", + INDEX idx_k1 (`k1`) USING INVERTED COMMENT '', + INDEX idx_k2 (`k2`) USING BITMAP COMMENT '', + INDEX idx_data (`data`) USING INVERTED COMMENT '' + ) ENGINE=OLAP + UNIQUE KEY(`k1`, `k2`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true" + ); + """ + + sql """ INSERT INTO ${indexTbName2} VALUES (1, 20, 300); """ + sql """ INSERT INTO ${indexTbName2} VALUES (1, 20, 400); """ + + qt_sql20 """ SELECT * FROM ${indexTbName2} ORDER BY k1,k2,data; """ + qt_sql21 """ SELECT * FROM ${indexTbName2} WHERE k1 = 1 ORDER BY k1,k2,data; """ + qt_sql22 """ SELECT * FROM ${indexTbName2} WHERE k2 = 20 ORDER BY k1,k2,data; """ + qt_sql23 """ SELECT * FROM ${indexTbName2} WHERE data = 300 ORDER BY k1,k2,data; """ + qt_sql24 """ SELECT * FROM ${indexTbName2} WHERE data = 400 ORDER BY k1,k2,data; """ + qt_sql25 """ SELECT k2 FROM ${indexTbName2} WHERE k1 = 1 ORDER BY k1,k2,data; """ + qt_sql26 """ SELECT k1 FROM ${indexTbName2} WHERE k2 = 20 ORDER BY k1,k2,data; """ + qt_sql27 """ SELECT k1, k2 FROM ${indexTbName2} WHERE data = 300 ORDER BY k1,k2,data; """ + qt_sql28 """ SELECT k1, k2 FROM ${indexTbName2} WHERE data = 400 ORDER BY k1,k2,data; """ + + + // mor + sql "DROP TABLE IF EXISTS ${indexTbName3}" + + sql """ + CREATE TABLE ${indexTbName3} ( + `k1` int(11) NULL COMMENT "", + `k2` varchar(20) NULL COMMENT "", + `data` text NULL COMMENT "", + INDEX idx_k1 (`k1`) USING INVERTED COMMENT '', + INDEX idx_k2 (`k2`) USING BITMAP COMMENT '', + INDEX idx_data (`data`) USING INVERTED COMMENT '' + ) ENGINE=OLAP + UNIQUE KEY(`k1`, `k2`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "enable_unique_key_merge_on_write" = "false", + "disable_auto_compaction" = "true" + ); + """ + + sql """ INSERT INTO ${indexTbName3} VALUES (1, 20, 300); """ + sql """ INSERT INTO ${indexTbName3} VALUES (1, 20, 400); """ + + qt_sql30 """ SELECT * FROM ${indexTbName3} ORDER BY k1,k2,data; """ + qt_sql31 """ SELECT * FROM ${indexTbName3} WHERE k1 = 1 ORDER BY k1,k2,data; """ + qt_sql32 """ SELECT * FROM ${indexTbName3} WHERE k2 = 20 ORDER BY k1,k2,data; """ + qt_sql33 """ SELECT * FROM ${indexTbName3} WHERE data = 300 ORDER BY k1,k2,data; """ + qt_sql34 """ SELECT * FROM ${indexTbName3} WHERE data = 400 ORDER BY k1,k2,data; """ + qt_sql35 """ SELECT k2 FROM ${indexTbName3} WHERE k1 = 1 ORDER BY k1,k2,data; """ + qt_sql36 """ SELECT k1 FROM ${indexTbName3} WHERE k2 = 20 ORDER BY k1,k2,data; """ + qt_sql37 """ SELECT k1, k2 FROM ${indexTbName3} WHERE data = 300 ORDER BY k1,k2,data; """ + qt_sql38 """ SELECT k1, k2 FROM ${indexTbName3} WHERE data = 400 ORDER BY k1,k2,data; """ +} \ No newline at end of file From d11c4658f2cdf8991098f795d9cd36e8d1baa150 Mon Sep 17 00:00:00 2001 From: zhengyu Date: Thu, 21 Mar 2024 17:50:08 +0800 Subject: [PATCH 08/14] [fix](analysis) sorted partition when do sample analyze (#32606) Signed-off-by: freemandealer --- .../java/org/apache/doris/statistics/OlapAnalysisTask.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java index 0a99f6b32de69ad..e6dd46e9fc74e2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -250,7 +251,9 @@ protected Pair, Long> calcActualSampleTablets(boolean forPartitionCol List sampleTabletIds = new ArrayList<>(); long actualSampledRowCount = 0; boolean enough = false; - for (Partition p : olapTable.getPartitions()) { + List sortedPartitions = olapTable.getPartitions().stream().sorted( + Comparator.comparing(Partition::getName)).collect(Collectors.toList()); + for (Partition p : sortedPartitions) { MaterializedIndex materializedIndex = info.indexId == -1 ? p.getBaseIndex() : p.getIndex(info.indexId); if (materializedIndex == null) { continue; From d77b8ceddfb2eddd12ce6b25395b68658a534ad4 Mon Sep 17 00:00:00 2001 From: AlexYue Date: Thu, 21 Mar 2024 17:50:23 +0800 Subject: [PATCH 09/14] Pick "[enhance](S3) Print the oss request id for each error s3 request #32499" (#32607) --- be/src/io/fs/s3_file_reader.cpp | 9 +++++---- be/src/io/fs/s3_file_system.cpp | 4 ++-- be/src/io/fs/s3_file_writer.cpp | 26 ++++++++++++++++---------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index ceebc683a9419e3..29faec47d8eb188 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -96,10 +96,11 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea } auto outcome = client->GetObject(request); if (!outcome.IsSuccess()) { - return Status::IOError("failed to read from {}: {}, exception {}, error code {}", - _path.native(), outcome.GetError().GetMessage(), - outcome.GetError().GetExceptionName(), - outcome.GetError().GetResponseCode()); + return Status::IOError( + "failed to read from {}: {}, exception {}, error code {}, request id {}", + _path.native(), outcome.GetError().GetMessage(), + outcome.GetError().GetExceptionName(), outcome.GetError().GetResponseCode(), + outcome.GetError().GetRequestId()); } *bytes_read = outcome.GetResult().GetContentLength(); if (*bytes_read != bytes_req) { diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index cad49b4555c1b63..6423de4fa15869a 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -526,10 +526,10 @@ Status S3FileSystem::get_key(const Path& path, std::string* key) const { template std::string S3FileSystem::error_msg(const std::string& key, const AwsOutcome& outcome) const { - return fmt::format("(endpoint: {}, bucket: {}, key:{}, {}), {}, error code {}", + return fmt::format("(endpoint: {}, bucket: {}, key:{}, {}), {}, error code {}, request id {}", _s3_conf.endpoint, _s3_conf.bucket, key, outcome.GetError().GetExceptionName(), outcome.GetError().GetMessage(), - outcome.GetError().GetResponseCode()); + outcome.GetError().GetResponseCode(), outcome.GetError().GetRequestId()); } std::string S3FileSystem::error_msg(const std::string& key, const std::string& err) const { diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 39a8441f87bf4d8..1679385165de6b8 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -120,9 +120,10 @@ Status S3FileWriter::_create_multi_upload_request() { } return Status::IOError( "failed to create multipart upload(bucket={}, key={}, upload_id={}): {}, exception {}, " - "error code {}", + "error code {}, request id {}", _bucket, _path.native(), _upload_id, outcome.GetError().GetMessage(), - outcome.GetError().GetExceptionName(), outcome.GetError().GetResponseCode()); + outcome.GetError().GetExceptionName(), outcome.GetError().GetResponseCode(), + outcome.GetError().GetRequestId()); } void S3FileWriter::_wait_until_finish(std::string_view task_name) { @@ -174,9 +175,10 @@ Status S3FileWriter::abort() { } return Status::IOError( "failed to abort multipart upload(bucket={}, key={}, upload_id={}): {}, exception {}, " - "error code {}", + "error code {}, request id {}", _bucket, _path.native(), _upload_id, outcome.GetError().GetMessage(), - outcome.GetError().GetExceptionName(), outcome.GetError().GetResponseCode()); + outcome.GetError().GetExceptionName(), outcome.GetError().GetResponseCode(), + outcome.GetError().GetRequestId()); } Status S3FileWriter::close() { @@ -306,11 +308,12 @@ void S3FileWriter::_upload_one_part(int64_t part_num, S3FileBuffer& buf) { if (!upload_part_outcome.IsSuccess()) { auto s = Status::IOError( "failed to upload part (bucket={}, key={}, part_num={}, up_load_id={}): {}, " - "exception {}, error code {}", + "exception {}, error code {}, request id {}", _bucket, _path.native(), part_num, _upload_id, upload_part_outcome.GetError().GetMessage(), upload_part_outcome.GetError().GetExceptionName(), - upload_part_outcome.GetError().GetResponseCode()); + upload_part_outcome.GetError().GetResponseCode(), + upload_part_outcome.GetError().GetRequestId()); LOG_WARNING(s.to_string()); buf._on_failed(s); return; @@ -357,10 +360,11 @@ Status S3FileWriter::_complete() { if (!compute_outcome.IsSuccess()) { auto s = Status::IOError( "failed to create complete multi part upload (bucket={}, key={}): {}, exception " - "{}, error code {}", + "{}, error code {}, request id {}", _bucket, _path.native(), compute_outcome.GetError().GetMessage(), compute_outcome.GetError().GetExceptionName(), - compute_outcome.GetError().GetResponseCode()); + compute_outcome.GetError().GetResponseCode(), + compute_outcome.GetError().GetRequestId()); LOG_WARNING(s.to_string()); return s; } @@ -399,9 +403,11 @@ void S3FileWriter::_put_object(S3FileBuffer& buf) { auto response = _client->PutObject(request); if (!response.IsSuccess()) { _st = Status::InternalError( - "failed to put object (bucket={}, key={}), Error: [{}:{}, responseCode:{}]", + "failed to put object (bucket={}, key={}), Error: [{}:{}, responseCode:{}, request " + "id:{}]", _bucket, _path.native(), response.GetError().GetExceptionName(), - response.GetError().GetMessage(), response.GetError().GetResponseCode()); + response.GetError().GetMessage(), response.GetError().GetResponseCode(), + response.GetError().GetRequestId()); LOG(WARNING) << _st; buf._on_failed(_st); return; From 7b904cf6185ef48d4d0cc07025bd9671093f1b0b Mon Sep 17 00:00:00 2001 From: XuJianxu Date: Thu, 21 Mar 2024 17:57:39 +0800 Subject: [PATCH 10/14] [fix][regression] update ccr test framework (#32565) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 胥剑旭 --- .../org/apache/doris/regression/Config.groovy | 50 +++++++++++++++++++ .../doris/regression/suite/Suite.groovy | 35 ++++++++++++- .../regression/suite/SuiteContext.groovy | 17 +++++-- .../regression/suite/SyncerContext.groovy | 6 +-- .../pipeline/p0/conf/regression-conf.groovy | 5 ++ .../pipeline/p1/conf/regression-conf.groovy | 5 ++ 6 files changed, 108 insertions(+), 10 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index 4fc3705a09e01e4..cf6e112260cc275 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -41,6 +41,10 @@ class Config { public String jdbcPassword public String defaultDb + public String ccrDownstreamUrl + public String ccrDownstreamUser + public String ccrDownstreamPassword + public String feSourceThriftAddress public String feTargetThriftAddress public String feSyncerUser @@ -314,6 +318,9 @@ class Config { configToString(obj.sslCertificatePath) ) + config.ccrDownstreamUrl = configToString(obj.ccrDownstreamUrl) + config.ccrDownstreamUser = configToString(obj.ccrDownstreamUser) + config.ccrDownstreamPassword = configToString(obj.ccrDownstreamPassword) config.image = configToString(obj.image) config.dockerEndDeleteFiles = configToBoolean(obj.dockerEndDeleteFiles) config.excludeDockerTest = configToBoolean(obj.excludeDockerTest) @@ -543,6 +550,49 @@ class Config { return DriverManager.getConnection(dbUrl, jdbcUser, jdbcPassword) } + public static String buildUrlWithDbImpl(String jdbcUrl, String dbName) { + String urlWithDb = jdbcUrl + String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3) + if (urlWithoutSchema.indexOf("/") >= 0) { + if (jdbcUrl.contains("?")) { + // e.g: jdbc:mysql://locahost:8080/?a=b + urlWithDb = jdbcUrl.substring(0, jdbcUrl.lastIndexOf("?")) + urlWithDb = urlWithDb.substring(0, urlWithDb.lastIndexOf("/")) + urlWithDb += ("/" + dbName) + jdbcUrl.substring(jdbcUrl.lastIndexOf("?")) + } else { + // e.g: jdbc:mysql://locahost:8080/ + urlWithDb += dbName + } + } else { + // e.g: jdbc:mysql://locahost:8080 + urlWithDb += ("/" + dbName) + } + + return urlWithDb + } + + Connection getConnectionByArrowFlightSql(String dbName) { + Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver") + String arrowFlightSqlHost = otherConfigs.get("extArrowFlightSqlHost") + String arrowFlightSqlPort = otherConfigs.get("extArrowFlightSqlPort") + String arrowFlightSqlUrl = "jdbc:arrow-flight-sql://${arrowFlightSqlHost}:${arrowFlightSqlPort}" + + "/?useServerPrepStmts=false&useSSL=false&useEncryption=false" + // TODO jdbc:arrow-flight-sql not support connect db + String dbUrl = buildUrlWithDbImpl(arrowFlightSqlUrl, dbName) + tryCreateDbIfNotExist(dbName) + log.info("connect to ${dbUrl}".toString()) + String arrowFlightSqlJdbcUser = otherConfigs.get("extArrowFlightSqlUser") + String arrowFlightSqlJdbcPassword = otherConfigs.get("extArrowFlightSqlPassword") + return DriverManager.getConnection(dbUrl, arrowFlightSqlJdbcUser, arrowFlightSqlJdbcPassword) + } + + Connection getDownstreamConnectionByDbName(String dbName) { + String dbUrl = buildUrlWithDb(ccrDownstreamUrl, dbName) + tryCreateDbIfNotExist(dbName) + log.info("connect to ${dbUrl}".toString()) + return DriverManager.getConnection(dbUrl, ccrDownstreamUser, ccrDownstreamPassword) + } + String getDbNameByFile(File suiteFile) { String dir = new File(suitePath).relativePath(suiteFile.parentFile) // We put sql files under sql dir, so dbs and tables used by cases diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 9ba50881b88ef8f..4bc8beab8094a73 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -51,6 +51,7 @@ import java.util.stream.Collectors import java.util.stream.LongStream import static org.apache.doris.regression.util.DataUtils.sortByToString +import java.sql.Connection import java.sql.DriverManager import java.sql.PreparedStatement import java.sql.ResultSetMetaData @@ -259,9 +260,34 @@ class Suite implements GroovyInterceptable { return result } - def sql_return_maparray(String sqlStr) { + List> insert_into_sql_impl(Connection conn, String sqlStr, int num) { + logger.info("insert into " + num + " records") + def (result, meta) = JdbcUtils.executeToList(conn, sqlStr) + return result + } + + List> jdbc_insert_into_sql(String sqlStr, int num) { + return insert_into_sql_impl(context.getConnection(), sqlStr, num) + } + + List> arrow_flight_insert_into_sql(String sqlStr, int num) { + return insert_into_sql_impl(context.getArrowFlightSqlConnection(), (String) ("USE ${context.dbName};" + sqlStr), num) + } + + List> insert_into_sql(String sqlStr, int num) { + if (context.useArrowFlightSql()) { + return arrow_flight_insert_into_sql(sqlStr, num) + } else { + return jdbc_insert_into_sql(sqlStr, num) + } + } + + def sql_return_maparray(String sqlStr, Connection conn = null) { logger.info("Execute sql: ${sqlStr}".toString()) - def (result, meta) = JdbcUtils.executeToList(context.getConnection(), sqlStr) + if (conn == null) { + conn = context.getConnection() + } + def (result, meta) = JdbcUtils.executeToList(conn, sqlStr) // get all column names as list List columnNames = new ArrayList<>() @@ -516,6 +542,11 @@ class Suite implements GroovyInterceptable { return lines; } + + Connection getTargetConnection() { + return context.getTargetConnection(this) + } + boolean deleteFile(String filePath) { def file = new File(filePath) file.delete() diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy index 9cc21faae0777c5..b2a2cb7ba1e3933 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy @@ -152,9 +152,14 @@ class SuiteContext implements Closeable { return subJdbc.substring(0, subJdbc.indexOf("/")) } - private Map getSpec() { + private String getDownstreamJdbcNetInfo() { + String subJdbc = config.ccrDownstreamUrl.substring(config.ccrDownstreamUrl.indexOf("://") + 3) + return subJdbc.substring(0, subJdbc.indexOf("/")) + } + + private Map getSpec(String[] jdbc) { Map spec = Maps.newHashMap() - String[] jdbc = getJdbcNetInfo().split(":") + spec.put("host", jdbc[0]) spec.put("port", jdbc[1]) spec.put("user", config.feSyncerUser) @@ -165,7 +170,8 @@ class SuiteContext implements Closeable { } Map getSrcSpec() { - Map spec = getSpec() + String[] jdbc = getJdbcNetInfo().split(":") + Map spec = getSpec(jdbc) spec.put("thrift_port", config.feSourceThriftNetworkAddress.port.toString()) spec.put("database", dbName) @@ -173,7 +179,8 @@ class SuiteContext implements Closeable { } Map getDestSpec() { - Map spec = getSpec() + String[] jdbc = getDownstreamJdbcNetInfo().split(":") + Map spec = getSpec(jdbc) spec.put("thrift_port", config.feTargetThriftNetworkAddress.port.toString()) spec.put("database", "TEST_" + dbName) @@ -203,7 +210,7 @@ class SuiteContext implements Closeable { Connection getTargetConnection(Suite suite) { def context = getSyncer(suite).context if (context.targetConnection == null) { - context.targetConnection = config.getConnectionByDbName("TEST_" + dbName) + context.targetConnection = config.getDownstreamConnectionByDbName("TEST_" + dbName) } return context.targetConnection } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy index 388904ec2da4d2d..92214532fdcf4e5 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy @@ -147,8 +147,8 @@ class SyncerContext { return info } - FrontendClientImpl getMasterFrontClient() { - def result = suite.sql_return_maparray "select Host, RpcPort, IsMaster from frontends();" + FrontendClientImpl getMasterFrontClient(Connection conn) { + def result = suite.sql_return_maparray("select Host, RpcPort, IsMaster from frontends();", conn) logger.info("get master fe: ${result}") def masterHost = "" @@ -179,7 +179,7 @@ class SyncerContext { FrontendClientImpl getTargetFrontClient() { if (targetFrontendClient == null) { - targetFrontendClient = getMasterFrontClient() + targetFrontendClient = getMasterFrontClient(suite.getTargetConnection()) } return targetFrontendClient } diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy index 8cfe8811993c0f5..615cb4f81c09d6a 100644 --- a/regression-test/pipeline/p0/conf/regression-conf.groovy +++ b/regression-test/pipeline/p0/conf/regression-conf.groovy @@ -25,6 +25,11 @@ targetJdbcUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLo jdbcUser = "root" jdbcPassword = "" +ccrDownstreamUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLoadLocalInfile=true" +ccrDownstreamUser = "root" +ccrDownstreamPassword = "" +ccrDownstreamFeThriftAddress = "127.0.0.1:9020" + feSourceThriftAddress = "127.0.0.1:9020" feTargetThriftAddress = "127.0.0.1:9020" feSyncerUser = "root" diff --git a/regression-test/pipeline/p1/conf/regression-conf.groovy b/regression-test/pipeline/p1/conf/regression-conf.groovy index 8669e8fb5bd3edb..98f0d9173b218fb 100644 --- a/regression-test/pipeline/p1/conf/regression-conf.groovy +++ b/regression-test/pipeline/p1/conf/regression-conf.groovy @@ -25,6 +25,11 @@ targetJdbcUrl = "jdbc:mysql://172.19.0.2:9132/?useLocalSessionState=true&allowLo jdbcUser = "root" jdbcPassword = "" +ccrDownstreamUrl = "jdbc:mysql://172.19.0.2:9132/?useLocalSessionState=true&allowLoadLocalInfile=true" +ccrDownstreamUser = "root" +ccrDownstreamPassword = "" +ccrDownstreamFeThriftAddress = "127.0.0.1:9020" + feSourceThriftAddress = "127.0.0.1:9020" feTargetThriftAddress = "127.0.0.1:9020" feSyncerUser = "root" From 59c105026ef794b8a9cc22e97287aa79fa4a0a37 Mon Sep 17 00:00:00 2001 From: Lei Zhang <27994433+SWJTU-ZhangLei@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:57:53 +0800 Subject: [PATCH 11/14] [opt](fe) Reduce jvm heap memory consumed by profiles of BrokerLoadJob (#31985) (#32592) * it may cause FE OOM when there are a lot of broker load jobs if the profile is enabled --- .../java/org/apache/doris/load/loadv2/BrokerLoadJob.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 05828d52c3bc48f..27f1291fda56759 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -194,7 +194,9 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme Lists.newArrayList(fileGroupAggInfo.getAllTableIds())); // divide job into broker loading task by table List newLoadingTasks = Lists.newArrayList(); - this.jobProfile = new Profile("BrokerLoadJob " + id + ". " + label, true); + if (enableProfile) { + this.jobProfile = new Profile("BrokerLoadJob " + id + ". " + label, true); + } ProgressManager progressManager = Env.getCurrentProgressManager(); progressManager.registerProgressSimple(String.valueOf(id)); MetaLockUtils.readLockTables(tableList); @@ -325,6 +327,8 @@ private void writeProfile() { return; } jobProfile.update(createTimestamp, getSummaryInfo(true), true); + // jobProfile has been pushed into ProfileManager, remove reference in brokerLoadJob + jobProfile = null; } private Map getSummaryInfo(boolean isFinished) { From eb2834a38c473e48719bde8b2becb68cad7110a3 Mon Sep 17 00:00:00 2001 From: zhiqiang Date: Thu, 21 Mar 2024 19:03:54 +0800 Subject: [PATCH 12/14] [fix](datetime) fix datetime rounding on BE (#32075) (#32359) --- be/src/exec/tablet_info.cpp | 6 +- be/src/vec/data_types/data_type_time_v2.cpp | 4 +- be/src/vec/data_types/data_type_time_v2.h | 5 +- be/src/vec/runtime/vdatetime_value.cpp | 84 ++-- .../vec/data_types/datetime_round_test.cpp | 421 ++++++++++++++++++ be/test/vec/data_types/from_string_test.cpp | 15 +- be/test/vec/exprs/vexpr_test.cpp | 11 +- 7 files changed, 501 insertions(+), 45 deletions(-) create mode 100644 be/test/vec/data_types/datetime_round_test.cpp diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index ac99ea63abb5c81..af91938d3542c55 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -417,9 +417,11 @@ Status VOlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr, } column->insert_data(reinterpret_cast(&dt), 0); } else if (TypeDescriptor::from_thrift(t_expr.type).is_datetime_v2_type()) { - vectorized::DateV2Value dt; + vectorized::DateV2Value dt; + const int32_t scale = + t_expr.type.types.empty() ? -1 : t_expr.type.types.front().scalar_type.scale; if (!dt.from_date_str(t_expr.date_literal.value.c_str(), - t_expr.date_literal.value.size())) { + t_expr.date_literal.value.size(), scale)) { std::stringstream ss; ss << "invalid date literal in partition column, date=" << t_expr.date_literal; return Status::InternalError(ss.str()); diff --git a/be/src/vec/data_types/data_type_time_v2.cpp b/be/src/vec/data_types/data_type_time_v2.cpp index c4b397f6e707bf4..c92201ee08db4c4 100644 --- a/be/src/vec/data_types/data_type_time_v2.cpp +++ b/be/src/vec/data_types/data_type_time_v2.cpp @@ -151,9 +151,9 @@ void DataTypeDateTimeV2::to_string(const IColumn& column, size_t row_num, } Status DataTypeDateTimeV2::from_string(ReadBuffer& rb, IColumn* column) const { - auto* column_data = static_cast(column); + auto* column_data = assert_cast(column); UInt64 val = 0; - if (!read_datetime_v2_text_impl(val, rb)) { + if (!read_datetime_v2_text_impl(val, rb, _scale)) { return Status::InvalidArgument("parse date fail, string: '{}'", std::string(rb.position(), rb.count()).c_str()); } diff --git a/be/src/vec/data_types/data_type_time_v2.h b/be/src/vec/data_types/data_type_time_v2.h index e309773183ca3bf..c141e3fb5474b6a 100644 --- a/be/src/vec/data_types/data_type_time_v2.h +++ b/be/src/vec/data_types/data_type_time_v2.h @@ -135,7 +135,10 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase { Field get_field(const TExprNode& node) const override { DateV2Value value; - if (value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size())) { + const int32_t scale = + node.type.types.empty() ? -1 : node.type.types.front().scalar_type.scale; + if (value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size(), + scale)) { return value.to_date_int_val(); } else { throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT, diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp index 04f374a5ec4a24f..5815ec852505e2a 100644 --- a/be/src/vec/runtime/vdatetime_value.cpp +++ b/be/src/vec/runtime/vdatetime_value.cpp @@ -1990,7 +1990,6 @@ bool DateV2Value::from_date_str_base(const char* date_str, int len, int scale const static int allow_space_mask = 4 | 64; uint32_t date_val[MAX_DATE_PARTS] = {0}; int32_t date_len[MAX_DATE_PARTS] = {0}; - bool carry_bits[MAX_DATE_PARTS] = {false}; // Skip space character while (ptr < end && isspace(*ptr)) { @@ -2024,46 +2023,72 @@ bool DateV2Value::from_date_str_base(const char* date_str, int len, int scale int field_idx = 0; int field_len = year_len; long sec_offset = 0; + while (ptr < end && isdigit(*ptr) && field_idx < MAX_DATE_PARTS) { const char* start = ptr; int temp_val = 0; bool scan_to_delim = (!is_interval_format) && (field_idx != 6); - while (ptr < end && isdigit(*ptr) && (scan_to_delim || field_len--)) { // field_len <= 6 - temp_val = temp_val * 10 + (*ptr++ - '0'); + while (ptr < end && isdigit(*ptr) && (scan_to_delim || field_len--)) { // field_len <= 7 + temp_val = temp_val * 10 + (*ptr - '0'); + ptr++; } + if (field_idx == 6) { - // Microsecond - const auto ms_part = ptr - start; - temp_val *= int_exp10(std::max(0L, 6 - ms_part)); if constexpr (is_datetime) { + // round of microseconds + // 1. normalize to 7 digits for rounding + // 2. rounding + // 3. nomalize to 6 digits for storage if (scale >= 0) { - if (scale == 6 && ms_part >= 6) { - if (ptr <= end && isdigit(*ptr) && *ptr >= '5') { - temp_val += 1; - } + // do normalization + const auto ms_digit_count = ptr - start; + const auto normalizer = int_exp10(std::abs(7 - ms_digit_count)); + temp_val *= normalizer; + + // check round + const auto rounder = int_exp10(std::abs(7 - scale)); + const auto reminder = temp_val % rounder; + temp_val -= reminder; + + if (reminder >= 5 * normalizer) { + temp_val += rounder; + } + + // truncate to 6 digits + if (temp_val == int_exp10(7)) { + temp_val = 0; + sec_offset += 1; } else { - const int divisor = int_exp10(6 - scale); - int remainder = temp_val % divisor; - temp_val /= divisor; - if (scale < 6 && std::abs(remainder) >= (divisor >> 1)) { - temp_val += 1; - } - temp_val *= divisor; - if (temp_val == 1000000L) { - temp_val = 0; - date_val[field_idx - 1] += 1; - carry_bits[field_idx] = true; - } + temp_val /= 10; } } + + // move ptr to start of timezone or end + while (ptr < end && isdigit(*ptr)) { + ptr++; + } + } else { + // Microsecond + const auto ms_part = ptr - start; + temp_val *= int_exp10(std::max(0L, 6 - ms_part)); } } + // Impossible if (temp_val > 999999L) { return false; } + date_val[field_idx] = temp_val; - date_len[field_idx] = ptr - start; + + if (field_idx == 6) { + // select cast("2020-01-01 12:00:00.12345" as Datetime(4)) + // ptr - start will be 5, but scale is 4 + date_len[field_idx] = std::min(static_cast(ptr - start), scale); + } else { + date_len[field_idx] = ptr - start; + } + field_len = 2; if (ptr == end) { @@ -2110,7 +2135,14 @@ bool DateV2Value::from_date_str_base(const char* date_str, int len, int scale if (field_idx == 5) { if (*ptr == '.') { ptr++; - field_len = 6; + // for datetime, we need to discard the fraction part + // that beyond the scale + 1, and scale + 1 digit will + // be used to round the fraction part + if constexpr (is_datetime) { + field_len = std::min(7, scale + 1); + } else { + field_len = 6; + } } else if (isdigit(*ptr)) { field_idx++; break; @@ -2132,6 +2164,7 @@ bool DateV2Value::from_date_str_base(const char* date_str, int len, int scale } field_idx++; } + int num_field = field_idx; if (!is_interval_format) { year_len = date_len[0]; @@ -2154,11 +2187,12 @@ bool DateV2Value::from_date_str_base(const char* date_str, int len, int scale if (is_invalid(date_val[0], date_val[1], date_val[2], 0, 0, 0, 0)) { return false; } - format_datetime(date_val, carry_bits); + if (!check_range_and_set_time(date_val[0], date_val[1], date_val[2], date_val[3], date_val[4], date_val[5], date_val[6])) { return false; } + return sec_offset ? date_add_interval( TimeInterval {TimeUnit::SECOND, sec_offset, false}) : true; diff --git a/be/test/vec/data_types/datetime_round_test.cpp b/be/test/vec/data_types/datetime_round_test.cpp new file mode 100644 index 000000000000000..48dfe3fcb701dc2 --- /dev/null +++ b/be/test/vec/data_types/datetime_round_test.cpp @@ -0,0 +1,421 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "util/mysql_row_buffer.h" +#include "vec/core/field.h" +#include "vec/core/types.h" // UInt32 +#include "vec/data_types/data_type_factory.hpp" +#include "vec/data_types/data_type_time_v2.h" +#include "vec/io/reader_buffer.h" + +using namespace doris; +using namespace doris::vectorized; + +static void from_string_checker(UInt32 scale, const std::string& rounding, + const std::string& expected) { + // DataTypeDateTimeV2 + std::shared_ptr datetime_ptr = + std::dynamic_pointer_cast( + DataTypeFactory::instance().create_data_type( + doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2, 0, scale)); + + // constructor of ReadBuffer is not const(which seems not reasonable), + // so we need to cast away const + ReadBuffer rb(const_cast(rounding.c_str()), rounding.size()); + ColumnUInt64::MutablePtr column = ColumnUInt64::create(0); + // DataTypeDateTimeV2::from_string + auto rt = datetime_ptr->from_string(rb, &(*column)); + EXPECT_TRUE(rt.ok()); + EXPECT_EQ(rt.msg(), ""); // so that we can print error msg if failed + EXPECT_EQ(datetime_ptr->to_string(*column, 0), expected); +}; + +static void from_thrift_checker(UInt32 scale, const String& input, const String& expected) { + TScalarType tscale_type; + tscale_type.type = TPrimitiveType::DATETIMEV2; + tscale_type.precision = 18; + tscale_type.scale = scale; + + TTypeNode type_node; + type_node.type = TTypeNodeType::SCALAR; + type_node.scalar_type = tscale_type; + + TTypeDesc type_desc; + type_desc.types.push_back(type_node); + + TDateLiteral date_literal; + date_literal.value = input; + + TExprNode expr_node; + expr_node.node_type = TExprNodeType::DATE_LITERAL; + expr_node.type = type_desc; + expr_node.date_literal = date_literal; + + std::shared_ptr datetime_ptr = + std::dynamic_pointer_cast( + DataTypeFactory::instance().create_data_type( + doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2, 0, scale)); + + auto field = datetime_ptr->get_field(expr_node); + uint64_t value = 0; + // = datetime_ptr->get_storage_field_type(); + EXPECT_EQ(field.try_get(value), true); + auto column = datetime_ptr->create_column_const(1, field); + EXPECT_EQ(datetime_ptr->to_string(*column, 1), expected); +} + +static void serialization_checker(UInt32 scale, const std::string& input, + const std::string& expected) { + std::shared_ptr datetime_ptr = + std::dynamic_pointer_cast( + DataTypeFactory::instance().create_data_type( + doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2, 0, scale)); + + ReadBuffer rb(const_cast(input.c_str()), input.size()); + ColumnUInt64::MutablePtr column = ColumnUInt64::create(0); + auto rt = datetime_ptr->from_string(rb, &(*column)); + EXPECT_TRUE(rt.ok()); + auto serde = std::dynamic_pointer_cast(datetime_ptr->get_serde()); + MysqlRowBuffer mysql_rb; + rt = serde->write_column_to_mysql(*column, mysql_rb, 0, false); + EXPECT_TRUE(rt.ok()); + auto elem_size = static_cast(*mysql_rb.buf()); + if (elem_size != expected.size()) { + std::cerr << "Left size " << elem_size << " right size " << expected.size() << " left str " + << std::string_view(mysql_rb.buf() + 1, elem_size) << " right str " << expected + << std::endl; + ASSERT_TRUE(false); // terminate ut + } + EXPECT_EQ(std::string_view(mysql_rb.buf() + 1, expected.size()), expected); +} + +static std::multimap> + scale_with_input_and_expected = { + {0, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00"}}}, + {0, {{"2020-01-01 12:00:00.0", "2020-01-01 12:00:00"}}}, + {0, {{"2020-01-01 12:00:00.5", "2020-01-01 12:00:01"}}}, + {0, {{"2020-01-01 12:00:00.49", "2020-01-01 12:00:00"}}}, + {0, {{"2020-01-01 12:00:00.00", "2020-01-01 12:00:00"}}}, + {0, {{"2020-01-01 12:00:00.999999999999", "2020-01-01 12:00:01"}}}, + {0, {{"9999-12-31 23:59:59", "9999-12-31 23:59:59"}}}, + // normal cast, no rounding + {1, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.0"}}}, + {1, {{"2020-01-01 12:00:00.0", "2020-01-01 12:00:00.0"}}}, + {1, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.2", "2020-01-01 12:00:00.2"}}}, + {1, {{"2020-01-01 12:00:00.3", "2020-01-01 12:00:00.3"}}}, + {1, {{"2020-01-01 12:00:00.4", "2020-01-01 12:00:00.4"}}}, + {1, {{"2020-01-01 12:00:00.5", "2020-01-01 12:00:00.5"}}}, + {1, {{"2020-01-01 12:00:00.6", "2020-01-01 12:00:00.6"}}}, + {1, {{"2020-01-01 12:00:00.7", "2020-01-01 12:00:00.7"}}}, + {1, {{"2020-01-01 12:00:00.8", "2020-01-01 12:00:00.8"}}}, + {1, {{"2020-01-01 12:00:00.9", "2020-01-01 12:00:00.9"}}}, + // round test + {1, {{"2020-01-01 12:00:00.10", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.11", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.12", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.13", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.14", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.15", "2020-01-01 12:00:00.2"}}}, + {1, {{"2020-01-01 12:00:00.16", "2020-01-01 12:00:00.2"}}}, + {1, {{"2020-01-01 12:00:00.17", "2020-01-01 12:00:00.2"}}}, + {1, {{"2020-01-01 12:00:00.18", "2020-01-01 12:00:00.2"}}}, + {1, {{"2020-01-01 12:00:00.19", "2020-01-01 12:00:00.2"}}}, + {1, {{"2020-01-01 12:00:00.90", "2020-01-01 12:00:00.9"}}}, + {1, {{"2020-01-01 12:00:00.91", "2020-01-01 12:00:00.9"}}}, + {1, {{"2020-01-01 12:00:00.92", "2020-01-01 12:00:00.9"}}}, + {1, {{"2020-01-01 12:00:00.93", "2020-01-01 12:00:00.9"}}}, + {1, {{"2020-01-01 12:00:00.94", "2020-01-01 12:00:00.9"}}}, + {1, {{"2020-01-01 12:00:00.95", "2020-01-01 12:00:01.0"}}}, + {1, {{"2020-01-01 12:00:00.96", "2020-01-01 12:00:01.0"}}}, + {1, {{"2020-01-01 12:00:00.97", "2020-01-01 12:00:01.0"}}}, + {1, {{"2020-01-01 12:00:00.98", "2020-01-01 12:00:01.0"}}}, + {1, {{"2020-01-01 12:00:00.99", "2020-01-01 12:00:01.0"}}}, + // make sure we are doing truncate instead of round up from last digit + {1, {{"2020-01-01 12:00:00.140", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.141", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.142", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.143", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.144", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.145", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.146", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.147", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.148", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.149", "2020-01-01 12:00:00.1"}}}, + {1, {{"2020-01-01 12:00:00.150", "2020-01-01 12:00:00.2"}}}, + {1, {{"9999-12-31 23:59:59.9", "9999-12-31 23:59:59.9"}}}, + // normal cast, no rounding + {2, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.00"}}}, + {2, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.10"}}}, + {2, {{"2020-01-01 12:00:00.00", "2020-01-01 12:00:00.00"}}}, + {2, {{"2020-01-01 12:00:00.01", "2020-01-01 12:00:00.01"}}}, + {2, {{"2020-01-01 12:00:00.02", "2020-01-01 12:00:00.02"}}}, + {2, {{"2020-01-01 12:00:00.03", "2020-01-01 12:00:00.03"}}}, + {2, {{"2020-01-01 12:00:00.04", "2020-01-01 12:00:00.04"}}}, + {2, {{"2020-01-01 12:00:00.05", "2020-01-01 12:00:00.05"}}}, + {2, {{"2020-01-01 12:00:00.06", "2020-01-01 12:00:00.06"}}}, + {2, {{"2020-01-01 12:00:00.07", "2020-01-01 12:00:00.07"}}}, + {2, {{"2020-01-01 12:00:00.08", "2020-01-01 12:00:00.08"}}}, + {2, {{"2020-01-01 12:00:00.09", "2020-01-01 12:00:00.09"}}}, + {2, {{"2020-01-01 12:00:00.12", "2020-01-01 12:00:00.12"}}}, + // rounding + {2, {{"2020-01-01 12:00:00.990", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.991", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.992", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.993", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.994", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.995", "2020-01-01 12:00:01.00"}}}, + {2, {{"2020-01-01 12:00:00.996", "2020-01-01 12:00:01.00"}}}, + {2, {{"2020-01-01 12:00:00.997", "2020-01-01 12:00:01.00"}}}, + {2, {{"2020-01-01 12:00:00.998", "2020-01-01 12:00:01.00"}}}, + {2, {{"2020-01-01 12:00:00.999", "2020-01-01 12:00:01.00"}}}, + // make sure we are doing truncate instead of round from last digit + {2, {{"2020-01-01 12:00:00.9940", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.9941", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.9942", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.9943", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.9944", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.9945", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.9946", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.9947", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.9948", "2020-01-01 12:00:00.99"}}}, + {2, {{"2020-01-01 12:00:00.9949", "2020-01-01 12:00:00.99"}}}, + {2, {{"9999-12-31 23:59:59.99", "9999-12-31 23:59:59.99"}}}, + {3, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.000"}}}, + {3, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.100"}}}, + {3, {{"2020-01-01 12:00:00.00", "2020-01-01 12:00:00.000"}}}, + {3, {{"2020-01-01 12:00:00.01", "2020-01-01 12:00:00.010"}}}, + {3, {{"2020-01-01 12:00:00.001", "2020-01-01 12:00:00.001"}}}, + {3, {{"2020-01-01 12:00:00.002", "2020-01-01 12:00:00.002"}}}, + {3, {{"2020-01-01 12:00:00.003", "2020-01-01 12:00:00.003"}}}, + {3, {{"2020-01-01 12:00:00.004", "2020-01-01 12:00:00.004"}}}, + {3, {{"2020-01-01 12:00:00.005", "2020-01-01 12:00:00.005"}}}, + {3, {{"2020-01-01 12:00:00.006", "2020-01-01 12:00:00.006"}}}, + {3, {{"2020-01-01 12:00:00.007", "2020-01-01 12:00:00.007"}}}, + {3, {{"2020-01-01 12:00:00.008", "2020-01-01 12:00:00.008"}}}, + {3, {{"2020-01-01 12:00:00.009", "2020-01-01 12:00:00.009"}}}, + {3, {{"2020-01-01 12:00:00.123", "2020-01-01 12:00:00.123"}}}, + {3, {{"2020-01-01 12:00:00.999", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.123", "2020-01-01 12:00:00.123"}}}, + // rounding + {3, {{"2020-01-01 12:00:00.9990", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.9991", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.9992", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.9993", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.9994", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.9995", "2020-01-01 12:00:01.000"}}}, + {3, {{"2020-01-01 12:00:00.9996", "2020-01-01 12:00:01.000"}}}, + {3, {{"2020-01-01 12:00:00.9997", "2020-01-01 12:00:01.000"}}}, + {3, {{"2020-01-01 12:00:00.9998", "2020-01-01 12:00:01.000"}}}, + {3, {{"2020-01-01 12:00:00.9999", "2020-01-01 12:00:01.000"}}}, + // make sure we are doing truncate instead of round from last digit + {3, {{"2020-01-01 12:00:00.99940", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.99941", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.99942", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.99943", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.99944", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.99945", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.99946", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.99947", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.99948", "2020-01-01 12:00:00.999"}}}, + {3, {{"2020-01-01 12:00:00.99949", "2020-01-01 12:00:00.999"}}}, + {3, {{"9999-12-31 23:59:59.999", "9999-12-31 23:59:59.999"}}}, + // normal cast, no rounding + {4, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.0000"}}}, + {4, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.1000"}}}, + {4, {{"2020-01-01 12:00:00.01", "2020-01-01 12:00:00.0100"}}}, + {4, {{"2020-01-01 12:00:00.001", "2020-01-01 12:00:00.0010"}}}, + {4, {{"2020-01-01 12:00:00.0001", "2020-01-01 12:00:00.0001"}}}, + {4, {{"2020-01-01 12:00:00.0002", "2020-01-01 12:00:00.0002"}}}, + {4, {{"2020-01-01 12:00:00.0003", "2020-01-01 12:00:00.0003"}}}, + {4, {{"2020-01-01 12:00:00.0004", "2020-01-01 12:00:00.0004"}}}, + {4, {{"2020-01-01 12:00:00.0005", "2020-01-01 12:00:00.0005"}}}, + {4, {{"2020-01-01 12:00:00.0006", "2020-01-01 12:00:00.0006"}}}, + {4, {{"2020-01-01 12:00:00.0007", "2020-01-01 12:00:00.0007"}}}, + {4, {{"2020-01-01 12:00:00.0008", "2020-01-01 12:00:00.0008"}}}, + {4, {{"2020-01-01 12:00:00.0009", "2020-01-01 12:00:00.0009"}}}, + {4, {{"2020-01-01 12:00:00.1234", "2020-01-01 12:00:00.1234"}}}, + {4, {{"2020-01-01 12:00:00.9999", "2020-01-01 12:00:00.9999"}}}, + // rounding + {4, {{"2020-01-01 12:00:00.99990", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.99991", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.99992", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.99993", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.99994", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.99995", "2020-01-01 12:00:01.0000"}}}, + {4, {{"2020-01-01 12:00:00.99996", "2020-01-01 12:00:01.0000"}}}, + {4, {{"2020-01-01 12:00:00.99997", "2020-01-01 12:00:01.0000"}}}, + {4, {{"2020-01-01 12:00:00.99998", "2020-01-01 12:00:01.0000"}}}, + {4, {{"2020-01-01 12:00:00.99999", "2020-01-01 12:00:01.0000"}}}, + // make sure we are doing truncate instead of round from last digit + {4, {{"2020-01-01 12:00:00.999940", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.999941", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.999942", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.999943", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.999944", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.999945", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.999946", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.999947", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.999948", "2020-01-01 12:00:00.9999"}}}, + {4, {{"2020-01-01 12:00:00.999949", "2020-01-01 12:00:00.9999"}}}, + {4, {{"9999-12-31 23:59:59.9999", "9999-12-31 23:59:59.9999"}}}, + {5, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.00000"}}}, + {5, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.10000"}}}, + {5, {{"2020-01-01 12:00:00.01", "2020-01-01 12:00:00.01000"}}}, + {5, {{"2020-01-01 12:00:00.001", "2020-01-01 12:00:00.00100"}}}, + {5, {{"2020-01-01 12:00:00.0001", "2020-01-01 12:00:00.00010"}}}, + {5, {{"2020-01-01 12:00:00.00001", "2020-01-01 12:00:00.00001"}}}, + {5, {{"2020-01-01 12:00:00.00002", "2020-01-01 12:00:00.00002"}}}, + {5, {{"2020-01-01 12:00:00.00003", "2020-01-01 12:00:00.00003"}}}, + {5, {{"2020-01-01 12:00:00.00004", "2020-01-01 12:00:00.00004"}}}, + {5, {{"2020-01-01 12:00:00.00005", "2020-01-01 12:00:00.00005"}}}, + {5, {{"2020-01-01 12:00:00.00006", "2020-01-01 12:00:00.00006"}}}, + {5, {{"2020-01-01 12:00:00.00007", "2020-01-01 12:00:00.00007"}}}, + {5, {{"2020-01-01 12:00:00.00008", "2020-01-01 12:00:00.00008"}}}, + {5, {{"2020-01-01 12:00:00.00009", "2020-01-01 12:00:00.00009"}}}, + {5, {{"2020-01-01 12:00:00.12345", "2020-01-01 12:00:00.12345"}}}, + {5, {{"2020-01-01 12:00:00.99999", "2020-01-01 12:00:00.99999"}}}, + // rounding + {5, {{"2020-01-01 12:00:00.999990", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.999991", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.999992", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.999993", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.999994", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.999995", "2020-01-01 12:00:01.00000"}}}, + {5, {{"2020-01-01 12:00:00.999996", "2020-01-01 12:00:01.00000"}}}, + {5, {{"2020-01-01 12:00:00.999997", "2020-01-01 12:00:01.00000"}}}, + {5, {{"2020-01-01 12:00:00.999998", "2020-01-01 12:00:01.00000"}}}, + {5, {{"2020-01-01 12:00:00.999999", "2020-01-01 12:00:01.00000"}}}, + // make sure we are doing truncate instead of round from last digit + {5, {{"2020-01-01 12:00:00.9999940", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.9999941", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.9999942", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.9999943", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.9999944", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.9999945", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.9999946", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.9999947", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.9999948", "2020-01-01 12:00:00.99999"}}}, + {5, {{"2020-01-01 12:00:00.9999949", "2020-01-01 12:00:00.99999"}}}, + {5, {{"9999-12-31 23:59:59.99999", "9999-12-31 23:59:59.99999"}}}, + // normal cast, no rounding + {6, {{"2020-01-01 12:00:00", "2020-01-01 12:00:00.000000"}}}, + {6, {{"2020-01-01 12:00:00.1", "2020-01-01 12:00:00.100000"}}}, + {6, {{"2020-01-01 12:00:00.01", "2020-01-01 12:00:00.010000"}}}, + {6, {{"2020-01-01 12:00:00.001", "2020-01-01 12:00:00.001000"}}}, + {6, {{"2020-01-01 12:00:00.0001", "2020-01-01 12:00:00.000100"}}}, + {6, {{"2020-01-01 12:00:00.00001", "2020-01-01 12:00:00.000010"}}}, + {6, {{"2020-01-01 12:00:00.000001", "2020-01-01 12:00:00.000001"}}}, + {6, {{"2020-01-01 12:00:00.000002", "2020-01-01 12:00:00.000002"}}}, + {6, {{"2020-01-01 12:00:00.000003", "2020-01-01 12:00:00.000003"}}}, + {6, {{"2020-01-01 12:00:00.000004", "2020-01-01 12:00:00.000004"}}}, + {6, {{"2020-01-01 12:00:00.000005", "2020-01-01 12:00:00.000005"}}}, + {6, {{"2020-01-01 12:00:00.000006", "2020-01-01 12:00:00.000006"}}}, + {6, {{"2020-01-01 12:00:00.000007", "2020-01-01 12:00:00.000007"}}}, + {6, {{"2020-01-01 12:00:00.000008", "2020-01-01 12:00:00.000008"}}}, + {6, {{"2020-01-01 12:00:00.000009", "2020-01-01 12:00:00.000009"}}}, + {6, {{"2020-01-01 12:00:00.123456", "2020-01-01 12:00:00.123456"}}}, + {6, {{"2020-01-01 12:00:00.999999", "2020-01-01 12:00:00.999999"}}}, + // rounding + {6, {{"2020-01-01 12:00:00.9999990", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.9999991", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.9999992", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.9999993", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.9999994", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.9999995", "2020-01-01 12:00:01.000000"}}}, + {6, {{"2020-01-01 12:00:00.9999996", "2020-01-01 12:00:01.000000"}}}, + {6, {{"2020-01-01 12:00:00.9999997", "2020-01-01 12:00:01.000000"}}}, + {6, {{"2020-01-01 12:00:00.9999998", "2020-01-01 12:00:01.000000"}}}, + {6, {{"2020-01-01 12:00:00.9999999", "2020-01-01 12:00:01.000000"}}}, + // make sure we are doing truncate instead of round from last digit + {6, {{"2020-01-01 12:00:00.99999940", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.99999941", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.99999942", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.99999943", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.99999944", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.99999945", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.99999946", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.99999947", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.99999948", "2020-01-01 12:00:00.999999"}}}, + {6, {{"2020-01-01 12:00:00.99999949", "2020-01-01 12:00:00.999999"}}}, + {6, {{"9999-12-31 23:59:59.999999", "9999-12-31 23:59:59.999999"}}}, + // + {0, {{"2024-02-29 23:59:59.9", "2024-03-01 00:00:00"}}}, + {1, {{"2024-02-29 23:59:59.99", "2024-03-01 00:00:00.0"}}}, + {2, {{"2024-02-29 23:59:59.999", "2024-03-01 00:00:00.00"}}}, + {3, {{"2024-02-29 23:59:59.9999", "2024-03-01 00:00:00.000"}}}, + {4, {{"2024-02-29 23:59:59.99999", "2024-03-01 00:00:00.0000"}}}, + {5, {{"2024-02-29 23:59:59.999999", "2024-03-01 00:00:00.00000"}}}, + {6, {{"2024-02-29 23:59:59.9999999", "2024-03-01 00:00:00.000000"}}}, + // + {0, {{"2025-02-28 23:59:59.9999999", "2025-03-01 00:00:00"}}}, + {1, {{"2025-02-28 23:59:59.9999999", "2025-03-01 00:00:00.0"}}}, + {2, {{"2025-02-28 23:59:59.9999999", "2025-03-01 00:00:00.00"}}}, + {3, {{"2025-02-28 23:59:59.9999999", "2025-03-01 00:00:00.000"}}}, + {4, {{"2025-02-28 23:59:59.9999999", "2025-03-01 00:00:00.0000"}}}, + {5, {{"2025-02-28 23:59:59.9999999", "2025-03-01 00:00:00.00000"}}}, + {6, {{"2025-02-28 23:59:59.9999999", "2025-03-01 00:00:00.000000"}}}, + +}; + +namespace doris::vectorized { +// // make sure micro-seconds part of datetime has correct round behaviour +TEST(DatetimeRountTest, test_datetime_round_behaviour_basic) { + { + for (auto itr : scale_with_input_and_expected) { + for (const auto& [input, expected] : itr.second) { + from_string_checker(itr.first, input, expected); + } + } + } +} + +TEST(DatetimeRountTest, test_datetime_round_behaviour_get_field) { + { + for (auto itr : scale_with_input_and_expected) { + for (const auto& [input, expected] : itr.second) { + from_thrift_checker(itr.first, input, expected); + } + } + } +} + +TEST(DatetimeRountTest, test_datetime_round_behaviour_serialize) { + { + for (auto itr : scale_with_input_and_expected) { + for (const auto& [input, expected] : itr.second) { + serialization_checker(itr.first, input, expected); + } + } + } +} +} // namespace doris::vectorized diff --git a/be/test/vec/data_types/from_string_test.cpp b/be/test/vec/data_types/from_string_test.cpp index e3d3b5bd5d61efd..beeac5d96d8e689 100644 --- a/be/test/vec/data_types/from_string_test.cpp +++ b/be/test/vec/data_types/from_string_test.cpp @@ -16,6 +16,7 @@ // under the License. #include "gtest/gtest_pred_impl.h" +#include "olap/olap_common.h" #include "olap/types.h" // for TypeInfo #include "olap/wrapper_field.h" #include "vec/columns/column.h" @@ -225,7 +226,13 @@ TEST(FromStringTest, ScalaWrapperFieldVsDataType) { }; for (auto pair : date_scala_field_types) { auto type = pair.first; - DataTypePtr data_type_ptr = DataTypeFactory::instance().create_data_type(type, 0, 0); + DataTypePtr data_type_ptr = nullptr; + if (type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2) { + data_type_ptr = DataTypeFactory::instance().create_data_type(type, 0, 6); + } else { + data_type_ptr = DataTypeFactory::instance().create_data_type(type, 0, 0); + } + std::cout << "this type is " << data_type_ptr->get_name() << ": " << fmt::format("{}", type) << std::endl; @@ -262,12 +269,6 @@ TEST(FromStringTest, ScalaWrapperFieldVsDataType) { std::cout << "min(" << min_s << ") with datat_ype_str:" << min_s_d << std::endl; std::cout << "max(" << max_s << ") with datat_ype_str:" << max_s_d << std::endl; std::cout << "rand(" << rand_date << ") with datat_type_str:" << rand_s_d << std::endl; - if (FieldType::OLAP_FIELD_TYPE_DATETIMEV2 == type) { - // field to_string : %Y-%m-%d %H:%i:%s.%f vs data type to_string %Y-%m-%d %H:%i:%s - min_s = min_s.substr(0, min_s.find_last_of('.')); - max_s = max_s.substr(0, max_s.find_last_of('.')); - rand_date = rand_date.substr(0, rand_date.find_last_of('.')); - } // min wrapper field date to_string in macOS and linux system has different result // macOs equals with data type to_string(0000-01-01), but in linux is (0-01-01) if (FieldType::OLAP_FIELD_TYPE_DATE == type || diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp index 8c90debda6b6648..3fe5c60f6b15cd6 100644 --- a/be/test/vec/exprs/vexpr_test.cpp +++ b/be/test/vec/exprs/vexpr_test.cpp @@ -480,21 +480,16 @@ TEST(TEST_VEXPR, LITERALTEST) { uint8_t hour = 9; uint8_t minute = 12; uint8_t second = 46; - uint32_t microsecond = 999999; + uint32_t microsecond = 999999; // target scale is 4, so the microsecond will be rounded up DateV2Value datetime_v2; datetime_v2.set_time(year, month, day, hour, minute, second, microsecond); std::string date = datetime_v2.debug_string(); - __uint64_t dt; - memcpy(&dt, &datetime_v2, sizeof(__uint64_t)); VLiteral literal(create_literal(date, 4)); Block block; int ret = -1; - literal.execute(nullptr, &block, &ret); - auto ctn = block.safe_get_by_position(ret); - auto v = (*ctn.column)[0].get<__uint64_t>(); - EXPECT_EQ(v, dt); - EXPECT_EQ("1997-11-18 09:12:46.9999", literal.value()); + EXPECT_TRUE(literal.execute(nullptr, &block, &ret).ok()); + EXPECT_EQ("1997-11-18 09:12:47.0000", literal.value()); } // date { From a6d52f523176389c800b5a1978abb407bf8531b3 Mon Sep 17 00:00:00 2001 From: zhangstar333 <87313068+zhangstar333@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:05:37 +0800 Subject: [PATCH 13/14] [improve](function) add error msg if exceeded maximum default value in repeat function (#32219) (#32588) --- be/src/vec/functions/function_string.h | 39 ++++++++++++++----- be/test/vec/function/function_string_test.cpp | 21 ++++++---- .../string/test_string_basic.groovy | 5 ++- .../max_msg_size_of_result_receiver.groovy | 14 +++---- 4 files changed, 53 insertions(+), 26 deletions(-) diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h index c2dbed38d39020f..6f3eca6cc824ec1 100644 --- a/be/src/vec/functions/function_string.h +++ b/be/src/vec/functions/function_string.h @@ -29,6 +29,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -1174,6 +1177,14 @@ class FunctionStringRepeat : public IFunction { static FunctionPtr create() { return std::make_shared(); } String get_name() const override { return name; } size_t get_number_of_arguments() const override { return 2; } + std::string error_msg(int default_value, int repeat_value) const { + auto error_msg = fmt::format( + "The second parameter of repeat function exceeded maximum default value, " + "default_value is {}, and now input is {} . you could try change default value " + "greater than value eg: set repeat_max_num = {}.", + default_value, repeat_value, repeat_value + 10); + return error_msg; + } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { return make_nullable(std::make_shared()); @@ -1191,9 +1202,10 @@ class FunctionStringRepeat : public IFunction { if (auto* col1 = check_and_get_column(*argument_ptr[0])) { if (auto* col2 = check_and_get_column(*argument_ptr[1])) { - vector_vector(col1->get_chars(), col1->get_offsets(), col2->get_data(), - res->get_chars(), res->get_offsets(), null_map->get_data(), - context->state()->repeat_max_num()); + RETURN_IF_ERROR(vector_vector(col1->get_chars(), col1->get_offsets(), + col2->get_data(), res->get_chars(), + res->get_offsets(), null_map->get_data(), + context->state()->repeat_max_num())); block.replace_by_position( result, ColumnNullable::create(std::move(res), std::move(null_map))); return Status::OK(); @@ -1203,8 +1215,11 @@ class FunctionStringRepeat : public IFunction { if constexpr (use_old_function) { repeat = col2_const->get_int(0); } else { - repeat = std::min(col2_const->get_int(0), - context->state()->repeat_max_num()); + repeat = col2_const->get_int(0); + if (repeat > context->state()->repeat_max_num()) { + return Status::InvalidArgument( + error_msg(context->state()->repeat_max_num(), repeat)); + } } if (repeat <= 0) { null_map->get_data().resize_fill(input_rows_count, 0); @@ -1223,10 +1238,10 @@ class FunctionStringRepeat : public IFunction { argument_ptr[0]->get_name(), argument_ptr[1]->get_name()); } - void vector_vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets, - const ColumnInt32::Container& repeats, ColumnString::Chars& res_data, - ColumnString::Offsets& res_offsets, ColumnUInt8::Container& null_map, - const int repeat_max_num) { + Status vector_vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets, + const ColumnInt32::Container& repeats, ColumnString::Chars& res_data, + ColumnString::Offsets& res_offsets, ColumnUInt8::Container& null_map, + const int repeat_max_num) const { size_t input_row_size = offsets.size(); fmt::memory_buffer buffer; @@ -1240,7 +1255,10 @@ class FunctionStringRepeat : public IFunction { if constexpr (use_old_function) { repeat = repeats[i]; } else { - repeat = std::min(repeats[i], repeat_max_num); + repeat = repeats[i]; + if (repeat > repeat_max_num) { + return Status::InvalidArgument(error_msg(repeat_max_num, repeat)); + } } if (repeat <= 0) { StringOP::push_empty_string(i, res_data, res_offsets); @@ -1254,6 +1272,7 @@ class FunctionStringRepeat : public IFunction { res_data, res_offsets); } } + return Status::OK(); } // TODO: 1. use pmr::vector replace fmt_buffer may speed up the code diff --git a/be/test/vec/function/function_string_test.cpp b/be/test/vec/function/function_string_test.cpp index 03a580e5192bc45..73e35d9a09091cf 100644 --- a/be/test/vec/function/function_string_test.cpp +++ b/be/test/vec/function/function_string_test.cpp @@ -178,15 +178,20 @@ TEST(function_string_test, function_string_repeat_test) { std::string func_name = "repeat"; InputTypeSet input_types = {TypeIndex::String, TypeIndex::Int32}; - DataSet data_set = { - {{std::string("a"), 3}, std::string("aaa")}, - {{std::string("hel lo"), 2}, std::string("hel lohel lo")}, - {{std::string("hello word"), -1}, std::string("")}, - {{std::string(""), 1}, std::string("")}, - {{std::string("a"), 1073741825}, std::string("aaaaaaaaaa")}, // ut repeat max num 10 - {{std::string("HELLO,!^%"), 2}, std::string("HELLO,!^%HELLO,!^%")}, - {{std::string("你"), 2}, std::string("你你")}}; + DataSet data_set = {{{std::string("a"), 3}, std::string("aaa")}, + {{std::string("hel lo"), 2}, std::string("hel lohel lo")}, + {{std::string("hello word"), -1}, std::string("")}, + {{std::string(""), 1}, std::string("")}, + {{std::string("HELLO,!^%"), 2}, std::string("HELLO,!^%HELLO,!^%")}, + {{std::string("你"), 2}, std::string("你你")}}; check_function(func_name, input_types, data_set); + + { + DataSet data_set = {{{std::string("a"), 1073741825}, + std::string("aaaaaaaaaa")}}; // ut repeat max num 10 + Status st = check_function(func_name, input_types, data_set, true); + EXPECT_NE(Status::OK(), st); + } } TEST(function_string_test, function_string_reverse_test) { diff --git a/regression-test/suites/datatype_p0/string/test_string_basic.groovy b/regression-test/suites/datatype_p0/string/test_string_basic.groovy index 2aa9f9e86e45221..36fbddede2dedee 100644 --- a/regression-test/suites/datatype_p0/string/test_string_basic.groovy +++ b/regression-test/suites/datatype_p0/string/test_string_basic.groovy @@ -129,7 +129,10 @@ suite("test_string_basic") { (2, repeat("test1111", 131072)) """ order_qt_select_str_tb "select k1, md5(v1), length(v1) from ${tbName}" - + test { + sql """SELECT repeat("test1111", 131073 + 100);""" + exception "repeat function exceeded maximum default value" + } sql """drop table if exists test_string_cmp;""" sql """ diff --git a/regression-test/suites/variable_p0/max_msg_size_of_result_receiver.groovy b/regression-test/suites/variable_p0/max_msg_size_of_result_receiver.groovy index e7fead33d90ec0c..f9afdd8eadb3587 100644 --- a/regression-test/suites/variable_p0/max_msg_size_of_result_receiver.groovy +++ b/regression-test/suites/variable_p0/max_msg_size_of_result_receiver.groovy @@ -27,13 +27,14 @@ suite("max_msg_size_of_result_receiver") { ENGINE=OLAP DISTRIBUTED BY HASH(id) PROPERTIES("replication_num"="1") """ - + sql """set repeat_max_num=100000;""" + sql """set max_msg_size_of_result_receiver=90000;""" // so the test of repeat("a", 80000) could pass, and repeat("a", 100000) will be failed sql """ - INSERT INTO ${table_name} VALUES (104, repeat("a", ${MESSAGE_SIZE_BASE * 104})) + INSERT INTO ${table_name} VALUES (104, repeat("a", 80000)) """ sql """ - INSERT INTO ${table_name} VALUES (105, repeat("a", ${MESSAGE_SIZE_BASE * 105})) + INSERT INTO ${table_name} VALUES (105, repeat("a", 100000)) """ def with_exception = false @@ -44,10 +45,9 @@ suite("max_msg_size_of_result_receiver") { } assertEquals(with_exception, false) - try { - sql "SELECT * FROM ${table_name} WHERE id = 105" - } catch (Exception e) { - assertTrue(e.getMessage().contains('MaxMessageSize reached, try increase max_msg_size_of_result_receiver')) + test { + sql """SELECT * FROM ${table_name} WHERE id = 105;""" + exception "MaxMessageSize reached, try increase max_msg_size_of_result_receiver" } try { From 11f616ff34268b47d29d28846461a220857d445b Mon Sep 17 00:00:00 2001 From: deardeng <565620795@qq.com> Date: Thu, 21 Mar 2024 19:06:40 +0800 Subject: [PATCH 14/14] [fix](retry) Set query encounter rpc exception default retry times to 3 (#28555) (#32593) --- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index d39b0576a150ef4..e3aa4e368c3837f 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -890,7 +890,7 @@ public class Config extends ConfigBase { * You may reduce this number to avoid Avalanche disaster. */ @ConfField(mutable = true) - public static int max_query_retry_time = 1; + public static int max_query_retry_time = 3; /** * The number of point query retries in executor.