diff --git a/be/src/pipeline/exec/es_scan_operator.cpp b/be/src/pipeline/exec/es_scan_operator.cpp index 0e5018b85d6a7e..024ed6c53dcac8 100644 --- a/be/src/pipeline/exec/es_scan_operator.cpp +++ b/be/src/pipeline/exec/es_scan_operator.cpp @@ -84,9 +84,10 @@ Status EsScanLocalState::_init_scanners(std::list* sca std::to_string(vectorized::RuntimeFilterConsumer::_state->batch_size()); properties[ESScanReader::KEY_HOST_PORT] = get_host_and_port(es_scan_range->es_hosts); // push down limit to Elasticsearch - // if predicate in _conjunct_ctxs can not be processed by Elasticsearch, we can not push down limit operator to Elasticsearch + // if predicate in _conjuncts can not be processed by Elasticsearch, we can not push down limit operator to Elasticsearch if (p.limit() != -1 && - p.limit() <= vectorized::RuntimeFilterConsumer::_state->batch_size()) { + p.limit() <= vectorized::RuntimeFilterConsumer::_state->batch_size() && + p.conjuncts().empty()) { properties[ESScanReader::KEY_TERMINATE_AFTER] = std::to_string(p.limit()); } diff --git a/regression-test/data/external_table_p0/es/test_es_query.out b/regression-test/data/external_table_p0/es/test_es_query.out index a2289bec1b4927..1e103748dc44c2 100644 --- a/regression-test/data/external_table_p0/es/test_es_query.out +++ b/regression-test/data/external_table_p0/es/test_es_query.out @@ -193,6 +193,16 @@ I'm not null or empty [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +-- !sql_5_21 -- +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 + +-- !sql_5_22 -- +2022-08-08T12:10:10 2022-08-08 12 +2022-08-08T12:10:10 2022-08-08 12 + -- !sql_6_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 @@ -265,6 +275,16 @@ I'm not null or empty [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +-- !sql_6_21 -- +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 + +-- !sql_6_22 -- +2022-08-08T12:10:10 2022-08-08 12 +2022-08-08T12:10:10 2022-08-08 12 + -- !sql_7_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 @@ -370,6 +390,22 @@ I'm not null or empty [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +-- !sql_7_26 -- +2022-08-08T04:10:10 2022-08-08 +2022-08-08T04:10:10 2022-08-08 + +-- !sql_7_27 -- +2022-08-09T12:10:10 2022-08-09 +2022-08-09T12:10:10 2022-08-09 + +-- !sql_7_28 -- +2022-08-10T04:10:10 2022-08-10 +2022-08-10T04:10:10 2022-08-10 + +-- !sql_7_29 -- +2022-08-11T12:10:10 2022-08-11 +2022-08-11T12:10:10 2022-08-11 + -- !sql_7_26 -- value1 value2 @@ -475,6 +511,22 @@ I'm not null or empty [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +-- !sql_8_24 -- +2022-08-08T04:10:10 2022-08-08 +2022-08-08T04:10:10 2022-08-08 + +-- !sql_8_25 -- +2022-08-09T12:10:10 2022-08-09 +2022-08-09T12:10:10 2022-08-09 + +-- !sql_8_26 -- +2022-08-10T04:10:10 2022-08-10 +2022-08-10T04:10:10 2022-08-10 + +-- !sql_8_27 -- +2022-08-11T12:10:10 2022-08-11 +2022-08-11T12:10:10 2022-08-11 + -- !sql01 -- ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] @@ -608,6 +660,16 @@ I'm not null or empty [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +-- !sql_5_21 -- +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 + +-- !sql_5_22 -- +2022-08-08T12:10:10 2022-08-08 12 +2022-08-08T12:10:10 2022-08-08 12 + -- !sql_6_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10 @@ -680,6 +742,16 @@ I'm not null or empty [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +-- !sql_6_21 -- +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 +2022-08-08T20:10:10 2022-08-08 20 + +-- !sql_6_22 -- +2022-08-08T12:10:10 2022-08-08 12 +2022-08-08T12:10:10 2022-08-08 12 + -- !sql_7_02 -- [1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 @@ -785,6 +857,22 @@ I'm not null or empty [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +-- !sql_7_26 -- +2022-08-08T04:10:10 2022-08-08 +2022-08-08T04:10:10 2022-08-08 + +-- !sql_7_27 -- +2022-08-09T12:10:10 2022-08-09 +2022-08-09T12:10:10 2022-08-09 + +-- !sql_7_28 -- +2022-08-10T04:10:10 2022-08-10 +2022-08-10T04:10:10 2022-08-10 + +-- !sql_7_29 -- +2022-08-11T12:10:10 2022-08-11 +2022-08-11T12:10:10 2022-08-11 + -- !sql_7_26 -- value1 value2 @@ -890,3 +978,19 @@ I'm not null or empty [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" [{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White" +-- !sql_8_24 -- +2022-08-08T04:10:10 2022-08-08 +2022-08-08T04:10:10 2022-08-08 + +-- !sql_8_25 -- +2022-08-09T12:10:10 2022-08-09 +2022-08-09T12:10:10 2022-08-09 + +-- !sql_8_26 -- +2022-08-10T04:10:10 2022-08-10 +2022-08-10T04:10:10 2022-08-10 + +-- !sql_8_27 -- +2022-08-11T12:10:10 2022-08-11 +2022-08-11T12:10:10 2022-08-11 + diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy b/regression-test/suites/external_table_p0/es/test_es_query.groovy index 54b1c331de58d9..9d3d2cb80adaa2 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query.groovy +++ b/regression-test/suites/external_table_p0/es/test_es_query.groovy @@ -214,6 +214,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_5_18 """select message from test1 where not_null_or_empty(message)""" order_qt_sql_5_19 """select * from test1 where esquery(c_unsigned_long, '{"match":{"c_unsigned_long":0}}')""" order_qt_sql_5_20 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" + order_qt_sql_5_21 """select test6, substring(test6, 1, 13) from test2 where substring(test6, 1, 13) = '2022-08-08 20' limit 4;""" + order_qt_sql_5_22 """select test6, substring(test6, 1, 13) from test2 where substring(test6, 1, 13) = '2022-08-08 12' limit 4;""" try { sql """select * from composite_type_array;""" fail("Should not reach here") @@ -243,6 +245,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_6_18 """select message from test1 where not_null_or_empty(message)""" order_qt_sql_6_19 """select * from test1 where esquery(c_person, '{"match":{"c_person.name":"Andy"}}')""" order_qt_sql_6_20 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" + order_qt_sql_6_21 """select test6, substring(test6, 1, 13) from test2 where substring(test6, 1, 13) = '2022-08-08 20' limit 4;""" + order_qt_sql_6_22 """select test6, substring(test6, 1, 13) from test2 where substring(test6, 1, 13) = '2022-08-08 12' limit 4;""" try { sql """select * from composite_type_array;""" fail("Should not reach here") @@ -295,6 +299,10 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_7_23 """select * from test1 where level = 'debug'""" order_qt_sql_7_24 """select * from test1 where esquery(c_float, '{"match":{"c_float":1.1}}')""" order_qt_sql_7_25 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" + order_qt_sql_7_26 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-08' limit 2;""" + order_qt_sql_7_27 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-09' limit 2;""" + order_qt_sql_7_28 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-10' limit 2;""" + order_qt_sql_7_29 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-11' limit 2;""" try { sql """select * from composite_type_array;""" fail("Should not reach here") @@ -347,6 +355,10 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql_8_21 """select * from test1 where level = 'debug'""" order_qt_sql_8_22 """select * from test1 where esquery(c_ip, '{"match":{"c_ip":"192.168.0.1"}}')""" order_qt_sql_8_23 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;""" + order_qt_sql_8_24 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-08' limit 2;""" + order_qt_sql_8_25 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-09' limit 2;""" + order_qt_sql_8_26 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-10' limit 2;""" + order_qt_sql_8_27 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-11' limit 2;""" try { sql """select * from composite_type_array;""" fail("Should not reach here")