From 0b102bff35457c05a9b6af07655f33444beeb543 Mon Sep 17 00:00:00 2001 From: eldenmoon <15605149486@163.com> Date: Fri, 30 Aug 2024 13:40:39 +0800 Subject: [PATCH] [regression-test](cases) optimize some cases 1. modify some variant cases to concurrent and p1 2. rowsstore cases to concurrent --- be/src/common/config.cpp | 2 +- .../test_variant_index_format_v1.out | 0 .../compaction}/compaction_sparse_column.out | 0 .../test_compaction_extract_root.out | 0 .../test_compaction_uniq_keys_row_store.groovy | 2 +- ...est_compaction_uniq_keys_row_store_ck.groovy | 2 +- .../test_variant_index_format_v1.groovy | 2 +- regression-test/suites/variant_p0/nested.groovy | 17 +++-------------- .../suites/variant_p0/with_index/load.groovy | 3 +-- .../compaction}/compaction_sparse_column.groovy | 2 +- .../test_compaction_extract_root.groovy | 2 +- 11 files changed, 10 insertions(+), 22 deletions(-) rename regression-test/data/{inverted_index_p0 => inverted_index_p2}/test_variant_index_format_v1.out (100%) rename regression-test/data/{variant_p0 => variant_p1/compaction}/compaction_sparse_column.out (100%) rename regression-test/data/{variant_p0 => variant_p1/compaction}/test_compaction_extract_root.out (100%) rename regression-test/suites/{inverted_index_p0 => inverted_index_p2}/test_variant_index_format_v1.groovy (99%) rename regression-test/suites/{variant_p0 => variant_p1/compaction}/compaction_sparse_column.groovy (99%) rename regression-test/suites/{variant_p0 => variant_p1/compaction}/test_compaction_extract_root.groovy (99%) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index f051bb3e8569b0..d9033445001ac1 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -971,7 +971,7 @@ DEFINE_mInt64(workload_group_scan_task_wait_timeout_ms, "10000"); DEFINE_Bool(enable_index_apply_preds_except_leafnode_of_andnode, "true"); DEFINE_mDouble(variant_ratio_of_defaults_as_sparse_column, "1"); -DEFINE_mInt64(variant_threshold_rows_to_estimate_sparse_column, "1000"); +DEFINE_mInt64(variant_threshold_rows_to_estimate_sparse_column, "2048"); DEFINE_mBool(variant_throw_exeception_on_invalid_json, "false"); // block file cache diff --git a/regression-test/data/inverted_index_p0/test_variant_index_format_v1.out b/regression-test/data/inverted_index_p2/test_variant_index_format_v1.out similarity index 100% rename from regression-test/data/inverted_index_p0/test_variant_index_format_v1.out rename to regression-test/data/inverted_index_p2/test_variant_index_format_v1.out diff --git a/regression-test/data/variant_p0/compaction_sparse_column.out b/regression-test/data/variant_p1/compaction/compaction_sparse_column.out similarity index 100% rename from regression-test/data/variant_p0/compaction_sparse_column.out rename to regression-test/data/variant_p1/compaction/compaction_sparse_column.out diff --git a/regression-test/data/variant_p0/test_compaction_extract_root.out b/regression-test/data/variant_p1/compaction/test_compaction_extract_root.out similarity index 100% rename from regression-test/data/variant_p0/test_compaction_extract_root.out rename to regression-test/data/variant_p1/compaction/test_compaction_extract_root.out diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy index 15cdef1466c2be..86a338b301de22 100644 --- a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy +++ b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy @@ -18,7 +18,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods -suite("test_compaction_uniq_keys_row_store", "nonConcurrent") { +suite("test_compaction_uniq_keys_row_store", "p0") { def realDb = "regression_test_serving_p0" def tableName = realDb + ".compaction_uniq_keys_row_store_regression_test" sql "CREATE DATABASE IF NOT EXISTS ${realDb}" diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store_ck.groovy b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store_ck.groovy index 9d38d5e6c9af01..bf4e8dc1a51004 100644 --- a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store_ck.groovy +++ b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store_ck.groovy @@ -18,7 +18,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods -suite("test_compaction_uniq_keys_row_store_ck", "nonConcurrent") { +suite("test_compaction_uniq_keys_row_store_ck", "p0") { if (isCloudMode()) { logger.info("cloud does not support mow cluster key") return diff --git a/regression-test/suites/inverted_index_p0/test_variant_index_format_v1.groovy b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy similarity index 99% rename from regression-test/suites/inverted_index_p0/test_variant_index_format_v1.groovy rename to regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy index 153e8b82f56532..cc44ac2bb38234 100644 --- a/regression-test/suites/inverted_index_p0/test_variant_index_format_v1.groovy +++ b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_variant_index_format_v1", "p0") { +suite("test_variant_index_format_v1", "p2") { def calc_file_crc_on_tablet = { ip, port, tablet -> return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) } diff --git a/regression-test/suites/variant_p0/nested.groovy b/regression-test/suites/variant_p0/nested.groovy index bf162250bd81d6..a2a3355854f7c6 100644 --- a/regression-test/suites/variant_p0/nested.groovy +++ b/regression-test/suites/variant_p0/nested.groovy @@ -15,20 +15,12 @@ // specific language governing permissions and limitations // under the License. -suite("regression_test_variant_nested", "p0,nonConcurrent"){ +suite("regression_test_variant_nested", "p0"){ def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - def set_be_config = { key, value -> - for (String backend_id: backendId_to_backendIP.keySet()) { - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - } - try { - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") def table_name = "var_nested" sql "DROP TABLE IF EXISTS ${table_name}" @@ -135,9 +127,8 @@ suite("regression_test_variant_nested", "p0,nonConcurrent"){ qt_sql """ select * from var_nested order by k limit 101 """ - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") sql """INSERT INTO var_nested SELECT *, '{"k1":1, "k2": "some", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]], "nested1" : {"nested2" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "1000") where number > 200 limit 100;""" - sql """INSERT INTO var_nested SELECT *, '{"k2":1, "k3": "nice", "k4" : [1234], "k5" : 1.10000, "k6" : [[123]], "nested2" : {"nested1" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "5013") where number >= 400;""" + sql """INSERT INTO var_nested SELECT *, '{"k2":1, "k3": "nice", "k4" : [1234], "k5" : 1.10000, "k6" : [[123]], "nested2" : {"nested1" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "5013") where number >= 400 limit 1024;""" triger_compaction.call() qt_sql """select /*+SET_VAR(batch_size=1024,broker_load_batch_size=16352,disable_streaming_preaggregations=true,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_ @@ -145,10 +136,8 @@ parallel_pipeline_task_num=7,parallel_fragment_exec_instance_num=4,profile_level ,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=false,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_parallel_result_sink=true,sort_phase_num=0,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=false,enable_local_exchange=true,partitioned_hash_join_rows_threshold=8,partitioned_hash_agg_rows_threshold=8,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=1,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=true,enable_sort_spill=true,enable_agg_spill=true,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ * from var_nested where v['k2'] = 'some' order by k limit 10""" qt_sql """select * from var_nested where v['k2'] = 'some' and array_contains(cast(v['nested1']['nested2']['a'] as array), 10) order by k limit 1;""" - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") sql """INSERT INTO var_nested SELECT *, '{"k1":1, "k2": "some", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]], "nested1" : {"nested2" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "4096") where number > 1024 limit 1024;""" sql """INSERT INTO var_nested SELECT *, '{"k1":1, "k2": "what", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]], "nested1" : {"nested2" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "4096") where number > 2048 limit 1024;""" - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") sql """INSERT INTO var_nested SELECT *, '{"k1":1, "k2": "about", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]], "nested1" : {"nested2" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "4096") where number > 3072 limit 1024;""" sql """INSERT INTO var_nested SELECT *, '{"k1":1, "k2": "nested", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]], "nested1" : {"nested2" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "6000") where number > 4096 limit 1024;""" qt_sql """select * from var_nested where v['k2'] = 'what' order by k limit 10""" @@ -172,7 +161,7 @@ parallel_pipeline_task_num=7,parallel_fragment_exec_instance_num=4,profile_level DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "variant_enable_flatten_nested" = "true"); """ - sql """insert into var_nested2 select * from var_nested""" + sql """insert into var_nested2 select * from var_nested order by k limit 1024""" qt_sql """select * from var_nested2 order by k limit 10;""" qt_sql """select v['nested'] from var_nested2 where k < 10 order by k limit 10;""" // explode variant array diff --git a/regression-test/suites/variant_p0/with_index/load.groovy b/regression-test/suites/variant_p0/with_index/load.groovy index 93737e8a5b9dc8..ba46e7a9eeed8b 100644 --- a/regression-test/suites/variant_p0/with_index/load.groovy +++ b/regression-test/suites/variant_p0/with_index/load.groovy @@ -67,8 +67,7 @@ suite("regression_test_variant_with_index", "nonConcurrent"){ qt_sql_inv_3 """select * from var_with_index where inv match 'hello' and cast(v["a"] as int) > 0 order by k""" sql "truncate table var_with_index" // set back configs - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") - set_be_config.call("variant_threshold_rows_to_estimate_sparse_column", "1000") + set_be_config.call("variant_threshold_rows_to_estimate_sparse_column", "2048") // sql "truncate table ${table_name}" sql """insert into var_with_index values(1, '{"a1" : 0, "b1": 3}', 'hello world'), (2, '{"a2" : 123}', 'world'),(3, '{"a3" : 123}', 'hello world')""" sql """insert into var_with_index values(4, '{"b1" : 0, "b2": 3}', 'hello world'), (5, '{"b2" : 123}', 'world'),(6, '{"b3" : 123}', 'hello world')""" diff --git a/regression-test/suites/variant_p0/compaction_sparse_column.groovy b/regression-test/suites/variant_p1/compaction/compaction_sparse_column.groovy similarity index 99% rename from regression-test/suites/variant_p0/compaction_sparse_column.groovy rename to regression-test/suites/variant_p1/compaction/compaction_sparse_column.groovy index 0a7ea100373452..91f64c19a02d22 100644 --- a/regression-test/suites/variant_p0/compaction_sparse_column.groovy +++ b/regression-test/suites/variant_p1/compaction/compaction_sparse_column.groovy @@ -18,7 +18,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods import org.awaitility.Awaitility -suite("test_compaction_sparse_column", "nonConcurrent") { +suite("test_compaction_sparse_column", "p1,nonConcurrent") { def tableName = "test_compaction" try { diff --git a/regression-test/suites/variant_p0/test_compaction_extract_root.groovy b/regression-test/suites/variant_p1/compaction/test_compaction_extract_root.groovy similarity index 99% rename from regression-test/suites/variant_p0/test_compaction_extract_root.groovy rename to regression-test/suites/variant_p1/compaction/test_compaction_extract_root.groovy index 43f1048f1510c3..83dc0a559e6fc5 100644 --- a/regression-test/suites/variant_p0/test_compaction_extract_root.groovy +++ b/regression-test/suites/variant_p1/compaction/test_compaction_extract_root.groovy @@ -17,7 +17,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods -suite("test_compaction_extract_root", "nonConcurrent") { +suite("test_compaction_extract_root", "p1,nonConcurrent") { def tableName = "test_t" def backendId_to_backendIP = [:]