Skip to content

Commit

Permalink
[regression-test](cases) optimize some cases
Browse files Browse the repository at this point in the history
1. modify some variant cases to concurrent and p1
2. rowsstore cases to concurrent
  • Loading branch information
eldenmoon committed Aug 30, 2024
1 parent 20af746 commit 0b102bf
Show file tree
Hide file tree
Showing 11 changed files with 10 additions and 22 deletions.
2 changes: 1 addition & 1 deletion be/src/common/config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -971,7 +971,7 @@ DEFINE_mInt64(workload_group_scan_task_wait_timeout_ms, "10000");
DEFINE_Bool(enable_index_apply_preds_except_leafnode_of_andnode, "true");

DEFINE_mDouble(variant_ratio_of_defaults_as_sparse_column, "1");
DEFINE_mInt64(variant_threshold_rows_to_estimate_sparse_column, "1000");
DEFINE_mInt64(variant_threshold_rows_to_estimate_sparse_column, "2048");
DEFINE_mBool(variant_throw_exeception_on_invalid_json, "false");

// block file cache
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import org.codehaus.groovy.runtime.IOGroovyMethods


suite("test_compaction_uniq_keys_row_store", "nonConcurrent") {
suite("test_compaction_uniq_keys_row_store", "p0") {
def realDb = "regression_test_serving_p0"
def tableName = realDb + ".compaction_uniq_keys_row_store_regression_test"
sql "CREATE DATABASE IF NOT EXISTS ${realDb}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import org.codehaus.groovy.runtime.IOGroovyMethods


suite("test_compaction_uniq_keys_row_store_ck", "nonConcurrent") {
suite("test_compaction_uniq_keys_row_store_ck", "p0") {
if (isCloudMode()) {
logger.info("cloud does not support mow cluster key")
return
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

suite("test_variant_index_format_v1", "p0") {
suite("test_variant_index_format_v1", "p2") {
def calc_file_crc_on_tablet = { ip, port, tablet ->
return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet))
}
Expand Down
17 changes: 3 additions & 14 deletions regression-test/suites/variant_p0/nested.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,12 @@
// specific language governing permissions and limitations
// under the License.

suite("regression_test_variant_nested", "p0,nonConcurrent"){
suite("regression_test_variant_nested", "p0"){
def backendId_to_backendIP = [:]
def backendId_to_backendHttpPort = [:]
getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);

def set_be_config = { key, value ->
for (String backend_id: backendId_to_backendIP.keySet()) {
def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
}
}

try {
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")

def table_name = "var_nested"
sql "DROP TABLE IF EXISTS ${table_name}"
Expand Down Expand Up @@ -135,20 +127,17 @@ suite("regression_test_variant_nested", "p0,nonConcurrent"){
qt_sql """
select * from var_nested order by k limit 101
"""
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
sql """INSERT INTO var_nested SELECT *, '{"k1":1, "k2": "some", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]], "nested1" : {"nested2" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "1000") where number > 200 limit 100;"""
sql """INSERT INTO var_nested SELECT *, '{"k2":1, "k3": "nice", "k4" : [1234], "k5" : 1.10000, "k6" : [[123]], "nested2" : {"nested1" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "5013") where number >= 400;"""
sql """INSERT INTO var_nested SELECT *, '{"k2":1, "k3": "nice", "k4" : [1234], "k5" : 1.10000, "k6" : [[123]], "nested2" : {"nested1" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "5013") where number >= 400 limit 1024;"""
triger_compaction.call()

qt_sql """select /*+SET_VAR(batch_size=1024,broker_load_batch_size=16352,disable_streaming_preaggregations=true,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_
parallel_pipeline_task_num=7,parallel_fragment_exec_instance_num=4,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=false,parallel_scan_max_scanners_count=16
,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=false,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_parallel_result_sink=true,sort_phase_num=0,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=false,enable_local_exchange=true,partitioned_hash_join_rows_threshold=8,partitioned_hash_agg_rows_threshold=8,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=1,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=true,enable_sort_spill=true,enable_agg_spill=true,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ * from var_nested where v['k2'] = 'some' order by k limit 10"""
qt_sql """select * from var_nested where v['k2'] = 'some' and array_contains(cast(v['nested1']['nested2']['a'] as array<tinyint>), 10) order by k limit 1;"""

set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
sql """INSERT INTO var_nested SELECT *, '{"k1":1, "k2": "some", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]], "nested1" : {"nested2" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "4096") where number > 1024 limit 1024;"""
sql """INSERT INTO var_nested SELECT *, '{"k1":1, "k2": "what", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]], "nested1" : {"nested2" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "4096") where number > 2048 limit 1024;"""
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
sql """INSERT INTO var_nested SELECT *, '{"k1":1, "k2": "about", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]], "nested1" : {"nested2" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "4096") where number > 3072 limit 1024;"""
sql """INSERT INTO var_nested SELECT *, '{"k1":1, "k2": "nested", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]], "nested1" : {"nested2" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "6000") where number > 4096 limit 1024;"""
qt_sql """select * from var_nested where v['k2'] = 'what' order by k limit 10"""
Expand All @@ -172,7 +161,7 @@ parallel_pipeline_task_num=7,parallel_fragment_exec_instance_num=4,profile_level
DISTRIBUTED BY HASH(k) BUCKETS 1
properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "variant_enable_flatten_nested" = "true");
"""
sql """insert into var_nested2 select * from var_nested"""
sql """insert into var_nested2 select * from var_nested order by k limit 1024"""
qt_sql """select * from var_nested2 order by k limit 10;"""
qt_sql """select v['nested'] from var_nested2 where k < 10 order by k limit 10;"""
// explode variant array
Expand Down
3 changes: 1 addition & 2 deletions regression-test/suites/variant_p0/with_index/load.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,7 @@ suite("regression_test_variant_with_index", "nonConcurrent"){
qt_sql_inv_3 """select * from var_with_index where inv match 'hello' and cast(v["a"] as int) > 0 order by k"""
sql "truncate table var_with_index"
// set back configs
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95")
set_be_config.call("variant_threshold_rows_to_estimate_sparse_column", "1000")
set_be_config.call("variant_threshold_rows_to_estimate_sparse_column", "2048")
// sql "truncate table ${table_name}"
sql """insert into var_with_index values(1, '{"a1" : 0, "b1": 3}', 'hello world'), (2, '{"a2" : 123}', 'world'),(3, '{"a3" : 123}', 'hello world')"""
sql """insert into var_with_index values(4, '{"b1" : 0, "b2": 3}', 'hello world'), (5, '{"b2" : 123}', 'world'),(6, '{"b3" : 123}', 'hello world')"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import org.codehaus.groovy.runtime.IOGroovyMethods
import org.awaitility.Awaitility

suite("test_compaction_sparse_column", "nonConcurrent") {
suite("test_compaction_sparse_column", "p1,nonConcurrent") {
def tableName = "test_compaction"

try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

import org.codehaus.groovy.runtime.IOGroovyMethods

suite("test_compaction_extract_root", "nonConcurrent") {
suite("test_compaction_extract_root", "p1,nonConcurrent") {
def tableName = "test_t"

def backendId_to_backendIP = [:]
Expand Down

0 comments on commit 0b102bf

Please sign in to comment.