Skip to content

Commit

Permalink
2
Browse files Browse the repository at this point in the history
  • Loading branch information
stephen committed Jul 13, 2024
1 parent 311089b commit 320e8d7
Show file tree
Hide file tree
Showing 17 changed files with 196 additions and 169 deletions.
3 changes: 2 additions & 1 deletion regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock

String s3ak = getS3AK()
String s3sk = getS3SK()
def s3Endpoint = getS3Endpoint()

def cos = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cos}.zd.all_table order by c18"""
def oss = """select * from ${catalog_oss}.paimonossdb1.test_tableoss order by a"""
Expand All @@ -41,7 +42,7 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock
"warehouse" = "cosn://paimon-1308700295/paimoncos",
"cos.access_key" = "${s3ak}",
"cos.secret_key" = "${s3sk}",
"cos.endpoint" = "cos.ap-beijing.myqcloud.com"
"cos.endpoint" = "${s3Endpoint}"
);
"""
sql """
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/github_events_p2/load.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ suite("load") {
ak "${getS3AK()}"
sk "${getS3SK()}"
endpoint "http://${getS3Endpoint()}"
region "ap-beijing"
region "${getS3Region()}"
repository "regression_test_github_events"
snapshot "github_events"
timestamp "2022-03-23-12-19-51"
Expand Down
6 changes: 4 additions & 2 deletions regression-test/suites/load_p0/broker_load/test_compress_type.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
suite("test_compress_type", "load_p0") {
def tableName = "basic_data"
def s3BucketName = getS3BucketName()
def s3Endpoint = getS3Endpoint()
def s3Region = getS3Region()

// GZ/LZO/BZ2/LZ4FRAME/DEFLATE/LZOP
def compressTypes = [
Expand Down Expand Up @@ -138,8 +140,8 @@ suite("test_compress_type", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
"AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing",
"AWS_ENDPOINT" = "${s3Endpoint}",
"AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
"""
Expand Down
5 changes: 3 additions & 2 deletions regression-test/suites/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ suite("test_csv_with_enclose_and_escapeS3_load", "load_p0") {

def tableName = "test_csv_with_enclose_and_escape"
def s3BucketName = getS3BucketName()
def s3Endpoint = getS3Endpoint()

sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
Expand Down Expand Up @@ -93,8 +94,8 @@ suite("test_csv_with_enclose_and_escapeS3_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
"AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing",
"AWS_ENDPOINT" = "${s3Endpoint}",
"AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
${prop}
Expand Down
5 changes: 3 additions & 2 deletions regression-test/suites/load_p0/broker_load/test_etl_failed.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

suite("test_etl_failed", "load_p0") {
def s3BucketName = getS3BucketName()
def s3Endpoint = getS3Endpoint()
def tableName = "test_etl_failed"
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
Expand Down Expand Up @@ -47,8 +48,8 @@ suite("test_etl_failed", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
"AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing",
"AWS_ENDPOINT" = "${s3Endpoint}",
"AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
PROPERTIES(
Expand Down
11 changes: 7 additions & 4 deletions regression-test/suites/load_p0/broker_load/test_multi_table_load.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@

suite("test_multi_table_load", "load_p0") {
def s3BucketName = getS3BucketName()
def s3Endpoint = getS3Endpoint()
def s3Region = getS3Region()

def tableName = "test_multi_table_load"

sql """ DROP TABLE IF EXISTS ${tableName} """
Expand Down Expand Up @@ -129,8 +132,8 @@ suite("test_multi_table_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
"AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing",
"AWS_ENDPOINT" = "${s3Endpoint}",
"AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
properties(
Expand Down Expand Up @@ -178,8 +181,8 @@ suite("test_multi_table_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
"AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing",
"AWS_ENDPOINT" = "${s3Endpoint}",
"AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
properties(
Expand Down
6 changes: 4 additions & 2 deletions regression-test/suites/load_p0/broker_load/test_s3_load.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

suite("test_s3_load", "load_p0") {
def s3BucketName = getS3BucketName()
def s3Endpoint = getS3Endpoint()
def s3Region = getS3Region()
sql "create workload group if not exists broker_load_test properties ( 'cpu_share'='1024'); "

sql "set workload_group=broker_load_test;"
Expand Down Expand Up @@ -217,8 +219,8 @@ suite("test_s3_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
"AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing",
"AWS_ENDPOINT" = "${s3Endpoint}",
"AWS_REGION" = "${s3Region}",
"use_path_style" = "$attributes.usePathStyle",
"provider" = "${getS3Provider()}"
)
Expand Down
6 changes: 4 additions & 2 deletions regression-test/suites/load_p0/broker_load/test_seq_load.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

suite("test_seq_load", "load_p0") {
def s3BucketName = getS3BucketName()
def s3Endpoint = getS3Endpoint()
def s3Region = getS3Region()
def tableName = "uniq_tbl_basic_seq"

sql """ DROP TABLE IF EXISTS ${tableName} """
Expand Down Expand Up @@ -101,8 +103,8 @@ suite("test_seq_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
"AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing"
"AWS_ENDPOINT" = "${s3Endpoint}",
"AWS_REGION" = "${s3Region}"
)
properties(
"use_new_load_scan_node" = "true"
Expand Down
72 changes: 37 additions & 35 deletions regression-test/suites/load_p2/broker_load/test_broker_load.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

suite("test_broker_load_p2", "p2") {
def s3BucketName = getS3BucketName()
def s3Endpoint = getS3Endpoint()
def s3Region = getS3Region()
def tables = ["part",
"upper_case",
"reverse",
Expand Down Expand Up @@ -194,37 +196,37 @@ suite("test_broker_load_p2", "p2") {
"\\N"
]

def task_info = ["cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
"cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0"
def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
"cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0"
]

def error_msg = ["",
Expand Down Expand Up @@ -291,8 +293,8 @@ suite("test_broker_load_p2", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
"AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing",
"AWS_ENDPOINT" = "${s3Endpoint}",
"AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
properties(
Expand Down Expand Up @@ -404,8 +406,8 @@ suite("test_broker_load_p2", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
"AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing",
"AWS_ENDPOINT" = "${s3Endpoint}",
"AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
);
"""
Expand Down
11 changes: 6 additions & 5 deletions regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
// under the License.

suite("test_parquet_large_metadata_load_p2", "p2") {

def s3Endpoint = getS3Endpoint()
def s3Region = getS3Region()
def tables = ["parquet_large_metadata_100mb" // metadata size more than 100MB
]
def paths = ["s3://${getS3BucketName()}/regression/load/metadata/parquet_large_metadata_100mb.parquet"
Expand All @@ -30,7 +31,7 @@ suite("test_parquet_large_metadata_load_p2", "p2") {
"uri" = "https://${getS3BucketName()}.${getS3Endpoint()}/regression/load/metadata/parquet_large_metadata_100mb.parquet",
"s3.access_key" = "$ak",
"s3.secret_key" = "$sk",
"s3.region" = "ap-beijing",
"s3.region" = "${s3Region}",
"provider" = "${getS3Provider()}",
"format" = "parquet"
) order by `1`,`2` limit 5;
Expand All @@ -48,8 +49,8 @@ suite("test_parquet_large_metadata_load_p2", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
"AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing",
"AWS_ENDPOINT" = "${s3Endpoint}",
"AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
PROPERTIES
Expand All @@ -61,7 +62,7 @@ suite("test_parquet_large_metadata_load_p2", "p2") {
}

def etl_info = ["unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=45000"]
def task_info = ["cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0"]
def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0"]
def error_msg = [""]
// test unified load
if (enabled != null && enabled.equalsIgnoreCase("true")) {
Expand Down
18 changes: 10 additions & 8 deletions regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

suite("test_s3_load_properties", "p2") {
def s3BucketName = getS3BucketName()
def s3Endpoint = getS3Endpoint()
def s3Region = getS3Region()
sql "create workload group if not exists broker_load_test properties ( 'cpu_share'='1024'); "

sql "set workload_group=broker_load_test;"
Expand Down Expand Up @@ -87,33 +89,33 @@ suite("test_s3_load_properties", "p2") {
}
// TODO: should be success ?
// for (String table : basicTables) {
// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/${s3BucketName}/regression/load/data/basic_data.csv",
// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "", "").withPathStyle())
// }
//
// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/${s3BucketName}/regression/load/data/basic_data.csv",
// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").withPathStyle())
//
// for (String table : arrayTables) {
// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/${s3BucketName}/regression/load/data/basic_array_data.csv",
// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_array_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "", "").withPathStyle())
// }
//
// for (String table : basicTables) {
// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/${s3BucketName}/regression/load/data/basic_data.csv",
// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "", "").withPathStyle())
// }
//
// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/${s3BucketName}/regression/load/data/basic_data.csv",
// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").withPathStyle())
//
// for (String table : arrayTables) {
// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/${s3BucketName}/regression/load/data/basic_array_data.csv",
// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_array_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "", "").withPathStyle())
// }
Expand Down Expand Up @@ -482,8 +484,8 @@ suite("test_s3_load_properties", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
"AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing",
"AWS_ENDPOINT" = "${s3Endpoint}",
"AWS_REGION" = "${s3Region}",
"use_path_style" = "$attributes.usePathStyle",
"provider" = "${getS3Provider()}"
)
Expand Down
Loading

0 comments on commit 320e8d7

Please sign in to comment.