Skip to content

Commit

Permalink
[regression](s3) add default conf for s3 releated cases (apache#37952)
Browse files Browse the repository at this point in the history 
replace COS with OSS in the TeamCity pipeline to improve stability

<!--Describe your changes.-->

---------

Co-authored-by: stephen <[email protected]>

2
  • Loading branch information
@hello-stephen
hello-stephen authored and stephen committed Jul 29, 2024
1 parent 5b8dda4 commit f59b84f
Show file tree
Hide file tree
Showing 6 changed files with 106 additions and 17 deletions.
16 changes: 10 additions & 6 deletions regression-test/conf/regression-conf.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,16 @@ brokerName = "broker_name"

// broker load test config
enableBrokerLoad=true
ak=""
sk=""

// for s3 releated cases, "aliyun" or "aliyun-internal" or "tencent" or "huawei" or "azure" or "gcp"
// if s3Source is set, s3Endpoint s3BucketName s3Region s3Provider will be filled with default value if not set
s3Source="aliyun"
// s3Endpoint = ""
// s3BucketName = ""
// s3Region = ""
// s3Provider = ""
ak="***********"
sk="***********"

// jdbc connector test config
// To enable jdbc test, you need first start mysql/pg container.
Expand Down Expand Up @@ -161,10 +169,6 @@ aliYunSk="***********"
hwYunAk="***********"
hwYunSk="***********"

s3Endpoint = "cos.ap-hongkong.myqcloud.com"
s3BucketName = "doris-build-hk-1308700295"
s3Region = "ap-hongkong"

// iceberg rest catalog config
iceberg_rest_uri_port=18181

Expand Down
92 changes: 91 additions & 1 deletion regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,17 @@ class Config {
public boolean withOutLoadData
public String caseNamePrefix

public String s3Source

Config() {}

Config(String caseNamePrefix, String defaultDb, String jdbcUrl, String jdbcUser, String jdbcPassword,
Config(String s3Source, String caseNamePrefix, String defaultDb, String jdbcUrl, String jdbcUser, String jdbcPassword,
String feSourceThriftAddress, String feTargetThriftAddress, String feSyncerUser, String feSyncerPassword,
String syncerPassword, String feHttpAddress, String feHttpUser, String feHttpPassword, String metaServiceHttpAddress,
String suitePath, String dataPath, String realDataPath, String cacheDataPath, Boolean enableCacheData,
String testGroups, String excludeGroups, String testSuites, String excludeSuites,
String testDirectories, String excludeDirectories, String pluginPath, String sslCertificatePath) {
this.s3Source = s3Source
this.caseNamePrefix = caseNamePrefix
this.defaultDb = defaultDb
this.jdbcUrl = jdbcUrl
Expand Down Expand Up @@ -280,6 +283,16 @@ class Config {
log.info("withOutLoadData is ${config.withOutLoadData}".toString())
log.info("caseNamePrefix is ${config.caseNamePrefix}".toString())
log.info("dryRun is ${config.dryRun}".toString())
def s3SourceList = ["aliyun", "aliyun-internal", "tencent", "huawei", "azure", "gcp"]
if (s3SourceList.contains(config.s3Source)) {
log.info("s3Source is ${config.s3Source}".toString())
log.info("s3Provider is ${config.otherConfigs.get("s3Provider")}".toString())
log.info("s3BucketName is ${config.otherConfigs.get("s3BucketName")}".toString())
log.info("s3Region is ${config.otherConfigs.get("s3Region")}".toString())
log.info("s3Endpoint is ${config.otherConfigs.get("s3Endpoint")}".toString())
} else {
throw new Exception("The s3Source '${config.s3Source}' is invalid, optional values ${s3SourceList}")
}

Properties props = cmd.getOptionProperties("conf")
config.otherConfigs.putAll(props)
Expand All @@ -292,6 +305,7 @@ class Config {

static Config fromConfigObject(ConfigObject obj) {
def config = new Config(
configToString(obj.s3Source),
configToString(obj.caseNamePrefix),
configToString(obj.defaultDb),
configToString(obj.jdbcUrl),
Expand Down Expand Up @@ -344,6 +358,82 @@ class Config {
}

static void fillDefaultConfig(Config config) {
if (config.s3Source == null) {
config.s3Source = "aliyun"
log.info("Set s3Source to 'aliyun' because not specify.".toString())
}

if (config.otherConfigs.get("s3Provider") == null) {
def s3Provider = "OSS"
if (config.s3Source == "aliyun" || config.s3Source == "aliyun-internal") {
s3Provider = "OSS"
} else if (config.s3Source == "tencent") {
s3Provider = "COS"
} else if (config.s3Source == "huawei") {
s3Provider = "OBS"
} else if (config.s3Source == "azure") {
s3Provider = "AZURE"
} else if (config.s3Source == "gcp") {
s3Provider = "GCP"
}
config.otherConfigs.put("s3Provider", "${s3Provider}")
log.info("Set s3Provider to '${s3Provider}' because not specify.".toString())
}
if (config.otherConfigs.get("s3BucketName") == null) {
def s3BucketName = "doris-regression-hk"
if (config.s3Source == "aliyun") {
s3BucketName = "doris-regression-hk"
} else if (config.s3Source == "aliyun-internal") {
s3BucketName = "doris-regression"
} else if (config.s3Source == "tencent") {
s3BucketName = "doris-build-1308700295"
} else if (config.s3Source == "huawei") {
s3BucketName = "doris-build"
} else if (config.s3Source == "azure") {
s3BucketName = "qa-build"
} else if (config.s3Source == "gcp") {
s3BucketName = "doris-regression"
}
config.otherConfigs.put("s3BucketName", "${s3BucketName}")
log.info("Set s3BucketName to '${s3BucketName}' because not specify.".toString())
}
if (config.otherConfigs.get("s3Region") == null) {
def s3Region = "oss-cn-hongkong"
if (config.s3Source == "aliyun") {
s3Region = "oss-cn-hongkong"
} else if (config.s3Source == "aliyun-internal") {
s3Region = "oss-cn-beijing"
} else if (config.s3Source == "tencent") {
s3Region = "ap-beijing"
} else if (config.s3Source == "huawei") {
s3Region = "cn-north-4"
} else if (config.s3Source == "azure") {
s3Region = "azure-region"
} else if (config.s3Source == "gcp") {
s3Region = "us-central1"
}
config.otherConfigs.put("s3Region", "${s3Region}")
log.info("Set s3Region to '${s3Region}' because not specify.".toString())
}
if (config.otherConfigs.get("s3Endpoint") == null) {
def s3Endpoint = "oss-cn-hongkong.aliyuncs.com"
if (config.s3Source == "aliyun") {
s3Endpoint = "oss-cn-hongkong.aliyuncs.com"
} else if (config.s3Source == "aliyun-internal") {
s3Endpoint = "oss-cn-beijing-internal.aliyuncs.com"
} else if (config.s3Source == "tencent") {
s3Endpoint = "cos.ap-beijing.myqcloud.com"
} else if (config.s3Source == "huawei") {
s3Endpoint = "obs.cn-north-4.myhuaweicloud.com"
} else if (config.s3Source == "azure") {
s3Endpoint = "azure-endpoint"
} else if (config.s3Source == "gcp") {
s3Endpoint = "storage.googleapis.com"
}
config.otherConfigs.put("s3Endpoint", "${s3Endpoint}")
log.info("Set s3Endpoint to '${s3Endpoint}' because not specify.".toString())
}

if (config.caseNamePrefix == null) {
config.caseNamePrefix = ""
log.info("set caseNamePrefix to '' because not specify.".toString())
Expand Down
4 changes: 1 addition & 3 deletions regression-test/pipeline/external/conf/regression-conf.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,7 @@ es_8_port=39200

cacheDataPath = "/data/regression/"

s3Endpoint = "cos.ap-hongkong.myqcloud.com"
s3BucketName = "doris-build-hk-1308700295"
s3Region = "ap-hongkong"
s3Source="aliyun"

max_failure_num=50

Expand Down
4 changes: 1 addition & 3 deletions regression-test/pipeline/p0/conf/regression-conf.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,7 @@ es_8_port=39200

cacheDataPath = "/data/regression/"

s3Endpoint = "cos.ap-hongkong.myqcloud.com"
s3BucketName = "doris-build-hk-1308700295"
s3Region = "ap-hongkong"
s3Source="aliyun"

max_failure_num=50

Expand Down
4 changes: 1 addition & 3 deletions regression-test/pipeline/p1/conf/regression-conf.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,7 @@ excludeSuites = "test_profile,test_broker_load_p2,test_spark_load,test_refresh_m
excludeDirectories = "workload_manager_p1"
cacheDataPath="/data/regression/"

s3Endpoint = "cos.ap-hongkong.myqcloud.com"
s3BucketName = "doris-build-hk-1308700295"
s3Region = "ap-hongkong"
s3Source="aliyun"

max_failure_num=0

Expand Down
3 changes: 2 additions & 1 deletion regression-test/suites/connector_p0/spark_connector/spark_connector.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ suite("spark_connector", "connector") {
logger.info("start download spark doris demo ...")
logger.info("getS3Url ==== ${getS3Url()}")
def download_spark_jar = "/usr/bin/curl ${getS3Url()}/regression/spark-doris-connector-demo-jar-with-dependencies.jar --output spark-doris-demo.jar".execute().getText()
logger.info("finish download spark doris demo ...")
def out = "/usr/bin/ls -al spark-doris-demo.jar".execute().getText()
logger.info("finish download spark doris demo, out: ${out}")
def run_cmd = "java -jar spark-doris-demo.jar $context.config.feHttpAddress $context.config.feHttpUser regression_test_connector_p0_spark_connector.$tableName"
logger.info("run_cmd : $run_cmd")
def run_spark_jar = run_cmd.execute().getText()
Expand Down

0 comments on commit f59b84f

Please sign in to comment.