diff --git a/regression-test/pipeline/common/check-pr-if-need-run-build.sh b/regression-test/pipeline/common/check-pr-if-need-run-build.sh index df4cd3c5102c5a..9510ac12e529fc 100755 --- a/regression-test/pipeline/common/check-pr-if-need-run-build.sh +++ b/regression-test/pipeline/common/check-pr-if-need-run-build.sh @@ -169,7 +169,12 @@ need_run_ckb() { [[ "${af}" == 'gensrc'* ]] || [[ "${af}" == 'thirdparty'* ]] || [[ "${af}" == 'build.sh' ]] || - [[ "${af}" == 'env.sh' ]]; then + [[ "${af}" == 'env.sh' ]] || + [[ "${af}" == 'regression-test/pipeline/common/github-utils.sh' ]] || + [[ "${af}" == 'regression-test/pipeline/common/doris-utils.sh' ]] || + [[ "${af}" == 'regression-test/pipeline/common/oss-utils.sh' ]] || + [[ "${af}" == 'tools/tpch-tools/bin/run-tpch-queries.sh' ]] || + [[ "${af}" == 'regression-test/pipeline/tpch/tpch-sf100/'* ]]; then echo "clickbench performance related file changed, return need" && return 0 fi done diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh index 8cb3cc81e65f5a..3c226955b1552c 100644 --- a/regression-test/pipeline/common/github-utils.sh +++ b/regression-test/pipeline/common/github-utils.sh @@ -260,7 +260,12 @@ file_changed_ckb() { [[ "${af}" == 'gensrc'* ]] || [[ "${af}" == 'thirdparty'* ]] || [[ "${af}" == 'build.sh' ]] || - [[ "${af}" == 'env.sh' ]]; then + [[ "${af}" == 'env.sh' ]] || + [[ "${af}" == 'regression-test/pipeline/common/github-utils.sh' ]] || + [[ "${af}" == 'regression-test/pipeline/common/doris-utils.sh' ]] || + [[ "${af}" == 'regression-test/pipeline/common/oss-utils.sh' ]] || + [[ "${af}" == 'tools/tpch-tools/bin/run-tpch-queries.sh' ]] || + [[ "${af}" == 'regression-test/pipeline/tpch/tpch-sf100/'* ]]; then echo "clickbench performance related file changed, return need" && return 0 fi done diff --git a/regression-test/pipeline/tpch/tpch-sf100/deploy.sh b/regression-test/pipeline/tpch/tpch-sf100/deploy.sh index d018b6df1a98a7..64d816f0acadaf 100644 --- a/regression-test/pipeline/tpch/tpch-sf100/deploy.sh +++ b/regression-test/pipeline/tpch/tpch-sf100/deploy.sh @@ -88,35 +88,22 @@ storage_root_path=$(get_doris_conf_value "${DORIS_HOME}"/be/conf/be.conf storage mkdir -p "${meta_dir}" mkdir -p "${storage_root_path}" if ! start_doris_fe; then - echo "WARNING: Start doris fe failed at first time" + echo "ERROR: Start doris fe failed." print_doris_fe_log - echo "WARNING: delete meta_dir and storage_root_path, then retry" - rm -rf "${meta_dir:?}/"* - rm -rf "${storage_root_path:?}/"* - if ! start_doris_fe; then - need_backup_doris_logs=true - exit_flag=1 - fi + need_backup_doris_logs=true + exit_flag=1 fi if ! start_doris_be; then - echo "WARNING: Start doris be failed at first time" + echo "ERROR: Start doris be failed." print_doris_be_log - echo "WARNING: delete storage_root_path, then retry" - rm -rf "${storage_root_path:?}/"* - if ! start_doris_be; then - need_backup_doris_logs=true - exit_flag=1 - fi -fi -if ! add_doris_be_to_fe; then need_backup_doris_logs=true exit_flag=1 -else - # wait 10s for doris totally started, otherwize may encounter the error below, - # ERROR 1105 (HY000) at line 102: errCode = 2, detailMessage = Failed to find enough backend, please check the replication num,replication tag and storage medium. - sleep 10s fi +# wait 10s for doris totally started, otherwize may encounter the error below, +# ERROR 1105 (HY000) at line 102: errCode = 2, detailMessage = Failed to find enough backend, please check the replication num,replication tag and storage medium. +sleep 10s + echo "#### 5. set session variables" echo "TODO" diff --git a/regression-test/pipeline/tpch/tpch-sf100/prepare.sh b/regression-test/pipeline/tpch/tpch-sf100/prepare.sh index 5376f7c73087ac..07959a03d6f8e7 100644 --- a/regression-test/pipeline/tpch/tpch-sf100/prepare.sh +++ b/regression-test/pipeline/tpch/tpch-sf100/prepare.sh @@ -64,8 +64,6 @@ if ! [[ -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/oss-utils.sh && -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh && -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/github-utils.sh && - -f "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/load-tpch-data.sh && - -f "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/create-tpch-tables.sh && -f "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh ]]; then echo "ERROR: depending files missing" && exit 1 fi diff --git a/regression-test/pipeline/tpch/tpch-sf100/run.sh b/regression-test/pipeline/tpch/tpch-sf100/run.sh index b386fef3a21e9a..d9252f742c1ee6 100644 --- a/regression-test/pipeline/tpch/tpch-sf100/run.sh +++ b/regression-test/pipeline/tpch/tpch-sf100/run.sh @@ -80,48 +80,12 @@ check_tpch_result() { echo "#### 1. check if need to load data" SF="100" # SCALE FACTOR - if ${DEBUG:-false}; then - SF="100" - fi - TPCH_DATA_DIR="/data/tpch/sf_${SF}" # no / at the end - TPCH_DATA_DIR_LINK="${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/tpch-data # no / at the end + if ${DEBUG:-false}; then SF="1"; fi db_name="tpch_sf${SF}" sed -i "s|^export DB=.*$|export DB='${db_name}'|g" \ "${teamcity_build_checkoutDir}"/tools/tpch-tools/conf/doris-cluster.conf if ! check_tpch_table_rows "${db_name}" "${SF}"; then - echo "INFO: need to load tpch-sf${SF} data" - # prepare data - mkdir -p "${TPCH_DATA_DIR}" - ( - cd "${TPCH_DATA_DIR}" || exit 1 - declare -A table_file_count - table_file_count=(['region']=1 ['nation']=1 ['supplier']=1 ['customer']=1 ['part']=1 ['partsupp']=10 ['orders']=10 ['lineitem']=10) - for table_name in ${!table_file_count[*]}; do - if [[ ${table_file_count[${table_name}]} -eq 1 ]]; then - url="https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/tpch/sf${SF}/${table_name}.tbl" - if ! wget --continue -t3 -q "${url}"; then echo "ERROR: wget --continue ${url}" && exit 1; fi - elif [[ ${table_file_count[${table_name}]} -eq 10 ]]; then - ( - for i in {1..10}; do - url="https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/tpch/sf${SF}/${table_name}.tbl.${i}" - if ! wget --continue -t3 -q "${url}"; then echo "ERROR: wget --continue ${url}" && exit 1; fi - done - ) & - wait - fi - done - ) - # create table and load data - sed -i "s|^SCALE_FACTOR=[0-9]\+$|SCALE_FACTOR=${SF}|g" "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/create-tpch-tables.sh - bash "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/create-tpch-tables.sh - rm -rf "${TPCH_DATA_DIR_LINK}" - ln -s "${TPCH_DATA_DIR}" "${TPCH_DATA_DIR_LINK}" - bash "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/load-tpch-data.sh -c 10 - if ! check_tpch_table_rows "${db_name}" "${SF}"; then - exit 1 - fi - echo "INFO: sleep 10min to wait compaction done" && sleep 10m - data_reload="true" + echo "ERROR: check_tpch_table_rows failed." && exit 1 fi echo "#### 2. run tpch-sf${SF} query"