[github](action) fast success if pr no need to run teamcity pipeline

.github/workflows/auto_trigger_teamcity.yml

@@ -37,6 +37,7 @@ jobs:
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       ISSUE_COMMENT: ${{ github.event.comment.body }}
+
     steps:
       - name: Run pipeline by teamcity restful
         run: |
@@ -55,64 +56,286 @@ jobs:
           echo "pull_request_num : ${pull_request_num}"
           echo "encoded_string : ${encoded_string}"
           echo "latest_commit_id : ${{ env.LATEST_COMMIT }}"
-          
+          echo "github.event.issue.number: ${{ github.event.issue.number }}"
+
+          _get_pr_changed_files() {
+            usage_str="Usage:
+            _get_pr_changed_files <PULL_NUMBER> [OPTIONS]
+            note: https://github.com/apache/doris/pull/13259, PULL_NUMBER is 13259
+            OPTIONS can be one of [all|added|modified|removed], default is all
+            "
+            if ! curl --version >/dev/null; then echo 'error: curl required...' && return 1; fi
+            if ! command -v jq >/dev/null; then sudo yum install jq -y || sudo apt install -y jq; fi
+
+            PULL_NUMBER="${1:-${pull_request_num}}"
+            which_file="${2:-all}"
+            pr_url="https://github.com/${{ github.repository }}/pull/${PULL_NUMBER}"
+            try_times=10
+            # The number of results per page (max 100), Default 30.
+            per_page=100
+            file_name="${pull_request_num}_${latest_commit_id}_pr_change_files"
+            if [[ -f "${file_name}" ]]; then
+                echo "find cache ${file_name}"
+            else
+                while [[ ${try_times} -gt 0 ]]; do
+                    if curl \
+                        -H "Accept: application/vnd.github+json" \
+                        https://api.github.com/repos/${{ github.repository }}/pulls/"${PULL_NUMBER}"/files?per_page="${per_page}" \
+                        2>/dev/null >"${file_name}"; then
+                        break
+                    else
+                        try_times=$((try_times - 1))
+                    fi
+                done
+                if [[ ${try_times} = 0 ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 255; fi
+            fi
+
+            all_files=$(jq -r '.[] | .filename' "${file_name}")
+            added_files=$(jq -r '.[] | select(.status == "added") | .filename' "${file_name}")
+            modified_files=$(jq -r '.[] | select(.status == "modified") | .filename' "${file_name}")
+            removed_files=$(jq -r '.[] | select(.status == "removed") | .filename' "${file_name}")
+            echo "---------------------------------------------------------------"
+            cat "${file_name}"
+            echo "---------------------------------------------------------------"
+            if [[ -z "${all_files}" ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 255; fi
+
+            echo -e " https://github.com/${{ github.repository }}/${PULL_NUMBER}/files all change files:\n---------------------------------------------------------------"
+                  if [[ "${which_file:-all}" == "all" ]]; then
+                      echo -e "${all_files}\n" && export all_files
+                  elif [[ "${which_file}" == "added" ]]; then
+                      echo -e "${added_files}\n" && export added_files
+                  elif [[ "${which_file}" == "modified" ]]; then
+                      echo -e "${modified_files}\n" && export modified_files
+                  elif [[ "${which_file}" == "removed" ]]; then
+                      echo -e "${removed_files}\n" && export removed_files
+                  else
+                      return 1
+                  fi
+              }
+
+          _only_modified_regression_conf() {
+              if [[ -n ${added_files} ]]; then echo "Not only modified regression conf, find added files" && return 1; fi
+              for f in ${modified_files}; do
+                  if [[ "${f}" == "regression-test/pipeline/p0/conf/regression-conf.groovy" ]] ||
+                      [[ "${f}" == "regression-test/pipeline/p1/conf/regression-conf.groovy" ]]; then
+                      continue
+                  else
+                      echo "Not only modified regression conf" && return 1
+                  fi
+              done
+              echo "only modified regression conf" && return 0
+          }
+
+          need_run_fe_ut() {
+              if ! _get_pr_changed_files "$1"; then echo "get pr changed files failed, return need" && return 0; fi
+              if _only_modified_regression_conf; then echo "return no need" && return 1; fi
+              for af in ${all_files}; do
+                  if [[ "${af}" == 'fe'* ]] ||
+                      [[ "${af}" == 'fe_plugins'* ]] ||
+                      [[ "${af}" == 'bin/start_fe.sh' ]] ||
+                      [[ "${af}" == 'docs/zh-CN/docs/sql-manual/'* ]] ||
+                      [[ "${af}" == 'docs/en/docs/sql-manual/'* ]] ||
+                      [[ "${af}" == 'bin/stop_fe.sh' ]] ||
+                      [[ "${af}" == 'run-fe-ut.sh' ]]; then echo "fe-ut related file changed, return need" && return 0; fi
+              done
+              echo "return no need" && return 1
+          }
+
+          need_run_be_ut() {
+              if ! _get_pr_changed_files "$1"; then echo "get pr changed files failed, return need" && return 0; fi
+              if _only_modified_regression_conf; then echo "return no need" && return 1; fi
+              for af in ${all_files}; do
+                  if [[ "${af}" == 'be'* ]] ||
+                      [[ "${af}" == 'contrib'* ]] ||
+                      [[ "${af}" == 'thirdparty'* ]] ||
+                      [[ "${af}" == 'bin/start_be.sh' ]] ||
+                      [[ "${af}" == 'bin/stop_be.sh' ]] ||
+                      [[ "${af}" == 'run-be-ut.sh' ]]; then
+                      echo "be-ut related file changed, return need" && return 0
+                  fi
+              done
+              echo "return no need" && return 1
+          }
+
+          need_run_regression_p0() {
+              if ! _get_pr_changed_files "$1"; then echo "get pr changed files failed, return need" && return 0; fi
+              if _only_modified_regression_conf; then echo "return no need" && return 1; fi
+              for af in ${all_files}; do
+                  if [[ "${af}" == 'be'* ]] ||
+                      [[ "${af}" == 'bin'* ]] ||
+                      [[ "${af}" == 'conf'* ]] ||
+                      [[ "${af}" == 'contrib'* ]] ||
+                      [[ "${af}" == 'fe'* ]] ||
+                      [[ "${af}" == 'fe_plugins'* ]] ||
+                      [[ "${af}" == 'gensrc'* ]] ||
+                      [[ "${af}" == 'regression-test'* ]] ||
+                      [[ "${af}" == 'thirdparty'* ]] ||
+                      [[ "${af}" == 'ui'* ]] ||
+                      [[ "${af}" == 'webroot'* ]] ||
+                      [[ "${af}" == 'build.sh' ]] ||
+                      [[ "${af}" == 'env.sh' ]] ||
+                      [[ "${af}" == 'run-regression-test.sh' ]]; then
+                      echo "regression related file changed, return need" && return 0
+                  fi
+              done
+              echo "return no need" && return 1
+          }
+
+          need_run_regression_p1() {
+              need_run_regression_p0 "$1"
+          }
+
+          need_run_arm_regression_p0() {
+              if [[ $(($1 % 2)) -eq 0 ]]; then echo "the pull request id is even, return no need" && return 1; fi
+              need_run_regression_p0 "$1"
+          }
+
+          need_run_ckb() {
+              if ! _get_pr_changed_files "$1"; then echo "get pr changed files failed, return need" && return 0; fi
+              if _only_modified_regression_conf; then echo "return no need" && return 1; fi
+              for af in ${all_files}; do
+                  if [[ "${af}" == 'be'* ]] ||
+                      [[ "${af}" == 'bin'* ]] ||
+                      [[ "${af}" == 'conf'* ]] ||
+                      [[ "${af}" == 'fe'* ]] ||
+                      [[ "${af}" == 'gensrc'* ]] ||
+                      [[ "${af}" == 'thirdparty'* ]] ||
+                      [[ "${af}" == 'build.sh' ]] ||
+                      [[ "${af}" == 'env.sh' ]]; then
+                      echo "clickbench performance related file changed, return need" && return 0
+                  fi
+              done
+              echo "return no need" && return 1
+          }
+
+          _create_a_commit_status() {
+              usage="Usage: _create_a_commit_status <PIPELINE> <STATE>
+              PIPELINE options: [ p0 | p1 | feut | beut | compile | clickbench | external | arm ]
+              STATE options: [ success | failure | error | pending ]"
+
+              if [[ $# -ne 2 ]]; then echo "${usage}" && exit 1; fi
+
+              if [[ $1 == p0 ]]; then
+                  context="P0 Regression (Doris Regression)"
+              elif [[ $1 == p1 ]]; then
+                  context="P1 Regression (Doris Regression)"
+              elif [[ $1 == feut ]]; then
+                  context="FE UT (Doris FE UT)"
+              elif [[ $1 == beut ]]; then
+                  context="BE UT (Doris BE UT)"
+              elif [[ $1 == compile ]]; then
+                  context="COMPILE (DORIS_COMPILE)"
+              elif [[ $1 == clickbench ]]; then
+                  context="clickbench-new (clickbench)"
+              elif [[ $1 == external ]]; then
+                  context="External Regression (Doris External Regression)"
+              elif [[ $1 == arm ]]; then
+                  context="P0 Regression (ARM pipeline)"
+              else
+                  echo "${usage}" && exit 1
+              fi
+
+              if [[ "$2" == "success" || "$2" == "failure" || "$2" == "error" || "$2" == "pending" ]]; then
+                  state="$2"
+                  echo "create a commit status $2 on pipleline $1"
+              else
+                  echo "${usage}" && exit 1
+              fi
+
+              curl -s -L -X POST \
+                  -H "Accept: application/vnd.github+json" \
+                  -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+                  -H "X-GitHub-Api-Version: 2022-11-28" \
+                  https://api.github.com/repos/${{ github.repository }}/statuses/${{ env.LATEST_COMMIT }} \
+                  -d "{\"state\":\"${state}\",\"target_url\":\"\",\"description\":\"no need to run\",\"context\":\"${context}\"}"
+          }
+
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "buildall" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_Doris_FeUt Doris_DorisBeUt_BeUt Doris_DorisCompile_Compile Doris_Performance_Clickbench_ClickbenchNew Doris_ArmPipeline_P0Regression ${trigger_pipelines}"
+            comment_message="run compile p0 p1 external feut beut clickbench arm"
           fi
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "p0" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisRegression_P0Regression  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "nereids_p0" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisRegression_NereidsP0Regression  ${trigger_pipelines}"
+            if need_run_regression_p0; then
+              trigger_pipelines="Doris_DorisRegression_P0Regression  ${trigger_pipelines}"
+            else
+              _create_a_commit_status p0 success
+            fi
           fi
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "p1" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisRegression_P1Regression  ${trigger_pipelines}"
+            if need_run_regression_p1; then
+              trigger_pipelines="Doris_DorisRegression_P1Regression  ${trigger_pipelines}"
+            else
+              _create_a_commit_status p1 success
+            fi
           fi
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "feut" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_Doris_FeUt  ${trigger_pipelines}"
+            if need_run_fe_ut; then
+              trigger_pipelines="Doris_Doris_FeUt  ${trigger_pipelines}"
+            else
+              _create_a_commit_status feut success
+            fi
           fi
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "beut" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisBeUt_BeUt  ${trigger_pipelines}"
+            if need_run_be_ut; then
+              trigger_pipelines="Doris_DorisBeUt_BeUt  ${trigger_pipelines}"
+            else
+              _create_a_commit_status beut success
+            fi
           fi
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "compile" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisCompile_Compile  ${trigger_pipelines}"
+            if need_run_regression_p0; then
+              trigger_pipelines="Doris_DorisCompile_Compile  ${trigger_pipelines}"
+            else
+              _create_a_commit_status compile success
+            fi
           fi
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "clickbench" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_Performance_Clickbench_ClickbenchNew  ${trigger_pipelines}"
+            if need_run_ckb; then
+              trigger_pipelines="Doris_Performance_Clickbench_ClickbenchNew  ${trigger_pipelines}"
+            else
+              _create_a_commit_status clickbench success
+            fi
           fi
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "arm" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_ArmPipeline_P0Regression  ${trigger_pipelines}"
+            if need_run_arm_regression_p0; then
+              trigger_pipelines="Doris_ArmPipeline_P0Regression  ${trigger_pipelines}"
+            else
+              _create_a_commit_status arm success
+            fi
           fi
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "external" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_External_Regression  ${trigger_pipelines}"
+            if need_run_regression_p0; then
+              trigger_pipelines="Doris_External_Regression  ${trigger_pipelines}"
+            else
+              _create_a_commit_status external success
+            fi
           fi
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "just_for_test" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
             trigger_pipelines="Doris_DorisRegression_ExternalRegression  ${trigger_pipelines}"
           fi
-          
+
           if [ -z "${trigger_pipelines}" ];then
             echo "Just a general comment that doesn't match any pipeline rules"
           fi
-          
+
           echo "need run pipelines: ${trigger_pipelines}"
-          
+
           for pipeline in ${trigger_pipelines}
           do
             echo "-----------------------------------------------------------------"
             running_builds_command="curl -s -X GET ${teamcity_url}/app/rest/builds\?locator\=buildType\:${pipeline}\,branch:pull/${pull_request_num}\,running:true"
             echo "${running_builds_command}" 
             running_builds_list=$(eval "${running_builds_command}" > running_builds.json && jq -r '.build[].id' running_builds.json)
             echo "running_builds_list : ${running_builds_list}"
-          
+
             queue_builds_command="curl -s -X GET ${teamcity_url}/app/rest/buildQueue\?locator\=buildType\:${pipeline}"
             echo "${queue_builds_command}"
             eval "${queue_builds_command}" > queue_builds.json
             queue_builds_list=$(cat queue_builds.json | jq ".build[] | select(.branchName == \"pull/${pull_request_num}\") | .id" )
             echo "queue builds list: ${queue_builds_list}"
-          
+
             merged_list=("${running_builds_list[@]} ${queue_builds_list[@]}")
             echo "merged_list :  ${merged_list}"
-          
+
             for build in ${merged_list}; do
               running_commit_command="curl -s -X GET ${teamcity_url}/app/rest/builds/${build}"
               echo "${running_commit_command}"
@@ -126,7 +349,7 @@ jobs:
                 break
               fi
             done
-          
+
             if [ "_""${same_build_sign}" == "_false" ];then
               sleep 10s
               echo "there is no running build or queue build, so trigger a new !"
@@ -139,5 +362,3 @@ jobs:
               echo "-----------------------------------------------------------------"
             fi
           done
-          
-