Update file_factory.cpp

.github/workflows/clang-format.yml

@@ -17,7 +17,7 @@
 #
 
 ---
-name: BE Code Formatter
+name: Code Formatter
 
 on: [push, pull_request_target]
 
@@ -52,9 +52,11 @@ jobs:
         id: filter
         with:
           filters: |
-            be_changes:
+            changes:
               - 'be/src/**'
               - 'be/test/**'
+              - 'cloud/src/**'
+              - 'cloud/test/**'
 
       - name: Checkout clang-format-lint-action
         run: |
@@ -66,14 +68,14 @@ jobs:
           popd &>/dev/null
 
       - name: "Format it!"
-        if: ${{ steps.filter.outputs.be_changes == 'true' }}
+        if: ${{ steps.filter.outputs.changes == 'true' }}
         uses: ./.github/actions/clang-format-lint-action
-        id: be_clang_format
+        id: clang_format
         with:
-          source: "be/src be/test"
+          source: "be/src be/test cloud/src cloud/test"
           clangFormatVersion: 16
           inplace: False
 
       - name: Ignore it!
-        if: ${{ steps.filter.outputs.be_changes == 'false' }}
+        if: ${{ steps.filter.outputs.changes == 'false' }}
         run: echo 'No need to check.'

.github/workflows/comment-to-trigger-teamcity.yml

@@ -0,0 +1,332 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Comment to trigger teamcity
+
+on:
+  issue_comment:
+    types: [created, edited]
+
+jobs:
+  check-comment-if-need-to-trigger-teamcity:
+
+    # This job only runs for pull request comments, and comment body contains 'run'
+    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') }}
+
+    runs-on: ubuntu-latest
+    env:
+      COMMENT_BODY: ${{ github.event.comment.body }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+    - name: "Parse PR comment"
+      id: parse
+      run: |
+        COMMENT_BODY=$(echo "${COMMENT_BODY}" | xargs)
+        if [[ "${COMMENT_BODY}" == *'run buildall'* ||
+            "${COMMENT_BODY}" == *'run compile'* ||
+            "${COMMENT_BODY}" == *'run beut'* ||
+            "${COMMENT_BODY}" == *'run feut'* ||
+            "${COMMENT_BODY}" == *'run cloudut'* ||
+            "${COMMENT_BODY}" == *'run p0'* ||
+            "${COMMENT_BODY}" == *'run p1'* ||
+            "${COMMENT_BODY}" == *'run external'* ||
+            "${COMMENT_BODY}" == *'run cloud_p0'* ||
+            "${COMMENT_BODY}" == *'run cloud_p1'* ||
+            "${COMMENT_BODY}" == *'run arm'* ||
+            "${COMMENT_BODY}" == *'run performance'* ]]; then
+            echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT"
+        else
+            echo "comment_trigger=false" | tee -a "$GITHUB_OUTPUT"
+            echo "find no keyword in comment body, skip this action."
+            exit
+        fi
+
+        PULL_REQUEST_NUM="$(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}')"
+        COMMIT_ID_FROM_TRIGGER="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.head.sha')"
+        TARGET_BRANCH="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.base.ref')"
+        echo "PULL_REQUEST_NUM=${PULL_REQUEST_NUM}" | tee -a "$GITHUB_OUTPUT"
+        echo "COMMIT_ID_FROM_TRIGGER=${COMMIT_ID_FROM_TRIGGER}" | tee -a "$GITHUB_OUTPUT"
+        echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT"
+        echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT"
+
+        reg="run (buildall|compile|p0|p1|feut|beut|cloudut|external|clickbench|cloud_p0|cloud_p1|arm|performance)( [1-9]*[0-9]+)*"
+        COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')"
+        COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')"
+        echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT"
+        echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" | tee -a "$GITHUB_OUTPUT"
+
+    - name: "Checkout master"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) }}
+      uses: actions/checkout@v4
+
+    - name: "Check if pr need run build"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) }}
+      id: changes
+      run: |
+        source regression-test/pipeline/common/github-utils.sh
+        set -x
+        if _get_pr_changed_files "${{ steps.parse.outputs.PULL_REQUEST_NUM }}"; then
+          if file_changed_fe_ut; then
+            echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_fe_ut=false" | tee -a "$GITHUB_OUTPUT"
+          fi
+          if file_changed_be_ut; then
+            echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_be_ut=false" | tee -a "$GITHUB_OUTPUT"
+          fi
+          if file_changed_cloud_ut; then
+            echo "changed_cloud_ut=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_cloud_ut=false" | tee -a "$GITHUB_OUTPUT"
+          fi
+          if file_changed_regression_p0; then
+            echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
+            echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
+            echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_p0=false" | tee -a "$GITHUB_OUTPUT"
+            echo "changed_external=false" | tee -a "$GITHUB_OUTPUT"
+            echo "changed_arm=false" | tee -a "$GITHUB_OUTPUT"
+          fi
+          if file_changed_regression_p1; then
+            echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_p1=false" | tee -a "$GITHUB_OUTPUT"
+          fi
+          if file_changed_performance; then
+            echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_performance=false" | tee -a "$GITHUB_OUTPUT"
+          fi
+          if file_changed_cloud_p0; then
+            echo "changed_cloud_p0=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_cloud_p0=false" | tee -a "$GITHUB_OUTPUT"
+          fi
+          if file_changed_cloud_p1; then
+            echo "changed_cloud_p1=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_cloud_p1=false" | tee -a "$GITHUB_OUTPUT"
+          fi
+        else
+          echo "INFO: failed to _get_pr_changed_files, default trigger all"
+          echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_cloud_ut=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_cloud_p0=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_cloud_p1=true" | tee -a "$GITHUB_OUTPUT"
+        fi
+
+    # - name: "Setup tmate session"
+    #   if: ${{ failure() }}
+    #   uses: mxschmitt/action-tmate@v3
+
+    - name: "Trigger or Skip feut"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        set -x
+        trigger_or_skip_build \
+          "${{ steps.changes.outputs.changed_fe_ut }}" \
+          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+          "feut" \
+          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
+
+    - name: "Trigger or Skip beut"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        set -x
+        trigger_or_skip_build \
+          "${{ steps.changes.outputs.changed_be_ut }}" \
+          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+          "beut" \
+          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
+    - name: "Trigger or Skip cloudut"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloudut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        set -x
+        trigger_or_skip_build \
+          "${{ steps.changes.outputs.changed_cloud_ut }}" \
+          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+          "cloudut" \
+          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
+    - name: "Trigger or Skip compile"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["compile", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        set -x
+        if [[ "${{ steps.changes.outputs.changed_p0 }}" == "true" ]] || [[ "${{ steps.changes.outputs.changed_cloud_p0 }}" == "true" ]]; then
+          changed="true"
+        else
+          changed="false"
+        fi
+        trigger_or_skip_build \
+          "${changed}" \
+          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+          "compile" \
+          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
+    - name: "Trigger or Skip p0"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then
+          echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger p0" && exit
+        fi
+        set -x
+        trigger_or_skip_build \
+          "${{ steps.changes.outputs.changed_p0 }}" \
+          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+          "p0" \
+          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
+    - name: "Trigger or Skip p1"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["p1", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}
+        if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then
+          echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger p1" && exit
+        fi
+        set -x
+        trigger_or_skip_build \
+          "${{ steps.changes.outputs.changed_p1 }}" \
+          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+          "p1" \
+          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
+    - name: "Trigger or Skip external"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["external", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then
+          echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger external" && exit
+        fi
+        set -x
+        trigger_or_skip_build \
+          "${{ steps.changes.outputs.changed_external }}" \
+          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+          "external" \
+          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
+    - name: "Trigger or Skip cloud_p0"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloud_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then
+          echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p0" && exit
+        fi
+        set -x
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
+          echo "PR target branch in (master), need run cloud_p0"
+          trigger_or_skip_build \
+            "${{ steps.changes.outputs.changed_cloud_p0 }}" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "cloud_p0" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        else
+          echo "PR target branch not in (master), skip run cloud_p0"
+          trigger_or_skip_build \
+            "false" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "cloud_p0" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        fi
+
+    - name: "Trigger or Skip cloud_p1"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloud_p1", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then
+          echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p1" && exit
+        fi
+        set -x
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
+          echo "PR target branch in (master), need run cloud_p1"
+          trigger_or_skip_build \
+            "${{ steps.changes.outputs.changed_cloud_p1 }}" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "cloud_p1" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        else
+          echo "PR target branch not in (master), skip run cloud_p1"
+          trigger_or_skip_build \
+            "false" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "cloud_p1" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        fi
+
+    - name: "Trigger or Skip arm"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        set -x
+        trigger_or_skip_build \
+          "${{ steps.changes.outputs.changed_arm }}" \
+          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+          "arm" \
+          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
+    - name: "Trigger or Skip performance"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["performance", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        set -x
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then
+          echo "PR target branch in (master, branch-2.0), need run performance"
+          trigger_or_skip_build \
+            "${{ steps.changes.outputs.changed_performance }}" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "performance" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        else
+          echo "PR target branch not in (master, branch-2.0), skip run performance"
+          trigger_or_skip_build \
+            "false" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "performance" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        fi

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

@@ -30,7 +30,7 @@ public class Config extends ConfigBase {
 
     /**
      * sys_log_dir:
-     *      This specifies FE log dir. FE will produces 2 log files:
+     *      This specifies FE log dir. FE will produce 2 log files:
      *      fe.log:      all logs of FE process.
      *      fe.warn.log  all WARN and ERROR log of FE process.
      *

regression-test/pipeline/common/check-pr-if-need-run-build.sh

@@ -46,9 +46,10 @@ _get_pr_changed_files() {
     per_page=100
     file_name='pr_change_files'
     while [[ ${try_times} -gt 0 ]]; do
+        # TODO: 修改文件多于100个的情况未处理
         if curl \
             -H "Accept: application/vnd.github+json" \
-            https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}"/files?per_page="${per_page}" \
+            https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}"/files?page=1\&per_page="${per_page}" \
             2>/dev/null >"${file_name}"; then
             break
         else