Merge remote-tracking branch 'github_alibaba/feat/pipeline' into merg…

…eMain
alibaba · Dec 8, 2023 · fd4ac3b · fd4ac3b
2 parents 736b1b1 + 3e50f45
commit fd4ac3b
Show file tree

Hide file tree

Showing 37 changed files with 459 additions and 263 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail-build-linux:gcc_9.3.1-2
+FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail-build-linux:gcc_9.3.1-3
 
 ARG USERNAME=admin
 USER root

diff --git a/.github/workflows/build-core-ut.yaml b/.github/workflows/build-core-ut.yaml
@@ -0,0 +1,89 @@
+# Copyright 2021 iLogtail Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build Core Unit Test
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - 'example_config/**'
+      - 'docker/**'
+      - 'k8s_template/**'
+      - 'changes/**'
+      - 'licenses/**'
+      - 'CHANGELOG.md'
+  push:
+    branches:
+      - main
+      - 1.*
+
+jobs:
+  CI:
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        go-version: [1.19]
+        # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
+        runner: [ubuntu-latest]
+      fail-fast: true
+    steps:
+      # Clean up space to prevent action from running out of disk space.
+      - name: Free disk space
+        if: matrix.runner == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo -E apt-get -qq autoremove --purge
+          sudo -E apt-get -qq clean
+          
+      - name: Check disk space
+        run: |
+          df -hT $PWD
+
+      - name: Set up Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Build Unit Test
+        if: matrix.runner == 'ubuntu-latest'
+        env:
+          BUILD_LOGTAIL: OFF
+          BUILD_LOGTAIL_UT: ON
+          ENABLE_COMPATIBLE_MODE: ON
+          ENABLE_STATIC_LINK_CRT: ON
+          WITHOUTGDB: ON
+          # BUILD_TYPE: Debug # TODO: Uncomment when memory management is refined
+        run: make core
+
+      - name: Unit Test
+        if: matrix.runner == 'ubuntu-latest'
+        run: make unittest_core
+
+  result:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: [CI]
+    steps:
+      - name: Build Result
+        run: echo "Just to make the GitHub merge button green"
diff --git a/.github/workflows/build-core.yaml b/.github/workflows/build-core.yaml
@@ -37,15 +37,11 @@ jobs:
       matrix:
         go-version: [1.19]
         # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
-        runner: [ubuntu-latest, macos-latest, windows-2019]
+        runner: [ubuntu-latest]
       fail-fast: true
     steps:
-      # prepare ubuntu environment
-      - name: prepare ubuntu environment
-        if: matrix.runner == 'ubuntu-latest'
-        run: sudo apt-get clean && sudo apt-get update && sudo apt-get install -y libsystemd-dev
       # Clean up space to prevent action from running out of disk space.
-      - name: clean
+      - name: Free disk space
         if: matrix.runner == 'ubuntu-latest'
         run: |
           sudo rm -rf /usr/share/dotnet
@@ -54,39 +50,38 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           sudo -E apt-get -qq autoremove --purge
           sudo -E apt-get -qq clean
+
       - name: Check disk space
         run: |
           df -hT $PWD
-      # prepare windows environment
-      # https://github.com/actions/virtual-environments/issues/2549
-      - name: add mingw32 to path
-        if: matrix.runner == 'windows-2019'
-        shell: bash
-        run: |
-          echo "C:\msys64\mingw32\bin" >> $GITHUB_PATH
 
       - name: Set up Go ${{ matrix.go-version }}
         uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
-
-      - name: Check out code into the Go module directory
+          
+      - name: Check out code
         uses: actions/checkout@v2
         with:
           submodules: true
 
       - name: Build Binary
         if: matrix.runner == 'ubuntu-latest'
         env:
-          BUILD_LOGTAIL_UT: ON
+          BUILD_LOGTAIL_UT: OFF
           ENABLE_COMPATIBLE_MODE: ON
           ENABLE_STATIC_LINK_CRT: ON
           WITHOUTGDB: ON
         run: make core
 
-      - name: Unit Test
+      - name: Check compatibility
         if: matrix.runner == 'ubuntu-latest'
-        run: make unittest_core
+        env:
+          BUILD_LOGTAIL_UT: OFF
+          ENABLE_COMPATIBLE_MODE: ON
+          ENABLE_STATIC_LINK_CRT: ON
+          WITHOUTGDB: ON
+        run: make dist && scripts/check_glibc.sh
 
   result:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/build-pure-plugin.yaml b/.github/workflows/build-pure-plugin.yaml
@@ -44,8 +44,9 @@ jobs:
       - name: prepare ubuntu environment
         if: matrix.runner == 'ubuntu-latest'
         run: sudo apt-get clean && sudo apt-get update && sudo apt-get install -y libsystemd-dev
+
       # Clean up space to prevent action from running out of disk space.
-      - name: clean
+      - name: Free disk space
         if: matrix.runner == 'ubuntu-latest'
         run: |
           sudo rm -rf /usr/share/dotnet
@@ -54,9 +55,11 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           sudo -E apt-get -qq autoremove --purge
           sudo -E apt-get -qq clean
+
       - name: Check disk space
         run: |
           df -hT $PWD
+
       # prepare windows environment
       # https://github.com/actions/virtual-environments/issues/2549
       - name: add mingw32 to path
@@ -70,7 +73,7 @@ jobs:
         with:
           go-version: ${{ matrix.go-version }}
 
-      - name: Check out code into the Go module directory
+      - name: Check out code
         uses: actions/checkout@v2
         with:
           submodules: true

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -36,15 +36,11 @@ jobs:
       matrix:
         go-version: [1.19]
         # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
-        runner: [ubuntu-latest, macos-latest, windows-2019]
+        runner: [ubuntu-latest]
       fail-fast: true
     steps:
-      # prepare ubuntu environment
-      - name: prepare ubuntu environment
-        if: matrix.runner == 'ubuntu-latest'
-        run: sudo apt-get clean && sudo apt-get update && sudo apt-get install -y libsystemd-dev
       # Clean up space to prevent action from running out of disk space.
-      - name: clean
+      - name: Free disk space
         if: matrix.runner == 'ubuntu-latest'
         run: |
           sudo rm -rf /usr/share/dotnet
@@ -53,23 +49,17 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           sudo -E apt-get -qq autoremove --purge
           sudo -E apt-get -qq clean
+          
       - name: Check disk space
         run: |
           df -hT $PWD
-      # prepare windows environment
-      # https://github.com/actions/virtual-environments/issues/2549
-      - name: add mingw32 to path
-        if: matrix.runner == 'windows-2019'
-        shell: bash
-        run: |
-          echo "C:\msys64\mingw32\bin" >> $GITHUB_PATH
 
       - name: Set up Go ${{ matrix.go-version }}
         uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
-
-      - name: Check out code into the Go module directory
+          
+      - name: Check out code
         uses: actions/checkout@v2
         with:
           submodules: true
@@ -81,7 +71,7 @@ jobs:
           ENABLE_COMPATIBLE_MODE: ON
           ENABLE_STATIC_LINK_CRT: ON
           WITHOUTGDB: ON
-        run: make dist && scripts/check_glibc.sh
+        run: make dist
 
       - name: Build Docker
         if: matrix.runner == 'ubuntu-latest'

diff --git a/.github/workflows/e2e-core.yaml → .github/workflows/e2e-framework.yaml b/.github/workflows/e2e-core.yaml → .github/workflows/e2e-framework.yaml
@@ -39,7 +39,7 @@ jobs:
       fail-fast: true
     steps:
       # Clean up space to prevent action from running out of disk space.
-      - name: clean
+      - name: Free disk space
         if: matrix.runner == 'ubuntu-latest'
         run: |
           sudo rm -rf /usr/share/dotnet
@@ -48,9 +48,11 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           sudo -E apt-get -qq autoremove --purge
           sudo -E apt-get -qq clean
+
       - name: Check disk space
         run: |
           df -hT $PWD
+
       - name: Set up Go ${{ matrix.go-version }}
         uses: actions/setup-go@v2
         with:
@@ -66,12 +68,18 @@ jobs:
           sudo curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
           sudo chmod +x /usr/local/bin/docker-compose
 
-      - name: E2E Core Structure Test
+      - name: E2E Plugin Framework Test
         env:
           BUILD_LOGTAIL_UT: OFF
           WITHOUTGDB: ON
         run: make e2e-core
 
+      # Framework Test will provide local image that Unit Test required
+      - name: E2E Engine Unit Test
+        if: matrix.runner == 'ubuntu'
+        run: |
+          make unittest_e2e_engine
+
   result:
     runs-on: ubuntu-latest
     timeout-minutes: 60

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -48,15 +48,17 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           sudo -E apt-get -qq autoremove --purge
           sudo -E apt-get -qq clean
+
       - name: Check disk space
         run: |
           df -hT $PWD
+
       - name: Set up Go ${{ matrix.go-version }}
         uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
 
-      - name: Check out code into the Go module directory
+      - name: Check out code
         uses: actions/checkout@v2
         with:
           submodules: true
@@ -66,23 +68,18 @@ jobs:
           sudo curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
           sudo chmod +x /usr/local/bin/docker-compose
 
-      - name: Kernel version
-        run: uname -r
-      - name: Docker version
-        run: docker --version
+      - name: System environment
+        run: |
+          uname -r
+          docker --version
+          go version
 
-      - name: E2E Test
+      - name: E2E Behavior Test
         env:
           BUILD_LOGTAIL_UT: OFF
           WITHOUTGDB: ON
         run: make e2e
 
-      - name: UnitTest E2e Engine
-        if: matrix.runner == 'ubuntu'
-        run: |
-          go version
-          make unittest_e2e_engine
-
   result:
     runs-on: ubuntu-latest
     timeout-minutes: 60

diff --git a/.github/workflows/static-check.yaml b/.github/workflows/static-check.yaml
@@ -37,16 +37,16 @@ jobs:
     strategy:
       matrix:
         go-version: [ 1.19 ]
-        runner: [ ubuntu, macos ]
+        runner: [ ubuntu ]
       fail-fast: true
     steps:
-
       # prepare ubuntu environment
       - name: prepare ubuntu environment
         if: matrix.runner == 'ubuntu'
         run: sudo apt-get clean && sudo apt-get update && sudo apt-get install -y libsystemd-dev
+
       # Clean up space to prevent action from running out of disk space.
-      - name: clean
+      - name: Free disk space
         if: matrix.runner == 'ubuntu-latest'
         run: |
           sudo rm -rf /usr/share/dotnet
@@ -55,15 +55,17 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           sudo -E apt-get -qq autoremove --purge
           sudo -E apt-get -qq clean
+
       - name: Check disk space
         run: |
           df -hT $PWD
+
       - name: Set up Go ${{ matrix.go-version }}
         uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
 
-      - name: Check out code into the Go module directory
+      - name: Check out code
         uses: actions/checkout@v2
         with:
           submodules: true

diff --git a/config_server/service/Dockerfile b/config_server/service/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail-build-linux:gcc_9.3.1-1 as build
+FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail-build-linux:gcc_9.3.1-3 as build
 
 USER root
 WORKDIR /src