Github runner add benchmark

.github/workflows/benchmark.yaml

@@ -0,0 +1,107 @@
+# Copyright 2021 iLogtail Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Benchmark
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - 'docs/**'
+      - 'example_config/**'
+      - 'docker/**'
+      - 'k8s_template/**'
+      - 'changes/**'
+      - 'licenses/**'
+      - 'CHANGELOG.md'
+    types:
+      - closed
+
+jobs:
+  CI:
+    if: github.event.pull_request.merged == true
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        go-version: [ 1.19.10 ]
+        runner: [ ubuntu-latest ]
+      fail-fast: true
+    steps:
+      # Clean up space to prevent action from running out of disk space.
+      - name: clean
+        if: matrix.runner == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo -E apt-get -qq autoremove --purge
+          sudo -E apt-get -qq clean
+
+      - name: Check disk space
+        run: |
+          df -hT $PWD
+
+      - name: Set up Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Update Docker-compose to v2
+        run: |
+          sudo curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+
+      - name: System environment
+        run: |
+          uname -r
+          docker --version
+          go version
+
+      - name: Run benchmark
+        env:
+          BUILD_LOGTAIL_UT: OFF
+          WITHOUTGDB: ON
+        run: make benchmark
+
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: benchmark
+          tool: "customSmallerIsBetter"
+          output-file-path: "test/benchmark/report/combined_benchmark.json"
+          external-data-json-path: ./cache/benchmark-data.json
+          summary-always: true
+
+  result:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: [ CI ]
+    steps:
+      - name: Build Result
+        run: echo "Just to make the GitHub merge button green"

Makefile

@@ -185,7 +185,7 @@ e2e-performance: clean docker gocdocker
 
 .PHONY: unittest_e2e_engine
 unittest_e2e_engine: clean gocdocker
-	cd test && go test  $$(go list ./... | grep -Ev "engine|e2e") -coverprofile=../e2e-engine-coverage.txt -covermode=atomic -tags docker_ready
+	cd test && go test  $$(go list ./... | grep -Ev "engine|e2e|benchmark") -coverprofile=../e2e-engine-coverage.txt -covermode=atomic -tags docker_ready
 
 .PHONY: unittest_plugin
 unittest_plugin: clean import_plugins
@@ -209,6 +209,13 @@ unittest_pluginmanager: clean import_plugins
 	go test $$(go list ./...|grep -Ev "telegraf|external|envconfig"| grep -E "plugin_main|pluginmanager") -coverprofile .coretestCoverage.txt
 	mv ./plugins/input/prometheus/input_prometheus.go.bak ./plugins/input/prometheus/input_prometheus.go
 
+# benchmark
+.PHONY: benchmark
+benchmark: clean gocdocker e2edocker
+	./scripts/e2e.sh benchmark performance
+	./scripts/benchmark_collect_result.sh
+
+
 .PHONY: all
 all: clean import_plugins
 	./scripts/gen_build_scripts.sh all $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)

licenses/LICENSE_OF_TESTENGINE_DEPENDENCIES.md

@@ -16,7 +16,6 @@ When distributed in a binary form, Logtailplugin Test Engine may contain portion
 - [github.com/opencontainers/go-digest](https://pkg.go.dev/github.com/opencontainers/go-digest?tab=licenses)
 - [github.com/opencontainers/image-spec](https://pkg.go.dev/github.com/opencontainers/image-spec?tab=licenses)
 - [github.com/opencontainers/runc](https://pkg.go.dev/github.com/opencontainers/runc?tab=licenses)
-- [google.golang.org/genproto](https://pkg.go.dev/google.golang.org/genproto?tab=licenses)
 - [google.golang.org/grpc](https://pkg.go.dev/google.golang.org/grpc?tab=licenses)
 - [gopkg.in/ini.v1](https://pkg.go.dev/gopkg.in/ini.v1?tab=licenses)
 - [gopkg.in/yaml.v2](https://pkg.go.dev/gopkg.in/yaml.v2?tab=licenses)
@@ -46,6 +45,8 @@ When distributed in a binary form, Logtailplugin Test Engine may contain portion
 - [sigs.k8s.io/json](https://pkg.go.dev/sigs.k8s.io/json?tab=licenses)
 - [sigs.k8s.io/structured-merge-diff](https://pkg.go.dev/sigs.k8s.io/structured-merge-diff?tab=licenses)
 - [sigs.k8s.io/yaml](https://pkg.go.dev/sigs.k8s.io/yaml?tab=licenses)
+- [github.com/google/cadvisor](https://pkg.go.dev/github.com/google/cadvisor?tab=licenses)
+- [google.golang.org/genproto/googleapis/rpc](https://pkg.go.dev/google.golang.org/genproto/googleapis/rpc?tab=licenses)
 
 ## BSD licenses
 

scripts/benchmark_collect_result.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Define the input files and the output file
+input_files=($(find test/benchmark/report -type f -name '*benchmark.json'))
+output_file="test/benchmark/report/combined_benchmark.json"
+
+# Start the output file with an opening square bracket
+rm -f "$output_file"
+touch "$output_file"
+echo '[' > "$output_file"
+
+# Loop through each input file
+for i in "${!input_files[@]}"; do
+  # Read the file, remove the first and last line (the square brackets), and append to the output file
+  cat "${input_files[$i]}" | sed '1d;$d' >> "$output_file"
+
+  # If this is not the last file, append a comma to separate the arrays
+  if [ $i -lt $((${#input_files[@]} - 1)) ]; then
+    echo ',' >> "$output_file"
+  fi
+done
+
+# Finish the output file with a closing square bracket
+echo ']' >> "$output_file"

test/benchmark/benchmark_docker_test.go

@@ -0,0 +1,36 @@
+// Copyright 2024 iLogtail Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package e2e
+
+import (
+	"testing"
+
+	"github.com/cucumber/godog"
+)
+
+func TestE2EOnDockerComposePerformance(t *testing.T) {
+	suite := godog.TestSuite{
+		Name:                "E2EOnDockerCompose",
+		ScenarioInitializer: scenarioInitializer,
+		Options: &godog.Options{
+			Format:   "pretty",
+			Paths:    []string{"test_cases"},
+			Tags:     "@e2e-performance && @docker-compose && ~@ebpf",
+			TestingT: t,
+		},
+	}
+	if suite.Run() != 0 {
+		t.Fail()
+	}
+}

test/benchmark/main_test.go

@@ -0,0 +1,124 @@
+// Copyright 2024 iLogtail Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package e2e
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/cucumber/godog"
+
+	"github.com/alibaba/ilogtail/pkg/logger"
+	"github.com/alibaba/ilogtail/test/config"
+	"github.com/alibaba/ilogtail/test/engine/cleanup"
+	"github.com/alibaba/ilogtail/test/engine/control"
+	"github.com/alibaba/ilogtail/test/engine/setup"
+	"github.com/alibaba/ilogtail/test/engine/setup/monitor"
+	"github.com/alibaba/ilogtail/test/engine/setup/subscriber"
+	"github.com/alibaba/ilogtail/test/engine/trigger"
+	"github.com/alibaba/ilogtail/test/engine/verify"
+)
+
+func TestMain(m *testing.M) {
+	loggerOptions := []logger.ConfigOption{
+		logger.OptionAsyncLogger,
+	}
+	loggerOptions = append(loggerOptions, logger.OptionInfoLevel)
+	logger.InitTestLogger(loggerOptions...)
+
+	config.TestConfig = config.Config{}
+	// Log
+	config.TestConfig.GeneratedLogDir = os.Getenv("GENERATED_LOG_DIR")
+	if len(config.TestConfig.GeneratedLogDir) == 0 {
+		config.TestConfig.GeneratedLogDir = "/tmp/ilogtail"
+	}
+	config.TestConfig.WorkDir = os.Getenv("WORK_DIR")
+
+	// SSH
+	config.TestConfig.SSHUsername = os.Getenv("SSH_USERNAME")
+	config.TestConfig.SSHIP = os.Getenv("SSH_IP")
+	config.TestConfig.SSHPassword = os.Getenv("SSH_PASSWORD")
+
+	// K8s
+	config.TestConfig.KubeConfigPath = os.Getenv("KUBE_CONFIG_PATH")
+
+	// SLS
+	config.TestConfig.Project = os.Getenv("PROJECT")
+	config.TestConfig.Logstore = os.Getenv("LOGSTORE")
+	config.TestConfig.AccessKeyID = os.Getenv("ACCESS_KEY_ID")
+	config.TestConfig.AccessKeySecret = os.Getenv("ACCESS_KEY_SECRET")
+	config.TestConfig.Endpoint = os.Getenv("ENDPOINT")
+	config.TestConfig.Aliuid = os.Getenv("ALIUID")
+	config.TestConfig.QueryEndpoint = os.Getenv("QUERY_ENDPOINT")
+	config.TestConfig.Region = os.Getenv("REGION")
+	timeout, err := strconv.ParseInt(os.Getenv("RETRY_TIMEOUT"), 10, 64)
+	if err != nil {
+		timeout = 60
+	}
+	config.TestConfig.RetryTimeout = time.Duration(timeout) * time.Second
+	code := m.Run()
+	logger.Flush()
+	os.Exit(code)
+}
+
+func scenarioInitializer(ctx *godog.ScenarioContext) {
+	// Given
+	ctx.Given(`^\{(\S+)\} environment$`, setup.InitEnv)
+	ctx.Given(`^docker-compose type \{(\S+)\}$`, setup.SetDockerComposeType)
+	ctx.Given(`^iLogtail depends on containers \{(.*)\}`, setup.SetDockerComposeDependOn)
+	ctx.Given(`^iLogtail container mount \{(.*)\} to \{(.*)\}`, setup.MountVolume)
+	ctx.Given(`^iLogtail expose port \{(.*)\} to \{(.*)\}`, setup.ExposePort)
+	ctx.Given(`^\{(.*)\} local config as below`, control.AddLocalConfig)
+	ctx.Given(`^\{(.*)\} http config as below`, control.AddHTTPConfig)
+	ctx.Given(`^remove http config \{(.*)\}`, control.RemoveHTTPConfig)
+	ctx.Given(`^subcribe data from \{(\S+)\} with config`, subscriber.InitSubscriber)
+
+	// When
+	ctx.When(`^generate \{(\d+)\} regex logs, with interval \{(\d+)\}ms$`, trigger.RegexSingle)
+	ctx.When(`^generate logs to file, speed \{(\d+)\}MB/s, total \{(\d+)\}min, to file \{(.*)\}, template`, trigger.GenerateLogToFile)
+	ctx.When(`^generate \{(\d+)\} http logs, with interval \{(\d+)\}ms, url: \{(.*)\}, method: \{(.*)\}, body:`, trigger.HTTP)
+	ctx.When(`^add k8s label \{(.*)\}`, control.AddLabel)
+	ctx.When(`^remove k8s label \{(.*)\}`, control.RemoveLabel)
+	ctx.When(`^start docker-compose \{(\S+)\}`, setup.StartDockerComposeEnv)
+	ctx.When(`^start monitor \{(\S+)\}`, monitor.StartMonitor)
+
+	// Then
+	ctx.Then(`^there is \{(\d+)\} logs$`, verify.LogCount)
+	ctx.Then(`^there is at least \{(\d+)\} logs$`, verify.LogCountAtLeast)
+	ctx.Then(`^there is at least \{(\d+)\} logs with filter key \{(.*)\} value \{(.*)\}$`, verify.LogCountAtLeastWithFilter)
+	ctx.Then(`^the log fields match regex single`, verify.RegexSingle)
+	ctx.Then(`^the log fields match kv`, verify.LogFieldKV)
+	ctx.Then(`^the log tags match kv`, verify.TagKV)
+	ctx.Then(`^the context of log is valid$`, verify.LogContext)
+	ctx.Then(`^the log fields match`, verify.LogField)
+	ctx.Then(`^the log labels match`, verify.LogLabel)
+	ctx.Then(`^the logtail log contains \{(\d+)\} times of \{(.*)\}$`, verify.LogtailPluginLog)
+	ctx.Then(`wait \{(\d+)\} seconds`, func(ctx context.Context, t int) context.Context {
+		time.Sleep(time.Duration(t) * time.Second)
+		return ctx
+	})
+
+	// Cleanup
+	ctx.Before(func(ctx context.Context, sc *godog.Scenario) (context.Context, error) {
+		cleanup.HandleSignal()
+		return ctx, nil
+	})
+	ctx.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) {
+		cleanup.All()
+		return ctx, nil
+	})
+}

test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature

@@ -0,0 +1,14 @@
+@input
+Feature: performance file to blackhole filebeat
+  Performance file to blackhole filebeat
+
+  @e2e-performance @docker-compose
+  Scenario: PerformanceFileToBlackholeFilebeat
+    Given {docker-compose} environment
+    Given docker-compose type {benchmark}
+    When start docker-compose {performance_file_to_blackhole_filebeat}
+    When start monitor {filebeat}
+    When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template
+    """
+    {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"}
+    """

test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml

@@ -0,0 +1,10 @@
+version: '3.8'
+
+services:
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:8.14.2
+    user: root
+    volumes:
+      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - ./a.log:/home/filebeat/a.log:ro
+    command: filebeat -e --strict.perms=false

test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml

@@ -0,0 +1,18 @@
+filebeat.inputs:
+- type: filestream
+  id: input-file
+  paths:
+    - /home/filebeat/a.log
+
+processors:
+  - decode_json_fields:
+      fields: ["message"]
+      target: "json"
+  - drop_event:
+      when:
+        not:
+          equals:
+            json.user-agent: "no-agent"
+
+output.console:
+  pretty: true

test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature

@@ -0,0 +1,14 @@
+@input
+Feature: performance file to blackhole fluentbit
+  Performance file to blackhole fluentbit
+
+  @e2e-performance @docker-compose
+  Scenario: PerformanceFileToBlackholeFluentbit
+    Given {docker-compose} environment
+    Given docker-compose type {benchmark}
+    When start docker-compose {performance_file_to_blackhole_fluentbit}
+    When start monitor {fluent-bit}
+    When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template
+    """
+    {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"}
+    """