diff --git a/docs/cn/developer-guide/test/benchmark.md b/docs/cn/developer-guide/test/benchmark.md new file mode 100644 index 0000000000..ede95de9d6 --- /dev/null +++ b/docs/cn/developer-guide/test/benchmark.md @@ -0,0 +1,60 @@ +# Benchmark测试 + +## 工作原理 +与e2e测试工作原理相同,详情可见: +- [e2e-test.md](./e2e-test.md) + +## 测试流程 + +### 环境准备 + +目前仅支持通过docker-compose搭建测试环境,在准备开始进行benchmark测试前,您首先需要准备以下内容: + +- 测试环境:Docker-Compose环境(需在本地安装docker-compose) + +### 配置文件 + +对于每一个新的benchmark,您需要在`./test/benchmark/test_cases/`目录下创建一个新的feature配置文件。每个配置文件中可以包含多个测试场景,每个测试场景由一个或多个步骤组成。 + +配置文件的基本框架如下: + +```plain +@input +Feature: performance file to blackhole vector + Performance file to blackhole vector + + @e2e-performance @docker-compose + Scenario: PerformanceFileToBlackholeVector + Given {docker-compose} environment + Given docker-compose type {benchmark} + When start docker-compose {performance_file_to_blackhole_vector} + When start monitor {vector} + When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template + """ + {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} + """ +``` + +- `@e2e-performance @docker-compose`: 表示测试场景为e2e-performance,测试场景由本地docker-compose运行 +- `Given {docker-compose} environment`: 配置启动测试环境,以docker-compose环境启动测试 +- `Given docker-compose type {benchmark}`: 配置docker-compose启动模式,以benchmark模式启动docker-compose,`{}`中参数有两种选项,`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器,用作e2e测试;以`benchmark`模式启动会默认启动cadvisor容器,用于监控容器运行过程中的资源占用;若在配置文件中不配置该参数,则默认以上一个scenario的启动模式启动。 +- `When start docker-compose {scenario directory}`: `{}`中参数为当前scenario的文件夹名,该行动作会读取`scenario director`文件夹下的docker-compose.yaml文件,通过docker-compose命令启动所有容器 +- `When start monitor {vector}`: `{}`中参数为待监控的容器,该参数需要与docker-compose中的service name相同 +- `When generate logs to file`: 生成测试数据,其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) + +### 运行测试 + +在所有测试内容准备完毕后,您可以直接在test目录下以go test的方式运行E2E测试。根据您的需求,可以选择运行所有测试或者指定测试。 + +```shell +# 运行所有benchmark测试 +go test -v -timeout 30m -run ^TestE2EOnDockerComposePerformance$ github.com/alibaba/ilogtail/test/benchmark + +# 以正则表达式匹配Scenario运行测试,以下为运行Scenario以Vector为结尾的test_case +go test -v -timeout 30m -run ^TestE2EOnDockerComposePerformance$/^.*Vector$ github.com/alibaba/ilogtail/test/benchmark +``` + +### 测试结果 + +- 所有测试结果将以json格式记录在`test/benchmark/report/.json`中,目前记录了测试过程中CPU最大使用率、CPU平均使用率、内存最大使用率、内存平均使用率参数。 +- 运行`scripts/benchmark_collect_result.sh`会将`test/benchmark/report/`下所有结果收集并生成汇总结果到`test/benchmark/report/combines_result.json`,主要用于github benmark action收集本次测试数据并与历史数据汇总 \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/a.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml index 9e90378fa9..ce6b83c442 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml @@ -6,5 +6,5 @@ services: user: root volumes: - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - - ./a.log:/home/filebeat/a.log:ro + - .:/home/filebeat command: filebeat -e --strict.perms=false diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml index e2de4b2806..e93a2688e0 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml @@ -2,7 +2,8 @@ filebeat.inputs: - type: filestream id: input-file paths: - - /home/filebeat/a.log + - /home/filebeat/*.log + prospector.scanner.check_interval: 1s processors: - decode_json_fields: diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/a.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml index aace706b7b..aeb15b1f07 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml @@ -8,7 +8,7 @@ services: volumes: - ./main.conf:/tmp/main.conf - ./parsers.conf:/tmp/parsers.conf - - ./a.log:/home/fluentbit/a.log + - .:/home/fluentbit restart: always diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf index 7436cae6b6..d009876b60 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf @@ -3,8 +3,9 @@ [INPUT] name tail - path /home/fluentbit/a.log + path /home/fluentbit/*.log parser json + refresh_interval 1 [FILTER] name grep diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/a.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index f541582d5a..8c416bd168 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -12,7 +12,7 @@ Feature: performance file to blackhole iLogtail inputs: - Type: input_file FilePaths: - - /home/test-log/json.log + - /home/test-log/*.log processors: - Type: processor_parse_json_native SourceKey: content @@ -22,13 +22,13 @@ Feature: performance file to blackhole iLogtail FilterRegex: - ^no-agent$ flushers: - - Type: flusher_sls - Region: cn-hangzhou - Endpoint: cn-hangzhou.log.aliyuncs.com - Project: test_project - Logstore: test_logstore + - Type: flusher_sls + Region: cn-hangzhou + Endpoint: cn-hangzhou.log.aliyuncs.com + Project: test_project + Logstore: test_logstore """ - Given iLogtail container mount {./a.log} to {/home/test-log/json.log} + Given iLogtail container mount {.} to {/home/test-log} When start docker-compose {performance_file_to_blackhole_ilogtail} When start monitor {ilogtailC} When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_vector/a.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml index c133cb69ec..9c35dcd6d3 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml @@ -5,4 +5,4 @@ services: image: timberio/vector:0.39.0-debian volumes: - ./vector.yaml:/etc/vector/vector.yaml - - ./a.log:/home/vector-log/a.log \ No newline at end of file + - .:/home/vector-log \ No newline at end of file diff --git a/test/engine/setup/dockercompose/compose benchmark.go b/test/engine/setup/dockercompose/compose benchmark.go index 4b566c10a3..f82607db4c 100644 --- a/test/engine/setup/dockercompose/compose benchmark.go +++ b/test/engine/setup/dockercompose/compose benchmark.go @@ -30,7 +30,8 @@ import ( ) const ( - cadvisorTemplate = `version: '3.8' + benchmarkIdentifier = "benchmark" + cadvisorTemplate = `version: '3.8' services: cadvisor: image: gcr.io/cadvisor/cadvisor:v0.49.1 @@ -64,7 +65,7 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { if err := c.createComposeFile(); err != nil { return err } - compose := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, identifier).WithCommand([]string{"up", "-d", "--build"}) + compose := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, benchmarkIdentifier).WithCommand([]string{"up", "-d", "--build"}) strategyWrappers := withExposedService(compose) execError := compose.Invoke() if execError.Error != nil { @@ -79,7 +80,7 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { c.cli = cli list, err := cli.ContainerList(context.Background(), types.ContainerListOptions{ - Filters: filters.NewArgs(filters.Arg("name", "e2e-cadvisor")), + Filters: filters.NewArgs(filters.Arg("name", "benchmark-cadvisor")), }) if len(list) != 1 { logger.Errorf(context.Background(), "CADVISOR_COMPOSE_ALARM", "cadvisor container size is not equal 1, got %d count", len(list)) @@ -102,7 +103,7 @@ func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { } func (c *ComposeBenchmarkBooter) Stop() error { - execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, identifier).Down() + execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, benchmarkIdentifier).Down() if execError.Error != nil { logger.Error(context.Background(), "STOP_DOCKER_COMPOSE_ERROR", "stdout", execError.Stdout.Error(), "stderr", execError.Stderr.Error())