From 09dc8375b2b8aeb1243bdda478411294717f69ac Mon Sep 17 00:00:00 2001 From: Assassin718 <13038089398@163.com> Date: Thu, 11 Jul 2024 16:56:57 +0800 Subject: [PATCH] add file-to-blackhole performance test --- test/benchmark/main_test.go | 1 + .../a.log | 0 .../case.feature | 14 ++ .../docker-compose.yaml | 32 +++ .../filebeat.yml | 11 + .../a.log | 0 .../case.feature | 14 ++ .../docker-compose.yaml | 14 ++ .../main.conf | 10 + .../parsers.conf | 3 + .../case.feature | 3 - .../docker-compose.yaml | 2 +- .../a.log | 0 .../case.feature | 14 ++ .../docker-compose.yaml | 9 + .../vector.yaml | 11 + test/engine/cleanup/helper.go | 1 + .../setup/controller/docker_compose_boot.go | 4 +- test/engine/setup/docker_compose.go | 8 +- test/engine/setup/dockercompose/common.go | 11 +- .../setup/dockercompose/compose benchmark.go | 188 ++++++++++++++++++ test/engine/setup/monitor/monitor.go | 42 ++-- test/engine/setup/monitor/statistic.go | 4 +- test/engine/trigger/file.go | 40 ++-- 24 files changed, 378 insertions(+), 58 deletions(-) create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/a.log create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/a.log create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_vector/a.log create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml create mode 100644 test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml create mode 100644 test/engine/setup/dockercompose/compose benchmark.go diff --git a/test/benchmark/main_test.go b/test/benchmark/main_test.go index 1fd5eab4b2..fd1e68b501 100644 --- a/test/benchmark/main_test.go +++ b/test/benchmark/main_test.go @@ -78,6 +78,7 @@ func TestMain(m *testing.M) { func scenarioInitializer(ctx *godog.ScenarioContext) { // Given ctx.Given(`^\{(\S+)\} environment$`, setup.InitEnv) + ctx.Given(`^docker-compose type \{(\S+)\}$`, setup.SetDockerComposeType) ctx.Given(`^iLogtail depends on containers \{(.*)\}`, setup.SetDockerComposeDependOn) ctx.Given(`^iLogtail container mount \{(.*)\} to \{(.*)\}`, setup.MountVolume) ctx.Given(`^iLogtail expose port \{(.*)\} to \{(.*)\}`, setup.ExposePort) diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/a.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature new file mode 100644 index 0000000000..fbbff09956 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature @@ -0,0 +1,14 @@ +@input +Feature: performance file to blackhole filebeat + Performance file to blackhole filebeat + + @e2e-performance @docker-compose + Scenario: PerformanceFileToBlackholeFilebeat + Given {docker-compose} environment + Given docker-compose type {benchmark} + When start docker-compose {performance_file_to_blackhole_filebeat} + When start monitor {e2e-filebeat-1} + When generate logs to file, speed {10}MB/s, total {3}min, to file {./a.log}, template + """ + {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} + """ \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml new file mode 100644 index 0000000000..8e12d4a340 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml @@ -0,0 +1,32 @@ +version: '3.8' + +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.2 + environment: + - ELASTIC_PASSWORD=elastic + - discovery.type=single-node + - xpack.security.http.ssl.enabled=false + - xpack.license.self_generated.type=trial + ports: + - 9200:9200 + healthcheck: + test: ["CMD-SHELL", "curl -u elastic:elastic -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"] + interval: 10s + timeout: 5s + retries: 3 + restart: always + + filebeat: + image: docker.elastic.co/beats/filebeat:8.14.2 + user: root + volumes: + - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - ./a.log:/home/filebeat/a.log:ro + command: filebeat -e --strict.perms=false + environment: + - OUTPUT_ELASTICSEARCH_HOSTS=["elasticsearch:9200"] + depends_on: + elasticsearch: + condition: service_healthy + restart: always diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml new file mode 100644 index 0000000000..60c8e76687 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml @@ -0,0 +1,11 @@ +filebeat.inputs: +- type: filestream + id: input-file + paths: + - /home/filebeat/a.log +- decode_json_fields: + fields: ["message"] + target: "json" + +output.discard: + enabled: true diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/a.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature new file mode 100644 index 0000000000..6211ff8b79 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature @@ -0,0 +1,14 @@ +@input +Feature: performance file to blackhole fluentbit + Performance file to blackhole fluentbit + + @e2e-performance @docker-compose + Scenario: PerformanceFileToBlackholeFluentbit + Given {docker-compose} environment + Given docker-compose type {benchmark} + When start docker-compose {performance_file_to_blackhole_fluentbit} + When start monitor {e2e-fluent-bit-1} + When generate logs to file, speed {10}MB/s, total {3}min, to file {./a.log}, template + """ + {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} + """ \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml new file mode 100644 index 0000000000..aace706b7b --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml @@ -0,0 +1,14 @@ +version: '3.8' + +services: + + fluent-bit: + image: cr.fluentbit.io/fluent/fluent-bit + command: ["-c", "/tmp/main.conf"] + volumes: + - ./main.conf:/tmp/main.conf + - ./parsers.conf:/tmp/parsers.conf + - ./a.log:/home/fluentbit/a.log + restart: always + + diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf new file mode 100644 index 0000000000..baaf715b82 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf @@ -0,0 +1,10 @@ +[SERVICE] + Parsers+File /tmp/parsers.conf + +[INPUT] + Name tail + Path /home/fluentbit/a.log + +[OUTPUT] + Name stdout + Match non diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf new file mode 100644 index 0000000000..27cc2efc00 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf @@ -0,0 +1,3 @@ +[PARSER] + Name docker + Format json \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature index 763d9d2ed3..74a21b9839 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/case.feature @@ -5,9 +5,6 @@ Feature: performance file to blackhole iLogtail @e2e-performance @docker-compose Scenario: PerformanceFileToBlackholeiLogtail Given {docker-compose} environment - Given subcribe data from {grpc} with config - """ - """ Given {performance-file-to-blackhole-ilogtail-case} local config as below """ enable: true diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index 49bd5213c5..df9ceffb57 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -28,4 +28,4 @@ services: privileged: true devices: - /dev/kmsg - restart: unless-stopped + restart: always diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/a.log b/test/benchmark/test_cases/performance_file_to_blackhole_vector/a.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature new file mode 100644 index 0000000000..18c8d0a865 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/case.feature @@ -0,0 +1,14 @@ +@input +Feature: performance file to blackhole vector + Performance file to blackhole vector + + @e2e-performance @docker-compose + Scenario: PerformanceFileToBlackholeVector + Given {docker-compose} environment + Given docker-compose type {benchmark} + When start docker-compose {performance_file_to_blackhole_vector} + When start monitor {e2e-vector-1} + When generate logs to file, speed {10}MB/s, total {3}min, to file {./a.log}, template + """ + {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} + """ \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml new file mode 100644 index 0000000000..3763d9a3e6 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml @@ -0,0 +1,9 @@ +version: '3.8' + +services: + vector: + image: timberio/vector:0.39.0-debian + volumes: + - ./vector.yaml:/etc/vector/vector.yaml + - ./a.log:/home/vector-log/a.log + restart: always \ No newline at end of file diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml new file mode 100644 index 0000000000..cd13fc3489 --- /dev/null +++ b/test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml @@ -0,0 +1,11 @@ +sources: + input_file: + type: file + include: + - /home/vector-log/*.log + +sinks: + my_sink_id: + type: blackhole + inputs: + - input_file \ No newline at end of file diff --git a/test/engine/cleanup/helper.go b/test/engine/cleanup/helper.go index 6fd81f68f7..06827823f8 100644 --- a/test/engine/cleanup/helper.go +++ b/test/engine/cleanup/helper.go @@ -48,6 +48,7 @@ func All() { _, _ = GoTestCache(ctx) _, _ = StopMonitor(ctx) _, _ = DeleteContainers(ctx) + // FIXME: if this test case has no subscriber and the previous one has subscriber, it will panic if subscriber.TestSubscriber != nil { _ = subscriber.TestSubscriber.Stop() } diff --git a/test/engine/setup/controller/docker_compose_boot.go b/test/engine/setup/controller/docker_compose_boot.go index 1ffbc83636..394affe08d 100644 --- a/test/engine/setup/controller/docker_compose_boot.go +++ b/test/engine/setup/controller/docker_compose_boot.go @@ -27,9 +27,9 @@ import ( type BootController struct { } -func (c *BootController) Init() error { +func (c *BootController) Init(initType string) error { logger.Info(context.Background(), "boot controller is initializing....") - return dockercompose.Load() + return dockercompose.Load(initType) } func (c *BootController) Start(ctx context.Context) error { diff --git a/test/engine/setup/docker_compose.go b/test/engine/setup/docker_compose.go index 202a4c6230..cb4e51fcb0 100644 --- a/test/engine/setup/docker_compose.go +++ b/test/engine/setup/docker_compose.go @@ -16,10 +16,16 @@ import ( const dependencyHome = "test_cases" +var dockerComposeType = "e2e" + type DockerComposeEnv struct { BootController *controller.BootController } +func SetDockerComposeType(t string) { + dockerComposeType = t +} + func StartDockerComposeEnv(ctx context.Context, dependencyName string) (context.Context, error) { if dockerComposeEnv, ok := Env.(*DockerComposeEnv); ok { path := dependencyHome + "/" + dependencyName @@ -28,7 +34,7 @@ func StartDockerComposeEnv(ctx context.Context, dependencyName string) (context. return ctx, err } dockerComposeEnv.BootController = new(controller.BootController) - if err = dockerComposeEnv.BootController.Init(); err != nil { + if err = dockerComposeEnv.BootController.Init(dockerComposeType); err != nil { return ctx, err } diff --git a/test/engine/setup/dockercompose/common.go b/test/engine/setup/dockercompose/common.go index f9662ef8d7..9f35c3b90e 100644 --- a/test/engine/setup/dockercompose/common.go +++ b/test/engine/setup/dockercompose/common.go @@ -33,10 +33,17 @@ type Booter interface { } // Load configuration to the Booter. -func Load() error { +func Load(loadType string) error { mu.Lock() defer mu.Unlock() - instance = NewComposeBooter() + switch loadType { + case "e2e": + instance = NewComposeBooter() + case "benchmark": + instance = NewComposeBenchmarkBooter() + default: + return errors.New("invalid load type") + } return nil } diff --git a/test/engine/setup/dockercompose/compose benchmark.go b/test/engine/setup/dockercompose/compose benchmark.go new file mode 100644 index 0000000000..4b566c10a3 --- /dev/null +++ b/test/engine/setup/dockercompose/compose benchmark.go @@ -0,0 +1,188 @@ +// Copyright 2021 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dockercompose + +import ( + "context" + "os" + + "github.com/docker/docker/api/types" + dockertypes "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/client" + "github.com/testcontainers/testcontainers-go" + "gopkg.in/yaml.v3" + + "github.com/alibaba/ilogtail/pkg/logger" + "github.com/alibaba/ilogtail/test/config" +) + +const ( + cadvisorTemplate = `version: '3.8' +services: + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.49.1 + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + ports: + - "8080:8080" + privileged: true + devices: + - /dev/kmsg + restart: unless-stopped +` +) + +// ComposeBooter control docker-compose to start or stop containers. +type ComposeBenchmarkBooter struct { + cli *client.Client + cadvisorID string +} + +// NewComposeBooter create a new compose booter. +func NewComposeBenchmarkBooter() *ComposeBenchmarkBooter { + return &ComposeBenchmarkBooter{} +} + +func (c *ComposeBenchmarkBooter) Start(ctx context.Context) error { + if err := c.createComposeFile(); err != nil { + return err + } + compose := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, identifier).WithCommand([]string{"up", "-d", "--build"}) + strategyWrappers := withExposedService(compose) + execError := compose.Invoke() + if execError.Error != nil { + logger.Error(context.Background(), "START_DOCKER_COMPOSE_ERROR", + "stdout", execError.Error.Error()) + return execError.Error + } + cli, err := CreateDockerClient() + if err != nil { + return err + } + c.cli = cli + + list, err := cli.ContainerList(context.Background(), types.ContainerListOptions{ + Filters: filters.NewArgs(filters.Arg("name", "e2e-cadvisor")), + }) + if len(list) != 1 { + logger.Errorf(context.Background(), "CADVISOR_COMPOSE_ALARM", "cadvisor container size is not equal 1, got %d count", len(list)) + return err + } + c.cadvisorID = list[0].ID + + // the docker engine cannot access host on the linux platform, more details please see: https://github.com/docker/for-linux/issues/264 + cmd := []string{ + "sh", + "-c", + "env |grep HOST_OS|grep Linux && ip -4 route list match 0/0|awk '{print $3\" host.docker.internal\"}' >> /etc/hosts", + } + if err = c.exec(c.cadvisorID, cmd); err != nil { + return err + } + err = registerDockerNetMapping(strategyWrappers) + logger.Debugf(context.Background(), "registered net mapping: %v", networkMapping) + return err +} + +func (c *ComposeBenchmarkBooter) Stop() error { + execError := testcontainers.NewLocalDockerCompose([]string{config.CaseHome + finalFileName}, identifier).Down() + if execError.Error != nil { + logger.Error(context.Background(), "STOP_DOCKER_COMPOSE_ERROR", + "stdout", execError.Stdout.Error(), "stderr", execError.Stderr.Error()) + return execError.Error + } + _ = os.Remove(config.CaseHome + finalFileName) + return nil +} + +func (c *ComposeBenchmarkBooter) exec(id string, cmd []string) error { + cfg := dockertypes.ExecConfig{ + User: "root", + Cmd: cmd, + } + resp, err := c.cli.ContainerExecCreate(context.Background(), id, cfg) + if err != nil { + logger.Errorf(context.Background(), "DOCKER_EXEC_ALARM", "cannot create exec config: %v", err) + return err + } + err = c.cli.ContainerExecStart(context.Background(), resp.ID, dockertypes.ExecStartCheck{ + Detach: false, + Tty: false, + }) + if err != nil { + logger.Errorf(context.Background(), "DOCKER_EXEC_ALARM", "cannot start exec config: %v", err) + return err + } + return nil +} + +func (c *ComposeBenchmarkBooter) CopyCoreLogs() { +} + +func (c *ComposeBenchmarkBooter) createComposeFile() error { + // read the case docker compose file. + if _, err := os.Stat(config.CaseHome); os.IsNotExist(err) { + if err = os.MkdirAll(config.CaseHome, 0750); err != nil { + return err + } + } + _, err := os.Stat(config.CaseHome + config.DockerComposeFileName) + var bytes []byte + if err != nil { + if !os.IsNotExist(err) { + return err + } + } else { + if bytes, err = os.ReadFile(config.CaseHome + config.DockerComposeFileName); err != nil { + return err + } + } + cfg := c.getAdvisorConfig() + services := cfg["services"].(map[string]interface{}) + // merge docker compose file. + if len(bytes) > 0 { + caseCfg := make(map[string]interface{}) + if err = yaml.Unmarshal(bytes, &caseCfg); err != nil { + return err + } + newServices := caseCfg["services"].(map[string]interface{}) + for k := range newServices { + services[k] = newServices[k] + } + } + yml, err := yaml.Marshal(cfg) + if err != nil { + return err + } + return os.WriteFile(config.CaseHome+finalFileName, yml, 0600) +} + +// getLogtailpluginConfig find the docker compose configuration of the ilogtail. +func (c *ComposeBenchmarkBooter) getAdvisorConfig() map[string]interface{} { + cfg := make(map[string]interface{}) + f, _ := os.Create(config.CoverageFile) + _ = f.Close() + if err := yaml.Unmarshal([]byte(cadvisorTemplate), &cfg); err != nil { + panic(err) + } + bytes, _ := yaml.Marshal(cfg) + println(string(bytes)) + return cfg +} diff --git a/test/engine/setup/monitor/monitor.go b/test/engine/setup/monitor/monitor.go index 26a4677146..3d65521df6 100644 --- a/test/engine/setup/monitor/monitor.go +++ b/test/engine/setup/monitor/monitor.go @@ -6,6 +6,7 @@ import ( "log" "os" "path/filepath" + "strings" "sync/atomic" "time" @@ -24,20 +25,20 @@ var stopCh chan bool var isMonitoring atomic.Bool func StartMonitor(ctx context.Context, containerName string) (context.Context, error) { - stopCh = make(chan bool) - isMonitoring.Store(true) // connect to cadvisor client, err := client.NewClient("http://localhost:8080/") if err != nil { return ctx, err } // 获取机器信息 - _, err = client.MachineInfo() + request := &v1.ContainerInfoRequest{NumStats: 10} + _, err = client.DockerContainer(containerName, request) if err != nil { - // 处理错误 + fmt.Println("Error getting container info:", err) return ctx, err } - // fmt.Println("Machine Info:", machineInfo) + stopCh = make(chan bool) + isMonitoring.Store(true) fmt.Println("Start monitoring container:", containerName) go monitoring(client, containerName) return ctx, nil @@ -59,27 +60,11 @@ func monitoring(client *client.Client, containerName string) { } if _, err = os.Stat(reportDir); os.IsNotExist(err) { // 文件夹不存在，创建文件夹 - err := os.MkdirAll(reportDir, 0750) // 使用适当的权限 + err = os.MkdirAll(reportDir, 0750) if err != nil { log.Fatalf("Failed to create folder: %s", err) } } - reportDir = filepath.Join(reportDir, "performance.csv") - reportDir, err = filepath.Abs(reportDir) - if err != nil { - log.Fatalf("Failed to get absolute path: %s", err) - } - file, err := os.OpenFile(reportDir+"performance.csv", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - fmt.Println("Error creating file:", err) - return - } - header := "CPU Usage Max(%),CPU Usage Avg(%), Memory Usage Max(MB), Memory Usage Avg(MB)\n" - _, err = file.WriteString(header) - if err != nil { - fmt.Println("Error writring file header:", err) - return - } // new ticker ticker := time.NewTicker(interval * time.Second) defer ticker.Stop() @@ -89,9 +74,16 @@ func monitoring(client *client.Client, containerName string) { for { select { case <-stopCh: - fmt.Fprintln(file, monitorStatistic.cpu.maxVal, ",", monitorStatistic.cpu.avgVal, ",", monitorStatistic.mem.maxVal, ",", monitorStatistic.mem.avgVal) - if err = file.Close(); err != nil { - fmt.Println("Error closing file:", err) + isMonitoring.Store(false) + var builder strings.Builder + builder.WriteString("Metric,Value\n") + builder.WriteString(fmt.Sprintf("%s,%f\n", "CPU Usage Max(%)", monitorStatistic.cpu.maxVal)) + builder.WriteString(fmt.Sprintf("%s,%f\n", "CPU Usage Avg(%)", monitorStatistic.cpu.avgVal)) + builder.WriteString(fmt.Sprintf("%s,%f\n", "Memory Usage Max(MB)", monitorStatistic.mem.maxVal)) + builder.WriteString(fmt.Sprintf("%s,%f\n", "Memory Usage Avg(MB)", monitorStatistic.mem.avgVal)) + err = os.WriteFile(filepath.Join(reportDir, "monitor.csv"), []byte(builder.String()), 0600) + if err != nil { + log.Default().Printf("Failed to write monitor result: %s", err) } return case <-ticker.C: diff --git a/test/engine/setup/monitor/statistic.go b/test/engine/setup/monitor/statistic.go index be3494d5bf..442628dfe6 100644 --- a/test/engine/setup/monitor/statistic.go +++ b/test/engine/setup/monitor/statistic.go @@ -1,8 +1,6 @@ package monitor import ( - "fmt" - v1 "github.com/google/cadvisor/info/v1" ) @@ -48,7 +46,7 @@ func (m *Statistic) UpdateStatistic(stat *v1.ContainerStats) { m.cpu.Add(cpuUsageRateTotal) m.mem.Add(float64(stat.Memory.Usage) / 1024 / 1024) m.lastStat = stat - fmt.Println("CPU Usage Rate(%):", cpuUsageRateTotal, "CPU Usage Rate Max(%):", m.cpu.maxVal, "CPU Usage Rate Avg(%):", m.cpu.avgVal, "Memory Usage Max(MB):", m.mem.maxVal, "Memory Usage Avg(MB):", m.mem.avgVal) + // fmt.Println("CPU Usage Rate(%):", cpuUsageRateTotal, "CPU Usage Rate Max(%):", m.cpu.maxVal, "CPU Usage Rate Avg(%):", m.cpu.avgVal, "Memory Usage Max(MB):", m.mem.maxVal, "Memory Usage Avg(MB):", m.mem.avgVal) } func calculateCPUUsageRate(lastStat, stat *v1.ContainerStats) float64 { diff --git a/test/engine/trigger/file.go b/test/engine/trigger/file.go index 97a890a6ae..ceab6e5cf0 100644 --- a/test/engine/trigger/file.go +++ b/test/engine/trigger/file.go @@ -4,50 +4,48 @@ import ( "context" "fmt" "os" + "os/exec" "path/filepath" "time" - "github.com/alibaba/ilogtail/test/config" "golang.org/x/time/rate" + + "github.com/alibaba/ilogtail/test/config" ) func GenerateLogToFile(ctx context.Context, speed, totalTime int, path string, templateStr string) (context.Context, error) { - // 打开或创建文件 - if !filepath.IsAbs(path) { - path = filepath.Join(config.CaseHome, path) - } - file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - return ctx, err - } + // clear file + path = filepath.Join(config.CaseHome, path) + path = filepath.Clean(path) + _ = os.WriteFile(path, []byte{}, 0600) + + command := fmt.Sprintf("echo '%s' >> %s", templateStr, path) // interval = templateLen / speed interval := time.Microsecond * time.Duration(len(templateStr)/speed) limiter := rate.NewLimiter(rate.Every(interval), 1) - // 总时间控制 timeout := time.After(time.Minute * time.Duration(totalTime)) for { select { - case <-ctx.Done(): // 上下文取消 - if err := file.Close(); err != nil { - return ctx, err - } + // context is done + case <-ctx.Done(): + // clear file + _ = os.WriteFile(path, []byte{}, 0600) return ctx, ctx.Err() - case <-timeout: // 总时间到 - if err := file.Close(); err != nil { - return ctx, err - } + // all time is done + case <-timeout: + // clear file + _ = os.WriteFile(path, []byte{}, 0600) return ctx, nil default: if err := limiter.Wait(ctx); err != nil { return ctx, err } - if _, err := fmt.Fprintln(file, templateStr); err != nil { - return ctx, err - } + cmd := exec.Command("sh", "-c", command) + _ = cmd.Run() } } }