-
Notifications
You must be signed in to change notification settings - Fork 386
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add ilogtail file2blackhole benchmark
- Loading branch information
1 parent
4e25935
commit b17ae40
Showing
9 changed files
with
307 additions
and
33 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
37 changes: 37 additions & 0 deletions
37
test/e2e/test_cases/performance_file_to_blackhole_ilogtail/case.feature
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| @input | ||
| Feature: performance file to blackhole iLogtail | ||
| Performance file to blackhole iLogtail | ||
|
|
||
| @e2e-performance @docker-compose | ||
| Scenario: PerformanceFileToBlackholeiLogtail | ||
| Given {docker-compose} environment | ||
| Given subcribe data from {grpc} with config | ||
| """ | ||
| """ | ||
| Given {performance-file-to-blackhole-ilogtail-case} local config as below | ||
| """ | ||
| enable: true | ||
| inputs: | ||
| - Type: input_file | ||
| FilePaths: | ||
| - /home/test-log/json.log | ||
| processors: | ||
| - Type: processor_parse_json_native | ||
| SourceKey: content | ||
| - Type: processor_filter_regex_native | ||
| FilterKey: | ||
| - user-agent | ||
| FilterRegex: | ||
| - ^no-agent$ | ||
| flushers: | ||
| - Type: flusher_stdout | ||
| OnlyStdout: true | ||
| Tags: true | ||
| """ | ||
| Given iLogtail container mount {./a.log} to {/home/test-log/json.log} | ||
| When start docker-compose {performance_file_to_blackhole_ilogtail} | ||
| When start monitor {e2e-ilogtailC-1} | ||
| When generate {50} logs every {1}ms, total {1}min, to file {./a.log}, template | ||
| """ | ||
| {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"} | ||
| """ |
31 changes: 31 additions & 0 deletions
31
test/e2e/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| # Copyright 2021 iLogtail Authors | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| version: '3.8' | ||
|
|
||
| services: | ||
| cadvisor: | ||
| image: gcr.io/cadvisor/cadvisor:v0.49.1 | ||
| volumes: | ||
| - /:/rootfs:ro | ||
| - /var/run:/var/run:ro | ||
| - /sys:/sys:ro | ||
| - /var/lib/docker/:/var/lib/docker:ro | ||
| - /dev/disk/:/dev/disk:ro | ||
| ports: | ||
| - "8080:8080" | ||
| privileged: true | ||
| devices: | ||
| - /dev/kmsg | ||
| restart: unless-stopped |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| package cleanup | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "github.com/alibaba/ilogtail/test/engine/setup/monitor" | ||
| ) | ||
|
|
||
| func StopMonitor(ctx context.Context) (context.Context, error) { | ||
| return monitor.StopMonitor(ctx) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| package monitor | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "log" | ||
| "os" | ||
| "sync/atomic" | ||
| "time" | ||
|
|
||
| "github.com/google/cadvisor/client" | ||
| v1 "github.com/google/cadvisor/info/v1" | ||
|
|
||
| "github.com/alibaba/ilogtail/test/config" | ||
| ) | ||
|
|
||
| const ( | ||
| cadvisorURL = "http://localhost:8080/" | ||
| interval = 3 | ||
| ) | ||
|
|
||
| var stopCh chan bool | ||
| var isMonitoring atomic.Bool | ||
|
|
||
| func StartMonitor(ctx context.Context, containerName string) (context.Context, error) { | ||
| stopCh = make(chan bool) | ||
| isMonitoring.Store(true) | ||
| // connect to cadvisor | ||
| client, err := client.NewClient("http://localhost:8080/") | ||
| if err != nil { | ||
| return ctx, err | ||
| } | ||
| // 获取机器信息 | ||
| _, err = client.MachineInfo() | ||
| if err != nil { | ||
| // 处理错误 | ||
| return ctx, err | ||
| } | ||
| // fmt.Println("Machine Info:", machineInfo) | ||
| fmt.Println("Start monitoring container:", containerName) | ||
| go monitoring(client, containerName) | ||
| return ctx, nil | ||
| } | ||
|
|
||
| func StopMonitor(ctx context.Context) (context.Context, error) { | ||
| if isMonitoring.Load() { | ||
| stopCh <- true | ||
| } | ||
| return ctx, nil | ||
| } | ||
|
|
||
| func monitoring(client *client.Client, containerName string) { | ||
| // create csv file | ||
| reportDir := config.CaseHome + "/report/" | ||
| if _, err := os.Stat(reportDir); os.IsNotExist(err) { | ||
| // 文件夹不存在，创建文件夹 | ||
| err := os.MkdirAll(reportDir, 0755) // 使用适当的权限 | ||
| if err != nil { | ||
| log.Fatalf("Failed to create folder: %s", err) | ||
| } | ||
| } | ||
| file, err := os.OpenFile(reportDir+"performance.csv", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) | ||
| if err != nil { | ||
| fmt.Println("Error creating file:", err) | ||
| return | ||
| } | ||
| defer file.Close() | ||
| header := "Timestamp,CPU Total(%),CPU User(%),CPU System(%),CPU Load,Memory Usage(MB)\n" | ||
| _, err = file.WriteString(header) | ||
| if err != nil { | ||
| fmt.Println("Error writring file header:", err) | ||
| return | ||
| } | ||
| // new ticker | ||
| ticker := time.NewTicker(interval * time.Second) | ||
| defer ticker.Stop() | ||
| // read from cadvisor per interval seconds | ||
| request := &v1.ContainerInfoRequest{NumStats: 10} | ||
| var lastStat *v1.ContainerStats | ||
| for { | ||
| select { | ||
| case <-stopCh: | ||
| return | ||
| case <-ticker.C: | ||
| // 获取容器信息 | ||
| containerInfo, err := client.DockerContainer(containerName, request) | ||
| if err != nil { | ||
| fmt.Println("Error getting container info:", err) | ||
| return | ||
| } | ||
| for _, stat := range containerInfo.Stats { | ||
| if lastStat == nil { | ||
| lastStat = stat | ||
| continue | ||
| } | ||
| timestamp := stat.Timestamp | ||
| if !timestamp.After(lastStat.Timestamp) { | ||
| continue | ||
| } | ||
| // 计算cpu使用率 | ||
| cpuUsageRateTotal, cpuUsageRateUser, cpuUsageRateSys := calculateCpuUsageRate(lastStat, stat) | ||
| // 写入文件 | ||
| _, err := file.WriteString(fmt.Sprintf("%s,%f,%f,%f,%d,%d\n", | ||
| timestamp.Format("2006-01-02 15:04:05"), | ||
| cpuUsageRateTotal, | ||
| cpuUsageRateUser, | ||
| cpuUsageRateSys, | ||
| stat.Cpu.LoadAverage, | ||
| stat.Memory.Usage/1024/1024, | ||
| )) | ||
| if err != nil { | ||
| fmt.Println("Error writing file:", err) | ||
| return | ||
| } | ||
| lastStat = stat | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func calculateCpuUsageRate(lastStat, stat *v1.ContainerStats) (float64, float64, float64) { | ||
| if lastStat == nil { | ||
| return 0, 0, 0 | ||
| } | ||
| cpuUsageTotal := stat.Cpu.Usage.Total - lastStat.Cpu.Usage.Total | ||
| cpuUsageUser := stat.Cpu.Usage.User - lastStat.Cpu.Usage.User | ||
| cpuUsageSys := stat.Cpu.Usage.System - lastStat.Cpu.Usage.System | ||
| cpuUsageRateTotal := float64(cpuUsageTotal) / float64(stat.Timestamp.Sub(lastStat.Timestamp).Nanoseconds()) * 100 | ||
| cpuUsageRateUser := float64(cpuUsageUser) / float64(stat.Timestamp.Sub(lastStat.Timestamp).Nanoseconds()) * 100 | ||
| cpuUsageRateSys := float64(cpuUsageSys) / float64(stat.Timestamp.Sub(lastStat.Timestamp).Nanoseconds()) * 100 | ||
| return cpuUsageRateTotal, cpuUsageRateUser, cpuUsageRateSys | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| package trigger | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "time" | ||
|
|
||
| "github.com/alibaba/ilogtail/test/config" | ||
| ) | ||
|
|
||
| func GenerateLogToFile(ctx context.Context, cnt, interval, totalTime int, path string, templateStr string) (context.Context, error) { | ||
| // 打开或创建文件 | ||
| if !filepath.IsAbs(path) { | ||
| path = filepath.Join(config.CaseHome, path) | ||
| } | ||
| file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) | ||
| if err != nil { | ||
| return ctx, err | ||
| } | ||
| defer file.Close() | ||
|
|
||
| // 创建定时器 | ||
| ticker := time.NewTicker(time.Millisecond * time.Duration(interval)) | ||
| defer ticker.Stop() | ||
|
|
||
| // 总时间控制 | ||
| timeout := time.After(time.Minute * time.Duration(totalTime)) | ||
|
|
||
| for { | ||
| select { | ||
| case <-ctx.Done(): // 上下文取消 | ||
| return ctx, ctx.Err() | ||
| case <-timeout: // 总时间到 | ||
| return ctx, nil | ||
| case <-ticker.C: // 定时器触发 | ||
| for i := 0; i < cnt; i++ { | ||
| if _, err := fmt.Fprintln(file, templateStr); err != nil { | ||
| return ctx, err | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.