diff --git a/Makefile b/Makefile index 514595c1cd..52297fc99b 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,11 @@ degradation-test: @echo "Running degradation tests" @bash ./scripts/degradation-tester/degradation-check.sh +.PHONY: update-benchmarks +update-benchmarks: + @echo "Updating benchmarks" + @bash ./scripts/degradation-tester/update-benchmarks.sh + .PHONY: spec-test spec-test: @echo "Running spec tests" diff --git a/scripts/degradation-tester/README.md b/scripts/degradation-tester/README.md new file mode 100644 index 0000000000..6a6161559c --- /dev/null +++ b/scripts/degradation-tester/README.md @@ -0,0 +1,83 @@ +# Degradation Tester + +#### Continuous Benchmarking + +Degradation Tester is a tool for detecting performance degradation in Go projects. It allows comparing current benchmark results against previous ones to identify potential performance issues. + +#### Approving Performance Changes + +If you've made changes that affect performance, whether improvements or degradations, it's essential to regenerate the benchmarks and replace the existing files in the `benchmarks` directory. This ensures that future comparisons accurately reflect the impact of your changes and maintains the integrity of the performance testing process. + + To regenerate and update the benchmarks, use the `update-benchmarks`: + +```bash +make update-benchmarks +``` + +## Configuration + +To use the tool, you need to set up a YAML configuration file. Here is an example configuration: + +```yaml +DefaultOpDelta: 4.0 +DefaultAllocDelta: 0 +Packages: + - Path: "./message/validation" + Tests: + - Name: "VerifyRSASignature" + OpDelta: 3.0 + - Path: "./protocol/v2/types" + Tests: + - Name: "VerifyBLS" + OpDelta: 6.0 + - Name: "VerifyPKCS1v15" + OpDelta: 4.0 + - Name: "VerifyPKCS1v15FastHash" + OpDelta: 6.0 + - Name: "VerifyPSS" + OpDelta: 5.0 +``` + +- `DefaultOpDelta` and `DefaultAllocDelta` specify the default performance change thresholds (in percentages) that are allowed for all tests unless specific values are provided. +- `Path` refers to the directory containing the tests. +- `Tests` is a list of tests with names and permissible performance change thresholds. + +## Local Usage + +### Installation + +Install the benchmarks degradation checker tool: + +`cd scripts/degradation-tester/ && go install .` + +### Running Degradation Tests + +To run degradation tests, execute: + +```bash +make degradation-test +``` + +### Updating Benchmarks + +To update benchmarks, execute: + +```bash +make update-benchmarks +``` + +## Usage with GitHub Actions + +`.github/workflows/degradation-test.yml` containing a workflow for automatically running degradation tests as part of the CI/CD process + +### Scripts + +The `./scripts/degradation-tester` directory contains the following scripts: + +- `degradation-check.sh` - running degradation tests. Runs benchmarks, saves the new results, compares benchmarking results per all listed packages in `config.yaml` + +- `update-benchmarks.sh` - A script for updating benchmarks + +### Dependencies + +> Note: Ensure yq is installed in your system for proper YAML config parsing. diff --git a/scripts/degradation-tester/config.yaml b/scripts/degradation-tester/config.yaml index e915140c7d..9c1a250112 100644 --- a/scripts/degradation-tester/config.yaml +++ b/scripts/degradation-tester/config.yaml @@ -1,17 +1,17 @@ DefaultOpDelta: 4.0 DefaultAllocDelta: 0 -Tests: - - PackagePath: "./message/validation" - TestCases: +Packages: + - Path: "./message/validation" + Tests: - Name: "VerifyRSASignature" - OpDelta: 3.0 - - PackagePath: "./protocol/v2/types" - TestCases: + OpDelta: 10 + - Path: "./protocol/v2/types" + Tests: - Name: "VerifyBLS" OpDelta: 6.0 - Name: "VerifyPKCS1v15" - OpDelta: 3.0 + OpDelta: 4.0 - Name: "VerifyPKCS1v15FastHash" OpDelta: 6.0 - Name: "VerifyPSS" - OpDelta: 4.0 \ No newline at end of file + OpDelta: 5.0 \ No newline at end of file diff --git a/scripts/degradation-tester/degradation-check.sh b/scripts/degradation-tester/degradation-check.sh index 70853ea528..5b6fd97980 100644 --- a/scripts/degradation-tester/degradation-check.sh +++ b/scripts/degradation-tester/degradation-check.sh @@ -5,26 +5,26 @@ prefix="./scripts/degradation-tester" configFile="$prefix/config.yaml" benchmarksResults="$prefix/benchmarks" -packagePaths=($(yq e '.Tests[].PackagePath' $configFile)) +packagePaths=($(yq e '.Packages[].Path' $configFile)) for pkgPath in "${packagePaths[@]}"; do - packageName=$(basename "$pkgPath") - outputFile="${benchmarksResults}/${packageName}_results_new.txt" - oldBenchmarks="${benchmarksResults}/${packageName}_results_old.txt" - benchStatFile="${benchmarksResults}/${packageName}_benchstat.txt" + packageName=$(basename "$pkgPath") + outputFile="${benchmarksResults}/${packageName}_results_new.txt" + oldBenchmarks="${benchmarksResults}/${packageName}_results_old.txt" + benchStatFile="${benchmarksResults}/${packageName}_benchstat.txt" - go test -bench=. -count=10 -benchmem "$pkgPath" | tee "$outputFile" + go test -bench=. -count=10 -benchmem "$pkgPath" | tee "$outputFile" - benchstat "$oldBenchmarks" "$outputFile" | tee "${benchStatFile}" + benchstat "$oldBenchmarks" "$outputFile" | tee "${benchStatFile}" - degradation-tester "${configFile}" "${benchStatFile}" - if [ $? -ne 0 ]; then - echo "❌ Degradation tests have failed for ${packageName} package." - exit 1 - fi + degradation-tester "${configFile}" "${benchStatFile}" + if [ $? -ne 0 ]; then + echo "❌ Degradation tests have failed for ${packageName} package." + rm "${benchStatFile}" "${outputFile}" + exit 1 + fi - echo "✅ Degradation tests have passed for ${packageName} package." + echo "✅ Degradation tests have passed for ${packageName} package." - rm "${benchStatFile}" - rm "${outputFile}" + rm "${benchStatFile}" "${outputFile}" done diff --git a/scripts/degradation-tester/degradation-tester b/scripts/degradation-tester/degradation-tester deleted file mode 100755 index 150c2684e1..0000000000 Binary files a/scripts/degradation-tester/degradation-tester and /dev/null differ diff --git a/scripts/degradation-tester/main.go b/scripts/degradation-tester/main.go index be45f066b4..a6c5a07e3e 100644 --- a/scripts/degradation-tester/main.go +++ b/scripts/degradation-tester/main.go @@ -17,12 +17,12 @@ var allocThresholds map[string]float64 type Config struct { DefaultOpDelta float64 `yaml:"DefaultOpDelta"` DefaultAllocDelta float64 `yaml:"DefaultAllocDelta"` - Tests []BenchmarkingTestConfig `yaml:"Tests"` + Packages []BenchmarkingTestConfig `yaml:"Packages"` } type BenchmarkingTestConfig struct { - PackagePath string `yaml:"PackagePath"` - TestCases []TestCase `yaml:"TestCases"` + Path string `yaml:"Path"` + Tests []TestCase `yaml:"Tests"` } type TestCase struct { @@ -94,8 +94,8 @@ func loadConfig(filename string) { opThresholds = make(map[string]float64) allocThresholds = make(map[string]float64) - for _, pkg := range config.Tests { - for _, testCase := range pkg.TestCases { + for _, pkg := range config.Packages { + for _, testCase := range pkg.Tests { if testCase.OpDelta > 0 { opThresholds[testCase.Name] = testCase.OpDelta } diff --git a/scripts/degradation-tester/update-benchmarks.sh b/scripts/degradation-tester/update-benchmarks.sh new file mode 100644 index 0000000000..28941f01a6 --- /dev/null +++ b/scripts/degradation-tester/update-benchmarks.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +prefix="./scripts/degradation-tester" +configFile="$prefix/config.yaml" +benchmarksResults="$prefix/benchmarks" + +packagePaths=($(yq e '.Packages[].Path' $configFile)) + +for pkgPath in "${packagePaths[@]}"; do + packageName=$(basename "$pkgPath") + benchmarksPath="${benchmarksResults}/${packageName}_results_old.txt" + + echo "Updating benchmarks for ${packageName}" + + go test -bench=. -count=10 -benchmem "$pkgPath" | tee "$benchmarksPath" + + echo "✅ Benchmarks updated for ${packageName} package." +done