added readme, minor changes, update benchmarks script

Makefile

@@ -62,6 +62,11 @@ degradation-test:
 	@echo "Running degradation tests"
 	@bash ./scripts/degradation-tester/degradation-check.sh
 
+.PHONY: update-benchmarks
+update-benchmarks:
+	@echo "Updating benchmarks"
+	@bash ./scripts/degradation-tester/update-benchmarks.sh
+
 .PHONY: spec-test
 spec-test:
 	@echo "Running spec tests"

scripts/degradation-tester/README.md

@@ -0,0 +1,83 @@
+# Degradation Tester
+
+#### Continuous Benchmarking
+
+Degradation Tester is a tool for detecting performance degradation in Go projects. It allows comparing current benchmark results against previous ones to identify potential performance issues.
+
+#### Approving Performance Changes
+
+If you've made changes that affect performance, whether improvements or degradations, it's essential to regenerate the benchmarks and replace the existing files in the `benchmarks` directory. This ensures that future comparisons accurately reflect the impact of your changes and maintains the integrity of the performance testing process.
+
+ To regenerate and update the benchmarks, use the `update-benchmarks`:
+
+```bash
+make update-benchmarks
+```
+
+## Configuration
+
+To use the tool, you need to set up a YAML configuration file. Here is an example configuration:
+
+```yaml
+DefaultOpDelta: 4.0
+DefaultAllocDelta: 0
+Packages:
+  - Path: "./message/validation"
+    Tests:
+      - Name: "VerifyRSASignature"
+        OpDelta: 3.0
+  - Path: "./protocol/v2/types"
+    Tests:
+      - Name: "VerifyBLS"
+        OpDelta: 6.0
+      - Name: "VerifyPKCS1v15"
+        OpDelta: 4.0
+      - Name: "VerifyPKCS1v15FastHash"
+        OpDelta: 6.0
+      - Name: "VerifyPSS"
+        OpDelta: 5.0
+```
+
+- `DefaultOpDelta` and `DefaultAllocDelta` specify the default performance change thresholds (in percentages) that are allowed for all tests unless specific values are provided.
+- `Path` refers to the directory containing the tests.
+- `Tests` is a list of tests with names and permissible performance change thresholds.
+
+## Local Usage
+
+### Installation
+
+Install the benchmarks degradation checker tool:
+
+`cd scripts/degradation-tester/ && go install .`
+
+### Running Degradation Tests
+
+To run degradation tests, execute:
+
+```bash
+make degradation-test
+```
+
+### Updating Benchmarks
+
+To update benchmarks, execute:
+
+```bash
+make update-benchmarks
+```
+
+## Usage with GitHub Actions
+
+`.github/workflows/degradation-test.yml` containing a workflow for automatically running degradation tests as part of the CI/CD process
+
+### Scripts
+
+The `./scripts/degradation-tester` directory contains the following scripts:
+
+- `degradation-check.sh` - running degradation tests. Runs benchmarks, saves the new results, compares benchmarking results per all listed packages in `config.yaml`
+
+- `update-benchmarks.sh` - A script for updating benchmarks
+
+### Dependencies
+
+> Note: Ensure yq is installed in your system for proper YAML config parsing.

scripts/degradation-tester/config.yaml

@@ -1,17 +1,17 @@
 DefaultOpDelta: 4.0
 DefaultAllocDelta: 0
-Tests:
-  - PackagePath: "./message/validation"
-    TestCases:
+Packages:
+  - Path: "./message/validation"
+    Tests:
       - Name: "VerifyRSASignature"
-        OpDelta: 3.0
-  - PackagePath: "./protocol/v2/types"
-    TestCases:
+        OpDelta: 10
+  - Path: "./protocol/v2/types"
+    Tests:
       - Name: "VerifyBLS"
         OpDelta: 6.0
       - Name: "VerifyPKCS1v15"
-        OpDelta: 3.0
+        OpDelta: 4.0
       - Name: "VerifyPKCS1v15FastHash"
         OpDelta: 6.0
       - Name: "VerifyPSS"
-        OpDelta: 4.0
+        OpDelta: 5.0

scripts/degradation-tester/degradation-check.sh

@@ -5,26 +5,26 @@ prefix="./scripts/degradation-tester"
 configFile="$prefix/config.yaml"
 benchmarksResults="$prefix/benchmarks"
 
-packagePaths=($(yq e '.Tests[].PackagePath' $configFile))
+packagePaths=($(yq e '.Packages[].Path' $configFile))
 
 for pkgPath in "${packagePaths[@]}"; do
-    packageName=$(basename "$pkgPath")
-    outputFile="${benchmarksResults}/${packageName}_results_new.txt"
-    oldBenchmarks="${benchmarksResults}/${packageName}_results_old.txt"
-    benchStatFile="${benchmarksResults}/${packageName}_benchstat.txt"
+  packageName=$(basename "$pkgPath")
+  outputFile="${benchmarksResults}/${packageName}_results_new.txt"
+  oldBenchmarks="${benchmarksResults}/${packageName}_results_old.txt"
+  benchStatFile="${benchmarksResults}/${packageName}_benchstat.txt"
 
-    go test -bench=. -count=10 -benchmem "$pkgPath" | tee "$outputFile"
+  go test -bench=. -count=10 -benchmem "$pkgPath" | tee "$outputFile"
 
-    benchstat "$oldBenchmarks" "$outputFile" | tee "${benchStatFile}"
+  benchstat "$oldBenchmarks" "$outputFile" | tee "${benchStatFile}"
 
-    degradation-tester "${configFile}" "${benchStatFile}"
-    if [ $? -ne 0 ]; then
-      echo "❌ Degradation tests have failed for ${packageName} package."
-      exit 1
-    fi
+  degradation-tester "${configFile}" "${benchStatFile}"
+  if [ $? -ne 0 ]; then
+    echo "❌ Degradation tests have failed for ${packageName} package."
+    rm "${benchStatFile}" "${outputFile}"
+    exit 1
+  fi
 
-    echo "✅ Degradation tests have passed for ${packageName} package."
+  echo "✅ Degradation tests have passed for ${packageName} package."
 
-    rm "${benchStatFile}"
-    rm "${outputFile}"
+  rm "${benchStatFile}" "${outputFile}"
 done

scripts/degradation-tester/main.go

@@ -17,12 +17,12 @@ var allocThresholds map[string]float64
 type Config struct {
 	DefaultOpDelta    float64                  `yaml:"DefaultOpDelta"`
 	DefaultAllocDelta float64                  `yaml:"DefaultAllocDelta"`
-	Tests             []BenchmarkingTestConfig `yaml:"Tests"`
+	Packages          []BenchmarkingTestConfig `yaml:"Packages"`
 }
 
 type BenchmarkingTestConfig struct {
-	PackagePath string     `yaml:"PackagePath"`
-	TestCases   []TestCase `yaml:"TestCases"`
+	Path  string     `yaml:"Path"`
+	Tests []TestCase `yaml:"Tests"`
 }
 
 type TestCase struct {
@@ -94,8 +94,8 @@ func loadConfig(filename string) {
 	opThresholds = make(map[string]float64)
 	allocThresholds = make(map[string]float64)
 
-	for _, pkg := range config.Tests {
-		for _, testCase := range pkg.TestCases {
+	for _, pkg := range config.Packages {
+		for _, testCase := range pkg.Tests {
 			if testCase.OpDelta > 0 {
 				opThresholds[testCase.Name] = testCase.OpDelta
 			}

scripts/degradation-tester/update-benchmarks.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+prefix="./scripts/degradation-tester"
+configFile="$prefix/config.yaml"
+benchmarksResults="$prefix/benchmarks"
+
+packagePaths=($(yq e '.Packages[].Path' $configFile))
+
+for pkgPath in "${packagePaths[@]}"; do
+    packageName=$(basename "$pkgPath")
+    benchmarksPath="${benchmarksResults}/${packageName}_results_old.txt"
+
+    echo "Updating benchmarks for ${packageName}"
+
+    go test -bench=. -count=10 -benchmem "$pkgPath" | tee "$benchmarksPath"
+
+    echo "✅ Benchmarks updated for ${packageName} package."
+done