added -format csv to benchstat. added basic test

scripts/degradation-tester/degradation-check.sh

@@ -15,7 +15,7 @@ for pkgPath in "${packagePaths[@]}"; do
 
   go test -bench=. -count=10 -benchmem "$pkgPath" | tee "$outputFile"
 
-  benchstat "$oldBenchmarks" "$outputFile" | tee "${benchStatFile}"
+  benchstat -format csv "$oldBenchmarks" "$outputFile" | tee "${benchStatFile}"
 
   degradation-tester "${configFile}" "${benchStatFile}"
   if [ $? -ne 0 ]; then
@@ -25,6 +25,5 @@ for pkgPath in "${packagePaths[@]}"; do
   fi
 
   echo "✅ Degradation tests have passed for ${packageName} package."
-
   rm "${benchStatFile}" "${outputFile}"
 done

scripts/degradation-tester/go.mod

@@ -3,3 +3,14 @@ module github.com/bloxapp/ssv/scripts/degradation-tester
 go 1.20
 
 require gopkg.in/yaml.v3 v3.0.1
+
+require (
+	github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect
+	golang.org/x/perf v0.0.0-20240305160248-5eefbfdba9dd // indirect
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/testify v1.9.0
+)

scripts/degradation-tester/go.sum

@@ -1,3 +1,13 @@
+github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g=
+github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/perf v0.0.0-20240305160248-5eefbfdba9dd h1:rglj7j7GZzz4GcR21t9lmupN+8ALC8L//rdzE/50vLE=
+golang.org/x/perf v0.0.0-20240305160248-5eefbfdba9dd/go.mod h1:9aZNLn0je8D5R0rbpRog/X1gTnJt4uajOXR4k1WpzXk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

scripts/degradation-tester/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"encoding/csv"
 	"fmt"
 	"math"
 	"os"
@@ -32,6 +33,21 @@ type TestCase struct {
 	AllocDelta float64 `yaml:"AllocDelta"`
 }
 
+type Benchmark struct {
+	TestName string
+	OldValue float64
+	OldCI    float64
+	NewValue float64
+	NewCI    float64
+	VsBase   string
+	P        string
+}
+
+type BenchStatOutput struct {
+	OsData     string
+	benchmarks []Benchmark
+}
+
 func main() {
 	if len(os.Args) < 3 {
 		fmt.Println("Usage: degradation-tester <config_filename> <results_filename>")
@@ -50,6 +66,14 @@ func main() {
 	}
 	defer file.Close()
 
+	totalErrors := checkFile(file, config)
+
+	if totalErrors > 0 {
+		os.Exit(1)
+	}
+}
+
+func checkFile(file *os.File, config *Config) int {
 	var currentSection string
 
 	scanner := bufio.NewScanner(file)
@@ -60,10 +84,13 @@ func main() {
 		switch {
 		case strings.Contains(line, "sec/op"):
 			currentSection = "sec/op"
+			continue
 		case strings.Contains(line, "B/op"):
 			currentSection = "B/op"
+			continue
 		case strings.Contains(line, "allocs/op"):
 			currentSection = "allocs/op"
+			continue
 		}
 
 		totalErrors += checkLine(config, line, currentSection)
@@ -73,9 +100,84 @@ func main() {
 		fmt.Printf("Error reading results file: %v\n", err)
 	}
 
-	if totalErrors > 0 {
+	return totalErrors
+}
+
+func checkLine(
+	config *Config,
+	line string,
+	section string,
+) int {
+	if line == "" {
+		return 0
+	}
+	csvRowReader := csv.NewReader(strings.NewReader(line))
+	csvRowReader.Comment = '#'
+	csvRowReader.Comma = ','
+
+	row, err := csvRowReader.Read()
+	if err != nil {
+		fmt.Printf("failed parsing CSV line %s with erorr: %v\n", line, err)
 		os.Exit(1)
 	}
+
+	if len(row) != 7 {
+		// ignore all except the becnhmark result lines with exactly 7 columns
+		return 0
+	}
+
+	// The "geomean" represents a statistical summary (geometric mean) of multiple test results,
+	// not an individual test result, hence we should just skip it
+	if row[0] == "geomean" {
+		return 0
+	}
+
+	normalizedTestName := normalizeTestName(row[0])
+
+	oldValue, err := strconv.ParseFloat(row[1], 64)
+	if err != nil {
+		fmt.Printf("⚠️ Error parsing float: %v\n", err)
+		return 1
+	}
+	oldChangeStr := row[2]
+	newValue, err := strconv.ParseFloat(row[3], 64)
+	if err != nil {
+		fmt.Printf("⚠️ Error parsing float: %v\n", err)
+		return 1
+	}
+	newChangeStr := row[4]
+
+	oldChange, err := strconv.ParseFloat(strings.TrimSuffix(oldChangeStr, "%"), 64)
+	if err != nil {
+		fmt.Printf("⚠️ Error parsing float: %v\n", err)
+		return 1
+	}
+	newChange, err := strconv.ParseFloat(strings.TrimSuffix(newChangeStr, "%"), 64)
+	if err != nil {
+		fmt.Printf("⚠️ Error parsing float: %v\n", err)
+		return 1
+	}
+
+	threshold := getThresholdForTestCase(config, normalizedTestName, section)
+
+	if math.Abs(oldChange-newChange) > threshold {
+		fmt.Printf("❌ Change in section %s for test %s exceeds threshold: %s\n", section, normalizedTestName, newChangeStr)
+		return 1
+	}
+
+	b := &Benchmark{
+		TestName: row[0],
+		OldValue: oldValue,
+		OldCI:    oldChange,
+		NewValue: newValue,
+		NewCI:    newChange,
+		VsBase:   row[5],
+		P:        row[6],
+	}
+
+	fmt.Printf("Debug: bench [TestName %s], [OldValue %f], [OldCI %f], [NewValue %f], [NewCI %f], [VsBase %s], [P %s]\n", b.TestName, b.OldValue, b.OldCI, b.NewValue, b.NewCI, b.VsBase, b.P)
+
+	return 0
 }
 
 func loadConfig(filename string) *Config {
@@ -107,7 +209,7 @@ func loadConfig(filename string) *Config {
 	return config
 }
 
-func checkLine(
+func checkLine2(
 	config *Config,
 	line string,
 	section string,

scripts/degradation-tester/main_test.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseBenchStatFile(t *testing.T) {
+	benchstatCsvFileName := "benchstat.csv"
+	f, err := os.Create(benchstatCsvFileName)
+	require.NoError(t, err)
+
+	_, err = f.WriteString(stubBenchStatCSVFileData)
+	require.NoError(t, err)
+
+	cfg := &Config{
+		DefaultAllocDelta: 10,
+		DefaultOpDelta:    10,
+	}
+	f, err = os.Open(benchstatCsvFileName)
+	require.NoError(t, err)
+	totalErrors := checkFile(f, cfg)
+	require.Equal(t, 0, totalErrors)
+
+	// clear test artefacts
+	defer func() {
+		_ = os.Remove(benchstatCsvFileName)
+	}()
+}
+
+const stubBenchStatCSVFileData = `
+goos: darwin
+goarch: amd64
+pkg: github.com/bloxapp/ssv/message/validation
+cpu: xxxx
+,./scripts/degradation-tester/benchmarks/validation_results_old.txt,,./scripts/degradation-tester/benchmarks/validation_results_new.txt
+,sec/op,CI,sec/op,CI,vs base,P
+VerifyRSASignature-8,0.000266723,2%,0.0002658285,1%,~,p=0.123 n=10
+geomean,0.0002667229999999998,,0.0002658284999999999,,-0.34%
+
+,./scripts/degradation-tester/benchmarks/validation_results_old.txt,,./scripts/degradation-tester/benchmarks/validation_results_new.txt
+,B/op,CI,B/op,CI,vs base,P
+VerifyRSASignature-8,1904.5,22%,1892.5,22%,~,p=0.927 n=10
+geomean,1904.4999999999998,,1892.4999999999995,,-0.63%
+
+,./scripts/degradation-tester/benchmarks/validation_results_old.txt,,./scripts/degradation-tester/benchmarks/validation_results_new.txt
+,allocs/op,CI,allocs/op,CI,vs base,P
+VerifyRSASignature-8,9.5,16%,9.5,16%,~,p=0.878 n=10
+geomean,9.500000000000002,,9.500000000000002,,+0.00%
+`