Add degradation tests into CI

.github/workflows/degradation-test.yml

@@ -0,0 +1,41 @@
+name: Benchmarks Degradation Test
+
+on:
+  push:
+    branches:
+      - "**"
+
+jobs:
+  performance-degradation-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Setup make
+        run: sudo apt-get update && sudo apt-get install make
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20.x"
+
+      - name: Tidy up dependencies
+        run: go mod tidy
+
+      - name: Install yq
+        run: sudo snap install yq
+
+      - name: Install benchstat
+        run: go install golang.org/x/perf/cmd/benchstat@latest
+
+      - name: Build degradation tester tool
+        run: |
+          cd ./scripts/degradation-tester
+          go install .
+
+      - name: Run degradation-test
+        run: |
+          make degradation-test
+        env:
+          GO111MODULE: on

.github/workflows/manual-benchmarks-update-.yml

@@ -0,0 +1,37 @@
+name: Manual Benchmarks Update
+
+on: workflow_dispatch
+
+jobs:
+  optional-benchmark-update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }} # Ensures the PR's head is checked out
+
+      - name: Setup make
+        run: sudo apt-get update && sudo apt-get install make
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.20.x"
+
+      - name: Tidy up dependencies
+        run: go mod tidy
+
+      - name: Update benchmarks
+        if: github.event_name == 'workflow_dispatch'
+        run: make update-benchmarks
+
+      - name: Update repo
+        uses: EndBug/[email protected]
+        with:
+          # limit the scope of the commit with the benchmarks dir path only
+          add: "scripts/degradation-tester/benchmarks"
+          default_author: github_actor
+          pathspec_error_handling: exitImmediately
+          message: "benchmarks updated"
+

Makefile

@@ -57,6 +57,16 @@ unit-test:
 	@echo "Running unit tests"
 	@go test -tags blst_enabled -timeout 20m -race -covermode=atomic -coverprofile=coverage.out -p 1 `go list ./... | grep -ve "spectest\|integration\|ssv/scripts/"`
 
+.PHONY: degradation-test
+degradation-test:
+	@echo "Running degradation tests"
+	@bash ./scripts/degradation-tester/degradation-check.sh
+
+.PHONY: update-benchmarks
+update-benchmarks:
+	@echo "Updating benchmarks"
+	@bash ./scripts/degradation-tester/update-benchmarks.sh
+
 .PHONY: spec-test
 spec-test:
 	@echo "Running spec tests"

message/validation/rsa_benchmark_test.go

@@ -0,0 +1,113 @@
+package validation
+
+import (
+	eth2apiv1 "github.com/attestantio/go-eth2-client/api/v1"
+	"github.com/bloxapp/ssv/networkconfig"
+	"github.com/bloxapp/ssv/operator/keys"
+	"github.com/bloxapp/ssv/operator/storage"
+	beaconprotocol "github.com/bloxapp/ssv/protocol/v2/blockchain/beacon"
+	ssvtypes "github.com/bloxapp/ssv/protocol/v2/types"
+	"github.com/bloxapp/ssv/storage/basedb"
+	"github.com/bloxapp/ssv/storage/kv"
+	"go.uber.org/zap/zaptest"
+	"testing"
+
+	specqbft "github.com/bloxapp/ssv-spec/qbft"
+	spectypes "github.com/bloxapp/ssv-spec/types"
+	spectestingutils "github.com/bloxapp/ssv-spec/types/testingutils"
+	"github.com/bloxapp/ssv/network/commons"
+	registrystorage "github.com/bloxapp/ssv/registry/storage"
+	"github.com/ethereum/go-ethereum/common"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	pspb "github.com/libp2p/go-libp2p-pubsub/pb"
+	"github.com/stretchr/testify/require"
+)
+
+func BenchmarkVerifyRSASignature(b *testing.B) {
+	logger := zaptest.NewLogger(b)
+	db, err := kv.NewInMemory(logger, basedb.Options{})
+	require.NoError(b, err)
+
+	ns, err := storage.NewNodeStorage(logger, db)
+	require.NoError(b, err)
+
+	const validatorIndex = 123
+	const operatorID = spectypes.OperatorID(1)
+
+	ks := spectestingutils.Testing4SharesSet()
+	share := &ssvtypes.SSVShare{
+		Share: *spectestingutils.TestingShare(ks),
+		Metadata: ssvtypes.Metadata{
+			BeaconMetadata: &beaconprotocol.ValidatorMetadata{
+				Status: eth2apiv1.ValidatorStateActiveOngoing,
+				Index:  validatorIndex,
+			},
+			Liquidated: false,
+		},
+	}
+	err = ns.Shares().Save(nil, share)
+	require.NoError(b, err)
+
+	netCfg := networkconfig.TestNetwork
+
+	roleAttester := spectypes.BNRoleAttester
+
+	slot := netCfg.Beacon.FirstSlotAtEpoch(123456789)
+
+	validSignedMessage := spectestingutils.TestingProposalMessageWithHeight(ks.Shares[1], 1, specqbft.Height(slot))
+
+	encoded, err := validSignedMessage.Encode()
+	require.NoError(b, err)
+
+	message := &spectypes.SSVMessage{
+		MsgType: spectypes.SSVConsensusMsgType,
+		MsgID:   spectypes.NewMsgID(netCfg.Domain, share.ValidatorPubKey, roleAttester),
+		Data:    encoded,
+	}
+
+	encodedMsg, err := commons.EncodeNetworkMsg(message)
+	require.NoError(b, err)
+
+	privKey, err := keys.GeneratePrivateKey()
+	require.NoError(b, err)
+
+	pubKey, err := privKey.Public().Base64()
+	require.NoError(b, err)
+
+	od := &registrystorage.OperatorData{
+		ID:           operatorID,
+		PublicKey:    pubKey,
+		OwnerAddress: common.Address{},
+	}
+
+	found, err := ns.SaveOperatorData(nil, od)
+	require.NoError(b, err)
+	require.False(b, found)
+
+	signature, err := privKey.Sign(encodedMsg)
+	require.NoError(b, err)
+
+	encodedMsg = commons.EncodeSignedSSVMessage(encodedMsg, operatorID, signature)
+
+	topicID := commons.ValidatorTopicID(message.GetID().GetPubKey())
+	pMsg := &pubsub.Message{
+		Message: &pspb.Message{
+			Topic: &topicID[0],
+			Data:  encodedMsg,
+		},
+	}
+
+	mv := NewMessageValidator(netCfg, WithNodeStorage(ns)).(*messageValidator)
+
+	messageData := pMsg.GetData()
+	decMessageData, operatorIDX, signature, err := commons.DecodeSignedSSVMessage(messageData)
+	require.NoError(b, err)
+
+	messageData = decMessageData
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := mv.verifySignature(messageData, operatorIDX, signature)
+		require.NoError(b, err)
+	}
+}

scripts/degradation-tester/README.md

@@ -0,0 +1,91 @@
+# Degradation Tester
+
+### Continuous Benchmarking
+
+Degradation Tester is a tool for detecting performance degradation. It allows to compare current benchmark results against previous ones to identify potential performance issues.
+It's important to use the same environment and CPU for testing. We use the Github Actions workflow for the benchmarks, which currently uses the `AMD EPYC 7763 64-Core Processor`.
+
+### Approving Performance Changes
+
+If you've made changes that affect performance, whether they are improvements or degradations, it's essential to regenerate the benchmarks and replace the existing files in the `benchmarks` directory. This ensures that future comparisons accurately reflect the impact of your changes and maintains the integrity of the performance testing process.
+
+To regenerate and update the benchmarks, use the `update-benchmarks` command in the Makefile. To update the benchmarks in the repo using Github Actions, you should use the manually triggered workflow for your Pull Request named **Manual Benchmarks Update**.
+
+To run the workflow:
+- Open the PR
+- Go to Actions
+- Choose `Manual Benchmarks Update` from the list of available actions
+- Click `Run workflow` and choose your branch from the dropdown menu
+
+The workflow is located in `.github/workflows/manual-benchmarks-update-.yml`.
+
+## Configuration
+
+To use the tool, you need to set up a YAML configuration file. Here is an example configuration:
+
+```yaml
+DefaultOpDelta: 4.0
+DefaultAllocDelta: 0
+Packages:
+  - Path: "./message/validation"
+    Tests:
+      - Name: "VerifyRSASignature"
+        OpDelta: 3.0
+  - Path: "./protocol/v2/types"
+    Tests:
+      - Name: "VerifyBLS"
+        OpDelta: 6.0
+      - Name: "VerifyPKCS1v15"
+        OpDelta: 4.0
+      - Name: "VerifyPKCS1v15FastHash"
+        OpDelta: 6.0
+      - Name: "VerifyPSS"
+        OpDelta: 5.0
+```
+
+- `DefaultOpDelta` and `DefaultAllocDelta` specify the default performance change thresholds (in percentages) that are allowed for all tests unless specific values are provided.
+- `Path` refers to the directory containing the tests.
+- `Tests` is a list of tests with names and permissible performance change thresholds.
+
+## Local Usage
+
+### Dependecies
+
+- `yq`
+- `benchstat`
+
+### Installation
+
+Install the benchmarks degradation checker tool:
+
+`cd scripts/degradation-tester/ && go install .`
+
+### Running Degradation Tests
+
+To run the degradation tests, execute:
+
+```bash
+make degradation-test
+```
+
+### Updating Benchmarks
+
+To replace the benchmarks with your local ones, execute:
+
+```bash
+make update-benchmarks
+```
+
+## Usage with GitHub Actions
+
+`.github/workflows/degradation-test.yml` containing a workflow for automatically running degradation tests as part of the CI/CD process
+
+`.github/workflows/manual-benchmarks-update-.yml`. containing a workflow for updating the benchmarks with manual triggering
+
+## Scripts
+
+The `./scripts/degradation-tester` directory contains the following scripts:
+
+- `degradation-check.sh` - running degradation tests. Runs benchmarks, saves the new results, compares benchmarking results per all listed packages in `config.yaml`
+
+- `update-benchmarks.sh` - A script for updating benchmarks

scripts/degradation-tester/benchmarks/message_validation_benchmarks_old.txt

@@ -0,0 +1,16 @@
+goos: linux
+goarch: amd64
+pkg: github.com/bloxapp/ssv/message/validation
+cpu: AMD EPYC 7763 64-Core Processor
+BenchmarkVerifyRSASignature-4   	    4729	    256788 ns/op	    1364 B/op	       7 allocs/op
+BenchmarkVerifyRSASignature-4   	    4712	    254533 ns/op	    1469 B/op	       8 allocs/op
+BenchmarkVerifyRSASignature-4   	    4722	    254782 ns/op	    1577 B/op	       8 allocs/op
+BenchmarkVerifyRSASignature-4   	    4735	    255405 ns/op	    1685 B/op	       9 allocs/op
+BenchmarkVerifyRSASignature-4   	    4573	    258438 ns/op	    1804 B/op	       9 allocs/op
+BenchmarkVerifyRSASignature-4   	    4694	    265556 ns/op	    1940 B/op	      10 allocs/op
+BenchmarkVerifyRSASignature-4   	    4491	    259423 ns/op	    2034 B/op	      10 allocs/op
+BenchmarkVerifyRSASignature-4   	    4677	    255195 ns/op	    2129 B/op	      10 allocs/op
+BenchmarkVerifyRSASignature-4   	    4364	    256322 ns/op	    2246 B/op	      11 allocs/op
+BenchmarkVerifyRSASignature-4   	    4694	    255884 ns/op	    2350 B/op	      11 allocs/op
+PASS
+ok  	github.com/bloxapp/ssv/message/validation	35.074s