feat: Reversion metrics

chain/test_chain.go

@@ -255,7 +255,7 @@ func (t *TestChain) Clone(onCreateFunc func(chain *TestChain) error) (*TestChain
 		// Now add each transaction/message to it.
 		messages := t.blocks[i].Messages
 		for j := 0; j < len(messages); j++ {
-			err = targetChain.PendingBlockAddTx(messages[j])
+			err, _ = targetChain.PendingBlockAddTx(messages[j])
 			if err != nil {
 				return nil, err
 			}
@@ -727,10 +727,10 @@ func (t *TestChain) PendingBlockCreateWithParameters(blockNumber uint64, blockTi
 // PendingBlockAddTx takes a message (internal txs) and adds it to the current pending block, updating the header
 // with relevant execution information. If a pending block was not created, an error is returned.
 // Returns an error if one occurred.
-func (t *TestChain) PendingBlockAddTx(message *core.Message, additionalTracers ...*TestChainTracer) error {
+func (t *TestChain) PendingBlockAddTx(message *core.Message, additionalTracers ...*TestChainTracer) (error, *core.ExecutionResult) {
 	// If we don't have a pending block, return an error
 	if t.pendingBlock == nil {
-		return errors.New("could not add tx to the chain's pending block because no pending block was created")
+		return errors.New("could not add tx to the chain's pending block because no pending block was created"), nil
 	}
 
 	// Create a gas pool indicating how much gas can be spent executing the transaction.
@@ -776,7 +776,7 @@ func (t *TestChain) PendingBlockAddTx(message *core.Message, additionalTracers .
 	var usedGas uint64
 	receipt, executionResult, err := vendored.EVMApplyTransaction(message, t.chainConfig, t.testChainConfig, &t.pendingBlock.Header.Coinbase, gasPool, t.state, t.pendingBlock.Header.Number, t.pendingBlock.Hash, tx, &usedGas, evm)
 	if err != nil {
-		return fmt.Errorf("test chain state write error when adding tx to pending block: %v", err)
+		return fmt.Errorf("test chain state write error when adding tx to pending block: %v", err), nil
 	}
 
 	// Create our message result
@@ -801,7 +801,7 @@ func (t *TestChain) PendingBlockAddTx(message *core.Message, additionalTracers .
 	// Emit our contract change events for this message
 	err = t.emitContractChangeEvents(false, messageResult)
 	if err != nil {
-		return err
+		return err, nil
 	}
 
 	// Emit our event for having added a new transaction to the pending block.
@@ -811,10 +811,10 @@ func (t *TestChain) PendingBlockAddTx(message *core.Message, additionalTracers .
 		TransactionIndex: len(t.pendingBlock.Messages),
 	})
 	if err != nil {
-		return err
+		return err, nil
 	}
 
-	return nil
+	return nil, executionResult
 }
 
 // PendingBlockCommit commits a pending block to the chain, so it is set as the new head. The pending block is set

chain/test_chain_test.go

@@ -260,7 +260,7 @@ func TestChainDynamicDeployments(t *testing.T) {
 						assert.NoError(t, err)
 
 						// Add our transaction to the block
-						err = chain.PendingBlockAddTx(&msg)
+						err, _ = chain.PendingBlockAddTx(&msg)
 						assert.NoError(t, err)
 
 						// Commit the pending block to the chain, so it becomes the new head.
@@ -385,7 +385,7 @@ func TestChainDeploymentWithArgs(t *testing.T) {
 					assert.NoError(t, err)
 
 					// Add our transaction to the block
-					err = chain.PendingBlockAddTx(&msg)
+					err, _ = chain.PendingBlockAddTx(&msg)
 					assert.NoError(t, err)
 
 					// Commit the pending block to the chain, so it becomes the new head.
@@ -494,7 +494,7 @@ func TestChainCloning(t *testing.T) {
 							assert.NoError(t, err)
 
 							// Add our transaction to the block
-							err = chain.PendingBlockAddTx(&msg)
+							err, _ = chain.PendingBlockAddTx(&msg)
 							assert.NoError(t, err)
 
 							// Commit the pending block to the chain, so it becomes the new head.
@@ -588,7 +588,7 @@ func TestChainCallSequenceReplayMatchSimple(t *testing.T) {
 							assert.NoError(t, err)
 
 							// Add our transaction to the block
-							err = chain.PendingBlockAddTx(&msg)
+							err, _ = chain.PendingBlockAddTx(&msg)
 							assert.NoError(t, err)
 
 							// Commit the pending block to the chain, so it becomes the new head.
@@ -627,7 +627,7 @@ func TestChainCallSequenceReplayMatchSimple(t *testing.T) {
 			_, err := recreatedChain.PendingBlockCreate()
 			assert.NoError(t, err)
 			for _, message := range chain.blocks[i].Messages {
-				err = recreatedChain.PendingBlockAddTx(message)
+				err, _ = recreatedChain.PendingBlockAddTx(message)
 				assert.NoError(t, err)
 			}
 			err = recreatedChain.PendingBlockCommit()

fuzzing/calls/call_sequence.go

@@ -3,12 +3,13 @@ package calls
 import (
 	"encoding/binary"
 	"fmt"
+	"github.com/crytic/medusa/fuzzing/executiontracer"
+	"github.com/ethereum/go-ethereum/core"
 	"strconv"
 
 	"github.com/crytic/medusa/chain"
 	chainTypes "github.com/crytic/medusa/chain/types"
 	fuzzingTypes "github.com/crytic/medusa/fuzzing/contracts"
-	"github.com/crytic/medusa/fuzzing/executiontracer"
 	"github.com/crytic/medusa/fuzzing/valuegeneration"
 	"github.com/crytic/medusa/logging"
 	"github.com/crytic/medusa/utils"
@@ -164,6 +165,10 @@ type CallSequenceElement struct {
 
 	// ExecutionTrace represents a verbose execution trace collected. Nil if an execution trace was not collected.
 	ExecutionTrace *executiontracer.ExecutionTrace `json:"-"`
+
+	// ExecutionResult provides a high-level overview of the result of executing the transaction. Nil if transaction
+	// has not been executed yet, or some other, higher-level error occurred.
+	ExecutionResult *core.ExecutionResult `json:"-"`
 }
 
 // NewCallSequenceElement returns a new CallSequenceElement struct to track a single call made within a CallSequence.

fuzzing/calls/call_sequence_execution.go

@@ -88,7 +88,8 @@ func ExecuteCallSequenceIteratively(chain *chain.TestChain, fetchElementFunc Exe
 			}
 
 			// Try to add our transaction to this block.
-			err = chain.PendingBlockAddTx(callSequenceElement.Call.ToCoreMessage(), additionalTracers...)
+			err, execResult := chain.PendingBlockAddTx(callSequenceElement.Call.ToCoreMessage(), additionalTracers...)
+			callSequenceElement.ExecutionResult = execResult
 
 			if err != nil {
 				// If we encountered a block gas limit error, this tx is too expensive to fit in this block.

fuzzing/config/config.go

@@ -133,6 +133,9 @@ type TestingConfig struct {
 	// even if this option is not enabled.
 	TraceAll bool `json:"traceAll"`
 
+	// ReversionMeasurement describes the configuration options used for measuring reversion rates of top level calls
+	ReversionMeasurement ReversionMeasurementConfig `json:"reversionMeasurement"`
+
 	// AssertionTesting describes the configuration used for assertion testing.
 	AssertionTesting AssertionTestingConfig `json:"assertionTesting"`
 
@@ -185,6 +188,12 @@ func (testCfg *TestingConfig) Validate() error {
 	return nil
 }
 
+// ReversionMeasurementConfig describe the configuration options used for measuring reversion rates of top level calls
+type ReversionMeasurementConfig struct {
+	// Enabled describes whether reversion measurement is enabled.
+	Enabled bool `json:"enabled"`
+}
+
 // AssertionTestingConfig describes the configuration options used for assertion testing
 type AssertionTestingConfig struct {
 	// Enabled describes whether testing is enabled.

fuzzing/config/config_defaults.go

@@ -64,6 +64,9 @@ func GetDefaultProjectConfig(platform string) (*ProjectConfig, error) {
 				TraceAll:                     false,
 				TargetFunctionSignatures:     []string{},
 				ExcludeFunctionSignatures:    []string{},
+				ReversionMeasurement: ReversionMeasurementConfig{
+					Enabled: true,
+				},
 				AssertionTesting: AssertionTestingConfig{
 					Enabled:         true,
 					TestViewMethods: false,

fuzzing/fuzzer.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/crytic/medusa/fuzzing/coverage"
+	"github.com/crytic/medusa/fuzzing/reversion"
 	"math/big"
 	"math/rand"
 	"os"
@@ -19,7 +21,6 @@ import (
 
 	"github.com/crytic/medusa/fuzzing/executiontracer"
 
-	"github.com/crytic/medusa/fuzzing/coverage"
 	"github.com/crytic/medusa/logging"
 	"github.com/crytic/medusa/logging/colors"
 	"github.com/rs/zerolog"
@@ -70,6 +71,8 @@ type Fuzzer struct {
 	// corpus stores a list of transaction sequences that can be used for coverage-guided fuzzing
 	corpus *corpus.Corpus
 
+	ReversionStats *reversion.ReversionMeasurer
+
 	// randomProvider describes the provider used to generate random values in the Fuzzer. All other random providers
 	// used by the Fuzzer's subcomponents are derived from this one.
 	randomProvider *rand.Rand
@@ -153,6 +156,7 @@ func NewFuzzer(config config.ProjectConfig) (*Fuzzer, error) {
 		contractDefinitions: make(fuzzerTypes.Contracts, 0),
 		testCases:           make([]TestCase, 0),
 		testCasesFinished:   make(map[string]TestCase),
+		ReversionStats:      reversion.CreateReversionMeasurer(config),
 		Hooks: FuzzerHooks{
 			NewCallSequenceGeneratorConfigFunc: defaultCallSequenceGeneratorConfigFunc,
 			NewShrinkingValueMutatorFunc:       defaultShrinkingValueMutatorFunc,
@@ -476,7 +480,7 @@ func chainSetupFromCompilations(fuzzer *Fuzzer, testChain *chain.TestChain) (*ex
 				}
 
 				// Add our transaction to the block
-				err = testChain.PendingBlockAddTx(msg.ToCoreMessage())
+				err, _ = testChain.PendingBlockAddTx(msg.ToCoreMessage())
 				if err != nil {
 					return nil, err
 				}
@@ -803,12 +807,15 @@ func (f *Fuzzer) Start() error {
 		return err
 	}
 
+	// Measure reversion stats across all workers
+	cleanup := f.ReversionStats.StartWorker(f.ctx)
+
 	// Run the main worker loop
 	err = f.spawnWorkersLoop(baseTestChain)
 	if err != nil {
 		f.logger.Error("Encountered an error in the main fuzzing loop", err)
 	}
-
+	cleanup()
 	// NOTE: After this point, we capture errors but do not return immediately, as we want to exit gracefully.
 
 	// If we have coverage enabled and a corpus directory set, write the corpus. We do this even if we had a
@@ -827,11 +834,16 @@ func (f *Fuzzer) Start() error {
 		err = fuzzerStoppingErr
 		f.logger.Error("FuzzerStopping event subscriber returned an error", err)
 	}
-
 	// Print our results on exit.
 	f.printExitingResults()
 
+	err = f.ReversionStats.BuildArtifact(f.logger, f.ContractDefinitions(), f.config.Fuzzing.CorpusDirectory)
+	if err != nil {
+		f.logger.Error("Failed to convert reversion metrics to an artifact", err)
+	}
+
 	// Finally, generate our coverage report if we have set a valid corpus directory.
+
 	if err == nil && f.config.Fuzzing.CorpusDirectory != "" {
 		coverageReportPath := filepath.Join(f.config.Fuzzing.CorpusDirectory, "coverage_report.html")
 		err = coverage.GenerateReport(f.compilations, f.corpus.CoverageMaps(), coverageReportPath)
@@ -841,6 +853,10 @@ func (f *Fuzzer) Start() error {
 			f.logger.Info("Coverage report saved to file: ", colors.Bold, coverageReportPath, colors.Reset)
 		}
 	}
+	err = f.ReversionStats.WriteReport(f.config.Fuzzing.CorpusDirectory, f.logger)
+	if err != nil {
+		f.logger.Error("Failed to write reversion metrics to disk", err)
+	}
 
 	// Return any encountered error.
 	return err

fuzzing/fuzzer_test_methods_test.go

@@ -56,6 +56,7 @@ func getFuzzerTestingProjectConfig(t *testing.T, compilationConfig *compilation.
 	assert.NoError(t, err)
 	projectConfig.Compilation = compilationConfig
 	projectConfig.Fuzzing.Workers = 3
+	projectConfig.Fuzzing.Testing.ReversionMeasurement.Enabled = true
 	projectConfig.Fuzzing.WorkerResetLimit = 50
 	projectConfig.Fuzzing.Timeout = 0
 	projectConfig.Fuzzing.TestLimit = 1_500_000

fuzzing/fuzzer_worker.go

@@ -541,6 +541,7 @@ func (fw *FuzzerWorker) run(baseTestChain *chain.TestChain) (bool, error) {
 		// Subscribe our chain event handlers
 		initializedChain.Events.ContractDeploymentAddedEventEmitter.Subscribe(fw.onChainContractDeploymentAddedEvent)
 		initializedChain.Events.ContractDeploymentRemovedEventEmitter.Subscribe(fw.onChainContractDeploymentRemovedEvent)
+		initializedChain.Events.PendingBlockCommitted.Subscribe(fw.Fuzzer().ReversionStats.OnPendingBlockCommittedEvent)
 
 		// Emit an event indicating the worker has created its chain.
 		err = fw.Events.FuzzerWorkerChainCreated.Publish(FuzzerWorkerChainCreatedEvent{
@@ -615,9 +616,10 @@ func (fw *FuzzerWorker) run(baseTestChain *chain.TestChain) (bool, error) {
 			}
 		}
 
-		// Emit an event indicating the worker is about to test a new call sequence.
+		// Emit an event indicating the worker has completed testing a call sequence.
 		err = fw.Events.CallSequenceTested.Publish(FuzzerWorkerCallSequenceTestedEvent{
-			Worker: fw,
+			Worker:   fw,
+			Sequence: callSequence,
 		})
 		if err != nil {
 			return false, fmt.Errorf("error returned by an event handler when a worker emitted an event indicating testing of a new call sequence has concluded: %v", err)

fuzzing/fuzzer_worker_events.go

@@ -3,6 +3,7 @@ package fuzzing
 import (
 	"github.com/crytic/medusa/chain"
 	"github.com/crytic/medusa/events"
+	"github.com/crytic/medusa/fuzzing/calls"
 	"github.com/crytic/medusa/fuzzing/contracts"
 	"github.com/ethereum/go-ethereum/common"
 )
@@ -93,4 +94,6 @@ type FuzzerWorkerCallSequenceTestingEvent struct {
 type FuzzerWorkerCallSequenceTestedEvent struct {
 	// Worker represents the instance of the fuzzing.FuzzerWorker for which the event occurred.
 	Worker *FuzzerWorker
+	// Sequence represents the call sequence that was tested
+	Sequence calls.CallSequence
 }