Storage Fees - Storage Used (#215)

cmd/util/cmd/execution-state-extract/execution_state_extract.go

@@ -20,15 +20,21 @@ func getStateCommitment(commits storage.Commits, blockHash flow.Identifier) (flo
 
 func extractExecutionState(dir string, targetHash flow.StateCommitment, outputDir string, log zerolog.Logger) error {
 
-	led, err := complete.NewLedger(dir, complete.DefaultCacheSize, &metrics.NoopCollector{}, log, nil, 0)
+	led, err := complete.NewLedger(
+		dir,
+		complete.DefaultCacheSize,
+		&metrics.NoopCollector{},
+		log,
+		nil,
+		complete.DefaultPathFinderVersion)
 	if err != nil {
 		return fmt.Errorf("cannot create ledger from write-a-head logs and checkpoints: %w", err)
 	}
 	filePath := path.Join(outputDir, "root.checkpoint")
 
 	newState, err := led.ExportCheckpointAt(targetHash,
-		[]ledger.Migration{migrations.MultipleContractMigration, migrations.AddMissingKeysMigration},
-		[]ledger.Reporter{},
+		[]ledger.Migration{migrations.StorageFeesMigration},
+		[]ledger.Reporter{migrations.StorageReporter{Log: log}},
 		complete.DefaultPathFinderVersion,
 		filePath)
 	if err != nil {

cmd/util/ledger/migrations/accounts.go

@@ -101,6 +101,10 @@ func (l *led) Get(owner, controller, key string) (flow.RegisterValue, error) {
 	return flow.RegisterValue(l.payloads[fk].Value), nil
 }
 
+func (l *led) RegisterUpdates() ([]flow.RegisterID, []flow.RegisterValue) {
+	panic("this method shouldn't be used here")
+}
+
 func (l *led) Delete(owner, controller, key string) {
 	fk := fullKey(owner, controller, key)
 	delete(l.payloads, fk)

cmd/util/ledger/migrations/multiple_contract_migration.go

@@ -61,21 +61,6 @@ func MultipleContractMigration(payloads []ledger.Payload) ([]ledger.Payload, err
 	return migratedPayloads, nil
 }
 
-func keyToRegisterId(key ledger.Key) (flow.RegisterID, error) {
-	if len(key.KeyParts) != 3 ||
-		key.KeyParts[0].Type != state.KeyPartOwner ||
-		key.KeyParts[1].Type != state.KeyPartController ||
-		key.KeyParts[2].Type != state.KeyPartKey {
-		return flow.RegisterID{}, fmt.Errorf("key not in expected format %s", key.String())
-	}
-
-	return flow.NewRegisterID(
-		string(key.KeyParts[0].Value),
-		string(key.KeyParts[1].Value),
-		string(key.KeyParts[2].Value),
-	), nil
-}
-
 func createContractNamesKey(originalKey ledger.Key) ledger.Key {
 	return ledger.Key{
 		KeyParts: []ledger.KeyPart{
@@ -114,7 +99,7 @@ func contractsRegister(contractsKey ledger.Key, contractNames []string) (ledger.
 const deferredValueOfContractValuePrefix = "contract\x1f"
 
 func migrateNonContractValue(p ledger.Payload, contractValueMappings map[string]string) ([]ledger.Payload, error) {
-	registerID, err := keyToRegisterId(p.Key)
+	registerID, err := keyToRegisterID(p.Key)
 	if err != nil {
 		return nil, err
 	}
@@ -189,7 +174,7 @@ func migrateContractValues(payloads []ledger.Payload) ([]ledger.Payload, map[str
 	errors := make([]error, 0)
 
 	for _, p := range payloads {
-		registerID, err := keyToRegisterId(p.Key)
+		registerID, err := keyToRegisterID(p.Key)
 		if err != nil {
 			// dont fail fast... try to collect errors so multiple errors can be addressed at once
 			errors = append(errors, err)

cmd/util/ledger/migrations/storage_fees_migration.go

@@ -0,0 +1,65 @@
+package migrations
+
+import (
+	fvm "github.com/onflow/flow-go/fvm/state"
+	"github.com/onflow/flow-go/ledger"
+	"github.com/onflow/flow-go/ledger/common/utils"
+	"github.com/onflow/flow-go/model/flow"
+)
+
+// iterates through registers keeping a map of register sizes
+// after it has reached the end it add storage used and storage capacity for each address
+func StorageFeesMigration(payload []ledger.Payload) ([]ledger.Payload, error) {
+	storageUsed := make(map[string]uint64)
+	newPayload := make([]ledger.Payload, len(payload))
+
+	for i, p := range payload {
+		err := incrementStorageUsed(p, storageUsed)
+		if err != nil {
+			return nil, err
+		}
+		newPayload[i] = p
+	}
+
+	for s, u := range storageUsed {
+		// this is the storage used by the storage_used register we are about to add
+		storageUsedByStorageUsed := fvm.RegisterSize(
+			flow.BytesToAddress([]byte(s)),
+			false, "storage_used",
+			make([]byte, 8))
+		u = u + uint64(storageUsedByStorageUsed)
+
+		newPayload = append(newPayload, ledger.Payload{
+			Key: registerIDToKey(flow.RegisterID{
+				Owner:      s,
+				Controller: "",
+				Key:        "storage_used",
+			}),
+			Value: utils.Uint64ToBinary(u),
+		})
+	}
+	return newPayload, nil
+}
+
+func incrementStorageUsed(p ledger.Payload, used map[string]uint64) error {
+	id, err := keyToRegisterID(p.Key)
+	if err != nil {
+		return err
+	}
+	if len([]byte(id.Owner)) != flow.AddressLength {
+		// not an address
+		return nil
+	}
+	if _, ok := used[id.Owner]; !ok {
+		used[id.Owner] = 0
+	}
+	used[id.Owner] = used[id.Owner] + uint64(registerSize(id, p))
+	return nil
+}
+
+func registerSize(id flow.RegisterID, p ledger.Payload) int {
+	address := flow.BytesToAddress([]byte(id.Owner))
+	isController := len(id.Controller) > 0
+	key := id.Key
+	return fvm.RegisterSize(address, isController, key, p.Value)
+}

cmd/util/ledger/migrations/storage_reporter.go

@@ -0,0 +1,110 @@
+package migrations
+
+import (
+	"math"
+	"strings"
+
+	"github.com/rs/zerolog"
+
+	"github.com/onflow/flow-go/ledger"
+	"github.com/onflow/flow-go/ledger/common/utils"
+	"github.com/onflow/flow-go/model/flow"
+)
+
+// iterates through registers keeping a map of register sizes
+// reports on storage metrics
+type StorageReporter struct {
+	Log zerolog.Logger
+}
+
+func (r StorageReporter) Report(payload []ledger.Payload) error {
+	r.Log.Info().Msg("Running Storage Reporter")
+	storageUsed := make(map[string]uint64)
+	average := 0.0
+	max := uint64(0)
+
+	for i, p := range payload {
+		id, err := keyToRegisterID(p.Key)
+		if err != nil {
+			return err
+		}
+		if len([]byte(id.Owner)) != flow.AddressLength {
+			// not an address
+			continue
+		}
+		if id.Key != "storage_used" {
+			continue
+		}
+		u, _, err := utils.ReadUint64(p.Value)
+		if err != nil {
+			return err
+		}
+		storageUsed[id.Owner] = u
+		average = average + (float64(u)-average)/(float64(i)+1.0)
+		if u > max {
+			max = u
+		}
+	}
+	r.Log.Info().
+		Msgf("Average storage used %g", average)
+	r.Log.Info().
+		Msgf("Max storage used %d", max)
+
+	bins := 150
+	binHeight := 30
+	w := float64(max) / float64(bins)
+	distribution := make([]float64, bins)
+
+	var maxBin = float64(0)
+
+	for s, u := range storageUsed {
+		var i = int(float64(u) / w)
+		if i >= bins {
+			i = bins - 1
+		}
+		distribution[i] = distribution[i] + 1
+
+		if maxBin < distribution[i] {
+			maxBin = distribution[i]
+		}
+		if i == bins-1 {
+			r.Log.Info().
+				Msgf("High storage usage address: %s, used: %d bytes", flow.BytesToAddress([]byte(s)), u)
+		}
+	}
+	maxBinAccounts := maxBin
+	maxBin = math.Log10(maxBin)
+
+	graph := make([]int, bins)
+
+	for i, d := range distribution {
+		var v float64
+		if d <= 0 {
+			v = 0
+		} else {
+			v = math.Log10(d)
+		}
+
+		graph[i] = int(math.Ceil((v / maxBin) * float64(binHeight)))
+	}
+
+	r.Log.Info().
+		Msgf("Logarithmic account storage distribution x[storage used] / y[log number of accounts (max = %f accounts)]:", maxBinAccounts)
+
+	var sb strings.Builder
+	for j := binHeight; j >= 0; j-- {
+		sb.WriteRune('\n')
+		for i := 0; i < bins; i++ {
+			if graph[i] >= j {
+				sb.WriteRune('#')
+			} else {
+				sb.WriteRune('.')
+			}
+		}
+	}
+
+	r.Log.Info().
+		Msg(sb.String())
+
+	return nil
+}

cmd/util/ledger/migrations/utils.go

@@ -0,0 +1,43 @@
+package migrations
+
+import (
+	"fmt"
+
+	"github.com/onflow/flow-go/engine/execution/state"
+	"github.com/onflow/flow-go/ledger"
+	"github.com/onflow/flow-go/model/flow"
+)
+
+func keyToRegisterID(key ledger.Key) (flow.RegisterID, error) {
+	if len(key.KeyParts) != 3 ||
+		key.KeyParts[0].Type != state.KeyPartOwner ||
+		key.KeyParts[1].Type != state.KeyPartController ||
+		key.KeyParts[2].Type != state.KeyPartKey {
+		return flow.RegisterID{}, fmt.Errorf("key not in expected format %s", key.String())
+	}
+
+	return flow.NewRegisterID(
+		string(key.KeyParts[0].Value),
+		string(key.KeyParts[1].Value),
+		string(key.KeyParts[2].Value),
+	), nil
+}
+
+func registerIDToKey(registerID flow.RegisterID) ledger.Key {
+	newKey := ledger.Key{}
+	newKey.KeyParts = []ledger.KeyPart{
+		{
+			Type:  state.KeyPartOwner,
+			Value: []byte(registerID.Owner),
+		},
+		{
+			Type:  state.KeyPartController,
+			Value: []byte(registerID.Controller),
+		},
+		{
+			Type:  state.KeyPartKey,
+			Value: []byte(registerID.Key),
+		},
+	}
+	return newKey
+}

engine/execution/state/bootstrap/bootstrap_test.go

@@ -44,7 +44,7 @@ func TestBootstrapLedger(t *testing.T) {
 }
 
 func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) {
-	var expectedStateCommitment, _ = hex.DecodeString("fc943c9727c51663327c983262d09bbfeda8a4c8a21224a82c9af6b8bd65bb8f")
+	var expectedStateCommitment, _ = hex.DecodeString("8186685d5f826f1aeee38556d1bb7321bec8cb6a09094aae169c106159a63157")
 
 	unittest.RunWithTempDir(t, func(dbDir string) {
 

engine/execution/state/delta/delta.go

@@ -51,19 +51,9 @@ func (d Delta) Set(owner, controller, key string, value flow.RegisterValue) {
 	}
 }
 
-// Delete records a deletion in this delta.
-func (d Delta) Delete(owner, controller, key string) {
-	k := toString(owner, controller, key)
-	d.Data[k] = flow.RegisterEntry{
-		Key:   toRegisterID(owner, controller, key),
-		Value: nil,
-	}
-}
-
 // RegisterUpdates returns all registers that were updated by this delta.
 // ids are returned sorted, in ascending order
 func (d Delta) RegisterUpdates() ([]flow.RegisterID, []flow.RegisterValue) {
-
 	data := make(flow.RegisterEntries, 0, len(d.Data))
 
 	for _, v := range d.Data {

engine/execution/state/delta/delta_test.go

@@ -50,31 +50,6 @@ func TestDelta_Set(t *testing.T) {
 	assert.True(t, exists)
 }
 
-func TestDelta_Delete(t *testing.T) {
-	registerID1 := "fruit"
-
-	t.Run("ValueNotSet", func(t *testing.T) {
-		d := delta.NewDelta()
-
-		d.Delete(registerID1, "", "")
-
-		b, exists := d.Get(registerID1, "", "")
-		assert.Nil(t, b)
-		assert.True(t, exists)
-	})
-
-	t.Run("ValueSet", func(t *testing.T) {
-		d := delta.NewDelta()
-
-		d.Set(registerID1, "", "", []byte("apple"))
-		d.Delete(registerID1, "", "")
-
-		b, exists := d.Get(registerID1, "", "")
-		assert.Nil(t, b)
-		assert.True(t, exists)
-	})
-}
-
 func TestDelta_MergeWith(t *testing.T) {
 	registerID1 := "fruit"
 
@@ -114,7 +89,7 @@ func TestDelta_MergeWith(t *testing.T) {
 		d2 := delta.NewDelta()
 
 		d1.Set(registerID1, "", "", flow.RegisterValue("apple"))
-		d1.Delete(registerID1, "", "")
+		d1.Set(registerID1, "", "", nil)
 
 		d2.Set(registerID1, "", "", flow.RegisterValue("orange"))
 
@@ -130,7 +105,7 @@ func TestDelta_MergeWith(t *testing.T) {
 
 		d1.Set(registerID1, "", "", flow.RegisterValue("apple"))
 
-		d2.Delete(registerID1, "", "")
+		d2.Set(registerID1, "", "", nil)
 
 		d1.MergeWith(d2)
 

engine/execution/state/delta/view.go

@@ -147,6 +147,10 @@ func (v *View) Set(owner, controller, key string, value flow.RegisterValue) {
 	v.delta.Set(owner, controller, key, value)
 }
 
+func (v *View) RegisterUpdates() ([]flow.RegisterID, []flow.RegisterValue) {
+	return v.delta.RegisterUpdates()
+}
+
 func (v *View) updateSpock(value []byte) error {
 	_, err := v.spockSecretHasher.Write(value)
 	if err != nil {
@@ -168,7 +172,7 @@ func (v *View) Touch(owner, controller, key string) {
 
 // Delete removes a register in this view.
 func (v *View) Delete(owner, controller, key string) {
-	v.delta.Delete(owner, controller, key)
+	v.Set(owner, controller, key, nil)
 }
 
 // Delta returns a record of the registers that were mutated in this view.