From b0fff9a1ddad843922fad0b82528aa0a86641506 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 18 Jul 2019 13:30:04 -0500 Subject: [PATCH 1/6] First pass at decoded packed storage slots --- libraries/shared/storage/utils/decoder.go | 40 +++++++++++++++++++ .../shared/storage/utils/decoder_test.go | 25 ++++++++++++ libraries/shared/storage/utils/value.go | 2 + pkg/history/populate_headers.go | 1 - 4 files changed, 67 insertions(+), 1 deletion(-) diff --git a/libraries/shared/storage/utils/decoder.go b/libraries/shared/storage/utils/decoder.go index 1a6ac31ca..88cd64742 100644 --- a/libraries/shared/storage/utils/decoder.go +++ b/libraries/shared/storage/utils/decoder.go @@ -33,6 +33,8 @@ func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, err return decodeAddress(row.StorageValue.Bytes()), nil case Bytes32: return row.StorageValue.Hex(), nil + case PackedSlot: + return decodePackedSlot(row.StorageValue.Bytes(), metadata.PackedTypes), nil default: panic(fmt.Sprintf("can't decode unknown type: %d", metadata.Type)) } @@ -51,3 +53,41 @@ func decodeUint48(raw []byte) string { func decodeAddress(raw []byte) string { return common.BytesToAddress(raw).Hex() } + +//this may need to return a slice of strings, a string for each item +func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) []string{ + storageSlot := raw + var results []string + //the reason we're using a map and not a slice is that golang doesn't guarantee the order of a slice + for _, valueType := range packedTypes { + lengthOfStorageSlot := len(storageSlot) + lengthOfItem := getLengthOfItem(valueType) + itemStartingIndex := lengthOfStorageSlot - lengthOfItem + value := storageSlot[itemStartingIndex:] + decodedValue := decodeIndividualItems(value, valueType) + results = append(results, decodedValue) + + //pop last item off slot before moving on + storageSlot = storageSlot[0:itemStartingIndex] + } + + return results +} + +func decodeIndividualItems(itemBytes []byte, valueType ValueType) string { + switch valueType { + case Uint48: + return decodeUint48(itemBytes) + default: + panic(fmt.Sprintf("can't decode unknown type: %d", valueType)) + } +} + +func getLengthOfItem(valueType ValueType) int{ + switch valueType { + case Uint48: + return 6 + default: + panic(fmt.Sprintf("ValueType %d not recognized", valueType)) + } +} diff --git a/libraries/shared/storage/utils/decoder_test.go b/libraries/shared/storage/utils/decoder_test.go index 9e7bb65a6..a01f40db1 100644 --- a/libraries/shared/storage/utils/decoder_test.go +++ b/libraries/shared/storage/utils/decoder_test.go @@ -49,6 +49,31 @@ var _ = Describe("Storage decoder", func() { Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String())) }) + Describe("when there are multiple items packed in the storage slot", func() { + It("decodes the first uint48 item packed in", func() { + packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30") + row := utils.StorageDiffRow{StorageValue: packedStorage} + packedTypes := map[int]utils.ValueType{} + packedTypes[0] = utils.Uint48 + packedTypes[1] = utils.Uint48 + + metadata := utils.StorageValueMetadata{ + Type: utils.PackedSlot, + PackedTypes: packedTypes, + } + + result, err := utils.Decode(row, metadata) + decodedValues := result.([]string) + + Expect(err).NotTo(HaveOccurred()) + expectedResult1 := big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String() + expectedResult2 := big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String() + Expect(decodedValues[0]).To(Equal(expectedResult1)) + Expect(decodedValues[1]).To(Equal(expectedResult2)) + }) + + }) + It("decodes address", func() { fakeAddress := common.HexToAddress("0x12345") row := utils.StorageDiffRow{StorageValue: fakeAddress.Hash()} diff --git a/libraries/shared/storage/utils/value.go b/libraries/shared/storage/utils/value.go index 095ff8fdb..0580b6770 100644 --- a/libraries/shared/storage/utils/value.go +++ b/libraries/shared/storage/utils/value.go @@ -23,6 +23,7 @@ const ( Uint48 Bytes32 Address + PackedSlot ) type Key string @@ -31,6 +32,7 @@ type StorageValueMetadata struct { Name string Keys map[Key]string Type ValueType + PackedTypes map[int]ValueType //type of each item packed and their order } func GetStorageValueMetadata(name string, keys map[Key]string, t ValueType) StorageValueMetadata { diff --git a/pkg/history/populate_headers.go b/pkg/history/populate_headers.go index f2abc93b2..1566cc642 100644 --- a/pkg/history/populate_headers.go +++ b/pkg/history/populate_headers.go @@ -61,4 +61,3 @@ func RetrieveAndUpdateHeaders(blockChain core.BlockChain, headerRepository datas } return len(blockNumbers), nil } - From ebfb4965c78ebd8d15181f632e02dd3e4cd0ee61 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 18 Jul 2019 16:18:20 -0500 Subject: [PATCH 2/6] Add tests for decoding packed storage --- libraries/shared/storage/utils/decoder.go | 19 +++- .../shared/storage/utils/decoder_test.go | 91 ++++++++++++++++--- libraries/shared/storage/utils/value.go | 1 + 3 files changed, 94 insertions(+), 17 deletions(-) diff --git a/libraries/shared/storage/utils/decoder.go b/libraries/shared/storage/utils/decoder.go index 88cd64742..149e41ada 100644 --- a/libraries/shared/storage/utils/decoder.go +++ b/libraries/shared/storage/utils/decoder.go @@ -29,6 +29,8 @@ func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, err return decodeUint256(row.StorageValue.Bytes()), nil case Uint48: return decodeUint48(row.StorageValue.Bytes()), nil + case Uint128: + return decodeUint128(row.StorageValue.Bytes()), nil case Address: return decodeAddress(row.StorageValue.Bytes()), nil case Bytes32: @@ -45,6 +47,11 @@ func decodeUint256(raw []byte) string { return n.String() } +func decodeUint128(raw []byte) string { + n := big.NewInt(0).SetBytes(raw) + return n.String() +} + func decodeUint48(raw []byte) string { n := big.NewInt(0).SetBytes(raw) return n.String() @@ -54,14 +61,13 @@ func decodeAddress(raw []byte) string { return common.BytesToAddress(raw).Hex() } -//this may need to return a slice of strings, a string for each item func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) []string{ storageSlot := raw var results []string //the reason we're using a map and not a slice is that golang doesn't guarantee the order of a slice for _, valueType := range packedTypes { lengthOfStorageSlot := len(storageSlot) - lengthOfItem := getLengthOfItem(valueType) + lengthOfItem := getNumberOfBytes(valueType) itemStartingIndex := lengthOfStorageSlot - lengthOfItem value := storageSlot[itemStartingIndex:] decodedValue := decodeIndividualItems(value, valueType) @@ -78,15 +84,20 @@ func decodeIndividualItems(itemBytes []byte, valueType ValueType) string { switch valueType { case Uint48: return decodeUint48(itemBytes) + case Uint128: + return decodeUint128(itemBytes) default: panic(fmt.Sprintf("can't decode unknown type: %d", valueType)) } } -func getLengthOfItem(valueType ValueType) int{ +func getNumberOfBytes(valueType ValueType) int{ + // 8 bits per byte switch valueType { case Uint48: - return 6 + return 48 / 8 + case Uint128: + return 128 / 8 default: panic(fmt.Sprintf("ValueType %d not recognized", valueType)) } diff --git a/libraries/shared/storage/utils/decoder_test.go b/libraries/shared/storage/utils/decoder_test.go index a01f40db1..edb02ee0a 100644 --- a/libraries/shared/storage/utils/decoder_test.go +++ b/libraries/shared/storage/utils/decoder_test.go @@ -17,11 +17,10 @@ package utils_test import ( - "math/big" - "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "math/big" "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" ) @@ -38,6 +37,17 @@ var _ = Describe("Storage decoder", func() { Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String())) }) + It("decodes uint128", func() { + fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000011123") + row := utils.StorageDiffRow{StorageValue: fakeInt} + metadata := utils.StorageValueMetadata{Type: utils.Uint128} + + result, err := utils.Decode(row, metadata) + + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String())) + }) + It("decodes uint48", func() { fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123") row := utils.StorageDiffRow{StorageValue: fakeInt} @@ -49,8 +59,20 @@ var _ = Describe("Storage decoder", func() { Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String())) }) + It("decodes address", func() { + fakeAddress := common.HexToAddress("0x12345") + row := utils.StorageDiffRow{StorageValue: fakeAddress.Hash()} + metadata := utils.StorageValueMetadata{Type: utils.Address} + + result, err := utils.Decode(row, metadata) + + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(fakeAddress.Hex())) + }) + Describe("when there are multiple items packed in the storage slot", func() { - It("decodes the first uint48 item packed in", func() { + It("decodes uint48 items", func() { + //this is a real storage data example packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30") row := utils.StorageDiffRow{StorageValue: packedStorage} packedTypes := map[int]utils.ValueType{} @@ -68,20 +90,63 @@ var _ = Describe("Storage decoder", func() { Expect(err).NotTo(HaveOccurred()) expectedResult1 := big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String() expectedResult2 := big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String() - Expect(decodedValues[0]).To(Equal(expectedResult1)) - Expect(decodedValues[1]).To(Equal(expectedResult2)) + Expect(decodedValues).To(ConsistOf(expectedResult1, expectedResult2)) }) - }) + It("decodes 5 uint48 items", func() { + //TODO: this packedStorageHex was generated by hand, it would be nice to test this against + //real storage data that has several items packed into it + packedStorageHex := "0000000A5D1AFFFFFFFFFFFE00000009F3C600000002A300000000002A30" - It("decodes address", func() { - fakeAddress := common.HexToAddress("0x12345") - row := utils.StorageDiffRow{StorageValue: fakeAddress.Hash()} - metadata := utils.StorageValueMetadata{Type: utils.Address} + packedStorage := common.HexToHash(packedStorageHex) + row := utils.StorageDiffRow{StorageValue: packedStorage} + packedTypes := map[int]utils.ValueType{} + packedTypes[0] = utils.Uint48 + packedTypes[1] = utils.Uint48 + packedTypes[2] = utils.Uint48 + packedTypes[3] = utils.Uint48 + packedTypes[4] = utils.Uint48 - result, err := utils.Decode(row, metadata) + metadata := utils.StorageValueMetadata{ + Type: utils.PackedSlot, + PackedTypes: packedTypes, + } - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal(fakeAddress.Hex())) + result, err := utils.Decode(row, metadata) + decodedValues := result.([]string) + + Expect(err).NotTo(HaveOccurred()) + expectedResult1 := big.NewInt(0).SetBytes(common.HexToHash("A5D1A").Bytes()).String() + expectedResult2 := big.NewInt(0).SetBytes(common.HexToHash("FFFFFFFFFFFE").Bytes()).String() + expectedResult3 := big.NewInt(0).SetBytes(common.HexToHash("9F3C6").Bytes()).String() + expectedResult4 := big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String() + expectedResult5 := big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String() + Expect(decodedValues).To(ConsistOf(expectedResult1, expectedResult2, expectedResult3, expectedResult4, expectedResult5)) + }) + + It("decodes 2 uint128 items", func() { + //TODO: this packedStorageHex was generated by hand, it would be nice to test this against + //real storage data that has several items packed into it + packedStorageHex := "000000038D7EA4C67FF8E502B6730000" + + "0000000000000000AB54A98CEB1F0AD2" + packedStorage := common.HexToHash(packedStorageHex) + row := utils.StorageDiffRow{StorageValue: packedStorage} + packedTypes := map[int]utils.ValueType{} + packedTypes[0] = utils.Uint128 + packedTypes[1] = utils.Uint128 + + metadata := utils.StorageValueMetadata{ + Type: utils.PackedSlot, + PackedTypes: packedTypes, + } + + result, err := utils.Decode(row, metadata) + decodedValues := result.([]string) + + Expect(err).NotTo(HaveOccurred()) + expectedResult1 := big.NewInt(0).SetBytes(common.HexToHash("000000038D7EA4C67FF8E502B6730000").Bytes()).String() + expectedResult2 := big.NewInt(0).SetBytes(common.HexToHash("AB54A98CEB1F0AD2").Bytes()).String() + Expect(decodedValues).To(ConsistOf(expectedResult1, expectedResult2)) + }) }) }) diff --git a/libraries/shared/storage/utils/value.go b/libraries/shared/storage/utils/value.go index 0580b6770..c64a8d554 100644 --- a/libraries/shared/storage/utils/value.go +++ b/libraries/shared/storage/utils/value.go @@ -21,6 +21,7 @@ type ValueType int const ( Uint256 ValueType = iota Uint48 + Uint128 Bytes32 Address PackedSlot From 51915f75063f25a66a55390b9eea4e607417e700 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 18 Jul 2019 17:09:07 -0500 Subject: [PATCH 3/6] Pass storage values to repo.create as a map --- .../shared/factories/storage/transformer.go | 2 + .../factories/storage/transformer_test.go | 50 +++++++++++++++++++ libraries/shared/mocks/storage_repository.go | 1 + libraries/shared/storage/utils/decoder.go | 14 +++--- .../shared/storage/utils/decoder_test.go | 27 +++++----- libraries/shared/storage/utils/value.go | 1 + 6 files changed, 74 insertions(+), 21 deletions(-) diff --git a/libraries/shared/factories/storage/transformer.go b/libraries/shared/factories/storage/transformer.go index fce044658..58282e85f 100644 --- a/libraries/shared/factories/storage/transformer.go +++ b/libraries/shared/factories/storage/transformer.go @@ -47,9 +47,11 @@ func (transformer Transformer) Execute(row utils.StorageDiffRow) error { if lookupErr != nil { return lookupErr } + //packed storage slots return a slice of decoded values value, decodeErr := utils.Decode(row, metadata) if decodeErr != nil { return decodeErr } + return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value) } diff --git a/libraries/shared/factories/storage/transformer_test.go b/libraries/shared/factories/storage/transformer_test.go index 354c106b2..09b91d80b 100644 --- a/libraries/shared/factories/storage/transformer_test.go +++ b/libraries/shared/factories/storage/transformer_test.go @@ -100,4 +100,54 @@ var _ = Describe("Storage transformer", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) + + Describe("when a storage row contains more than one item packed in storage", func() { + var ( + rawValue = common.HexToAddress("000000000000000000000000000000000000000000000002a300000000002a30") + fakeBlockNumber = 123 + fakeBlockHash = "0x67890" + packedTypes = make(map[int]utils.ValueType) + ) + packedTypes[0] = utils.Uint48 + packedTypes[1] = utils.Uint48 + + var fakeMetadata = utils.StorageValueMetadata{ + Name: "", + Keys: nil, + Type: utils.PackedSlot, + PackedTypes: packedTypes, + } + + It("passes the decoded data items to the repository", func() { + mappings.Metadata = fakeMetadata + fakeRow := utils.StorageDiffRow{ + Contract: common.Address{}, + BlockHash: common.HexToHash(fakeBlockHash), + BlockHeight: fakeBlockNumber, + StorageKey: common.Hash{}, + StorageValue: rawValue.Hash(), + } + + err := t.Execute(fakeRow) + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedBlockNumber).To(Equal(fakeBlockNumber)) + Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex())) + Expect(repository.PassedMetadata).To(Equal(fakeMetadata)) + expectedPassedValue := make(map[int]string) + expectedPassedValue[0]= "10800" + expectedPassedValue[1]= "172800" + Expect(repository.PassedValue.(map[int]string)).To(Equal(expectedPassedValue)) + }) + + It("returns error if creating a row fails", func() { + mappings.Metadata = fakeMetadata + repository.CreateErr = fakes.FakeError + + err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()}) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + }) }) diff --git a/libraries/shared/mocks/storage_repository.go b/libraries/shared/mocks/storage_repository.go index c4e351bee..9a22a3070 100644 --- a/libraries/shared/mocks/storage_repository.go +++ b/libraries/shared/mocks/storage_repository.go @@ -33,6 +33,7 @@ func (repository *MockStorageRepository) Create(blockNumber int, blockHash strin repository.PassedBlockNumber = blockNumber repository.PassedBlockHash = blockHash repository.PassedMetadata = metadata + repository.PassedValue = value return repository.CreateErr } diff --git a/libraries/shared/storage/utils/decoder.go b/libraries/shared/storage/utils/decoder.go index 149e41ada..c672cd314 100644 --- a/libraries/shared/storage/utils/decoder.go +++ b/libraries/shared/storage/utils/decoder.go @@ -18,9 +18,8 @@ package utils import ( "fmt" - "math/big" - "github.com/ethereum/go-ethereum/common" + "math/big" ) func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) { @@ -61,17 +60,20 @@ func decodeAddress(raw []byte) string { return common.BytesToAddress(raw).Hex() } -func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) []string{ +func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) map[int]string{ storageSlot := raw - var results []string + var results = map[int]string{} + //the reason we're using a map and not a slice is that golang doesn't guarantee the order of a slice - for _, valueType := range packedTypes { + numberOfTypes := len(packedTypes) + for position := 0; position < numberOfTypes; position++ { + valueType := packedTypes[position] lengthOfStorageSlot := len(storageSlot) lengthOfItem := getNumberOfBytes(valueType) itemStartingIndex := lengthOfStorageSlot - lengthOfItem value := storageSlot[itemStartingIndex:] decodedValue := decodeIndividualItems(value, valueType) - results = append(results, decodedValue) + results[position] = decodedValue //pop last item off slot before moving on storageSlot = storageSlot[0:itemStartingIndex] diff --git a/libraries/shared/storage/utils/decoder_test.go b/libraries/shared/storage/utils/decoder_test.go index edb02ee0a..67df8d520 100644 --- a/libraries/shared/storage/utils/decoder_test.go +++ b/libraries/shared/storage/utils/decoder_test.go @@ -85,12 +85,11 @@ var _ = Describe("Storage decoder", func() { } result, err := utils.Decode(row, metadata) - decodedValues := result.([]string) + decodedValues := result.(map[int]string) Expect(err).NotTo(HaveOccurred()) - expectedResult1 := big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String() - expectedResult2 := big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String() - Expect(decodedValues).To(ConsistOf(expectedResult1, expectedResult2)) + Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String())) + Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String())) }) It("decodes 5 uint48 items", func() { @@ -113,15 +112,14 @@ var _ = Describe("Storage decoder", func() { } result, err := utils.Decode(row, metadata) - decodedValues := result.([]string) + decodedValues := result.(map[int]string) Expect(err).NotTo(HaveOccurred()) - expectedResult1 := big.NewInt(0).SetBytes(common.HexToHash("A5D1A").Bytes()).String() - expectedResult2 := big.NewInt(0).SetBytes(common.HexToHash("FFFFFFFFFFFE").Bytes()).String() - expectedResult3 := big.NewInt(0).SetBytes(common.HexToHash("9F3C6").Bytes()).String() - expectedResult4 := big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String() - expectedResult5 := big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String() - Expect(decodedValues).To(ConsistOf(expectedResult1, expectedResult2, expectedResult3, expectedResult4, expectedResult5)) + Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String())) + Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String())) + Expect(decodedValues[2]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("9F3C6").Bytes()).String())) + Expect(decodedValues[3]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("FFFFFFFFFFFE").Bytes()).String())) + Expect(decodedValues[4]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("A5D1A").Bytes()).String())) }) It("decodes 2 uint128 items", func() { @@ -141,12 +139,11 @@ var _ = Describe("Storage decoder", func() { } result, err := utils.Decode(row, metadata) - decodedValues := result.([]string) + decodedValues := result.(map[int]string) Expect(err).NotTo(HaveOccurred()) - expectedResult1 := big.NewInt(0).SetBytes(common.HexToHash("000000038D7EA4C67FF8E502B6730000").Bytes()).String() - expectedResult2 := big.NewInt(0).SetBytes(common.HexToHash("AB54A98CEB1F0AD2").Bytes()).String() - Expect(decodedValues).To(ConsistOf(expectedResult1, expectedResult2)) + Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("AB54A98CEB1F0AD2").Bytes()).String())) + Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("38D7EA4C67FF8E502B6730000").Bytes()).String())) }) }) }) diff --git a/libraries/shared/storage/utils/value.go b/libraries/shared/storage/utils/value.go index c64a8d554..865bf9ae4 100644 --- a/libraries/shared/storage/utils/value.go +++ b/libraries/shared/storage/utils/value.go @@ -34,6 +34,7 @@ type StorageValueMetadata struct { Keys map[Key]string Type ValueType PackedTypes map[int]ValueType //type of each item packed and their order + PackedNames map[int]string //name of each item packed and their order } func GetStorageValueMetadata(name string, keys map[Key]string, t ValueType) StorageValueMetadata { From d30b4ea4f281a85c4cae5e346d8e150a2663d636 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Fri, 19 Jul 2019 09:40:02 -0500 Subject: [PATCH 4/6] Update GetStorageValueMetadata --- libraries/shared/storage/utils/value.go | 26 ++++++++++++++------ libraries/shared/storage/utils/value_test.go | 19 +++++++++++++- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/libraries/shared/storage/utils/value.go b/libraries/shared/storage/utils/value.go index 865bf9ae4..ba463ab13 100644 --- a/libraries/shared/storage/utils/value.go +++ b/libraries/shared/storage/utils/value.go @@ -30,17 +30,27 @@ const ( type Key string type StorageValueMetadata struct { - Name string - Keys map[Key]string - Type ValueType + Name string + Keys map[Key]string + Type ValueType + PackedNames map[int]string //name of each item packed and their order PackedTypes map[int]ValueType //type of each item packed and their order - PackedNames map[int]string //name of each item packed and their order } -func GetStorageValueMetadata(name string, keys map[Key]string, t ValueType) StorageValueMetadata { +func GetStorageValueMetadata(name string, keys map[Key]string, valueType ValueType) StorageValueMetadata { + return getMetadata(name, keys, valueType, nil, nil) +} + +func GetStorageValueMetadataForPackedSlot(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata { + return getMetadata(name, keys, valueType, packedNames, packedTypes) +} + +func getMetadata(name string, keys map[Key]string, t ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata { return StorageValueMetadata{ - Name: name, - Keys: keys, - Type: t, + Name: name, + Keys: keys, + Type: t, + PackedNames: packedNames, + PackedTypes: packedTypes, } } diff --git a/libraries/shared/storage/utils/value_test.go b/libraries/shared/storage/utils/value_test.go index 6af204997..f07a8fae2 100644 --- a/libraries/shared/storage/utils/value_test.go +++ b/libraries/shared/storage/utils/value_test.go @@ -7,7 +7,7 @@ import ( ) var _ = Describe("Storage value metadata getter", func() { - It("returns a storage value metadata instance with corresponding fields assigned", func() { + It("returns storage value metadata for a single storage variable", func() { metadataName := "fake_name" metadataKeys := map[utils.Key]string{"key": "value"} metadataType := utils.Uint256 @@ -19,4 +19,21 @@ var _ = Describe("Storage value metadata getter", func() { } Expect(utils.GetStorageValueMetadata(metadataName, metadataKeys, metadataType)).To(Equal(expectedMetadata)) }) + + It("returns metadata for a packed storage slot variables", func() { + metadataName := "fake_name" + metadataKeys := map[utils.Key]string{"key": "value"} + metadataType := utils.Uint256 + metadataPackedNames := map[int]string{0: "name"} + metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48} + + expectedMetadata := utils.StorageValueMetadata{ + Name: metadataName, + Keys: metadataKeys, + Type: metadataType, + PackedTypes: metadataPackedTypes, + PackedNames: metadataPackedNames, + } + Expect(utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)).To(Equal(expectedMetadata)) + }) }) From 69ad521482f3659d43aebe72e1ebb2649bddca76 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Fri, 19 Jul 2019 09:56:21 -0500 Subject: [PATCH 5/6] Apply go fmt changes and cleanup --- .../shared/factories/storage/transformer.go | 2 - .../factories/storage/transformer_test.go | 10 ++--- libraries/shared/mocks/storage_repository.go | 1 - libraries/shared/storage/utils/decoder.go | 41 +++++++++++-------- .../shared/storage/utils/decoder_test.go | 3 +- libraries/shared/storage/utils/value.go | 4 +- 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/libraries/shared/factories/storage/transformer.go b/libraries/shared/factories/storage/transformer.go index 58282e85f..fce044658 100644 --- a/libraries/shared/factories/storage/transformer.go +++ b/libraries/shared/factories/storage/transformer.go @@ -47,11 +47,9 @@ func (transformer Transformer) Execute(row utils.StorageDiffRow) error { if lookupErr != nil { return lookupErr } - //packed storage slots return a slice of decoded values value, decodeErr := utils.Decode(row, metadata) if decodeErr != nil { return decodeErr } - return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value) } diff --git a/libraries/shared/factories/storage/transformer_test.go b/libraries/shared/factories/storage/transformer_test.go index 09b91d80b..59982a201 100644 --- a/libraries/shared/factories/storage/transformer_test.go +++ b/libraries/shared/factories/storage/transformer_test.go @@ -103,10 +103,10 @@ var _ = Describe("Storage transformer", func() { Describe("when a storage row contains more than one item packed in storage", func() { var ( - rawValue = common.HexToAddress("000000000000000000000000000000000000000000000002a300000000002a30") + rawValue = common.HexToAddress("000000000000000000000000000000000000000000000002a300000000002a30") fakeBlockNumber = 123 - fakeBlockHash = "0x67890" - packedTypes = make(map[int]utils.ValueType) + fakeBlockHash = "0x67890" + packedTypes = make(map[int]utils.ValueType) ) packedTypes[0] = utils.Uint48 packedTypes[1] = utils.Uint48 @@ -135,8 +135,8 @@ var _ = Describe("Storage transformer", func() { Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex())) Expect(repository.PassedMetadata).To(Equal(fakeMetadata)) expectedPassedValue := make(map[int]string) - expectedPassedValue[0]= "10800" - expectedPassedValue[1]= "172800" + expectedPassedValue[0] = "10800" + expectedPassedValue[1] = "172800" Expect(repository.PassedValue.(map[int]string)).To(Equal(expectedPassedValue)) }) diff --git a/libraries/shared/mocks/storage_repository.go b/libraries/shared/mocks/storage_repository.go index 9a22a3070..c4e351bee 100644 --- a/libraries/shared/mocks/storage_repository.go +++ b/libraries/shared/mocks/storage_repository.go @@ -33,7 +33,6 @@ func (repository *MockStorageRepository) Create(blockNumber int, blockHash strin repository.PassedBlockNumber = blockNumber repository.PassedBlockHash = blockHash repository.PassedMetadata = metadata - repository.PassedValue = value return repository.CreateErr } diff --git a/libraries/shared/storage/utils/decoder.go b/libraries/shared/storage/utils/decoder.go index c672cd314..31e8caf15 100644 --- a/libraries/shared/storage/utils/decoder.go +++ b/libraries/shared/storage/utils/decoder.go @@ -18,8 +18,9 @@ package utils import ( "fmt" - "github.com/ethereum/go-ethereum/common" "math/big" + + "github.com/ethereum/go-ethereum/common" ) func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) { @@ -60,26 +61,30 @@ func decodeAddress(raw []byte) string { return common.BytesToAddress(raw).Hex() } -func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) map[int]string{ - storageSlot := raw - var results = map[int]string{} - - //the reason we're using a map and not a slice is that golang doesn't guarantee the order of a slice +func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) map[int]string { + storageSlotData := raw + decodedStorageSlotItems := map[int]string{} numberOfTypes := len(packedTypes) + for position := 0; position < numberOfTypes; position++ { - valueType := packedTypes[position] - lengthOfStorageSlot := len(storageSlot) - lengthOfItem := getNumberOfBytes(valueType) - itemStartingIndex := lengthOfStorageSlot - lengthOfItem - value := storageSlot[itemStartingIndex:] - decodedValue := decodeIndividualItems(value, valueType) - results[position] = decodedValue - - //pop last item off slot before moving on - storageSlot = storageSlot[0:itemStartingIndex] + //get length of remaining storage date + lengthOfStorageData := len(storageSlotData) + + //get item details (type, length, starting index, value bytes) + itemType := packedTypes[position] + lengthOfItem := getNumberOfBytes(itemType) + itemStartingIndex := lengthOfStorageData - lengthOfItem + itemValueBytes := storageSlotData[itemStartingIndex:] + + //decode item's bytes and set in results map + decodedValue := decodeIndividualItems(itemValueBytes, itemType) + decodedStorageSlotItems[position] = decodedValue + + //pop last item off raw slot data before moving on + storageSlotData = storageSlotData[0:itemStartingIndex] } - return results + return decodedStorageSlotItems } func decodeIndividualItems(itemBytes []byte, valueType ValueType) string { @@ -93,7 +98,7 @@ func decodeIndividualItems(itemBytes []byte, valueType ValueType) string { } } -func getNumberOfBytes(valueType ValueType) int{ +func getNumberOfBytes(valueType ValueType) int { // 8 bits per byte switch valueType { case Uint48: diff --git a/libraries/shared/storage/utils/decoder_test.go b/libraries/shared/storage/utils/decoder_test.go index 67df8d520..868828d65 100644 --- a/libraries/shared/storage/utils/decoder_test.go +++ b/libraries/shared/storage/utils/decoder_test.go @@ -17,10 +17,11 @@ package utils_test import ( + "math/big" + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "math/big" "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" ) diff --git a/libraries/shared/storage/utils/value.go b/libraries/shared/storage/utils/value.go index ba463ab13..36056255e 100644 --- a/libraries/shared/storage/utils/value.go +++ b/libraries/shared/storage/utils/value.go @@ -33,8 +33,8 @@ type StorageValueMetadata struct { Name string Keys map[Key]string Type ValueType - PackedNames map[int]string //name of each item packed and their order - PackedTypes map[int]ValueType //type of each item packed and their order + PackedNames map[int]string //position in map (zero indexed) => name of packed item + PackedTypes map[int]ValueType //position in map (zero indexed)=> type of packed item } func GetStorageValueMetadata(name string, keys map[Key]string, valueType ValueType) StorageValueMetadata { From 2c66afbcd7d2844982c68140c30f04ac0883cf71 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Tue, 23 Jul 2019 09:22:30 -0500 Subject: [PATCH 6/6] Address PR feedback - factor out bitsPerByte constant - panic if the necessary args aren't passed to GetStorageValueMetadataForPackedSlot - only have one decode integer method in the deocoder file --- libraries/shared/storage/utils/decoder.go | 31 ++++----- libraries/shared/storage/utils/value.go | 21 ++++-- libraries/shared/storage/utils/value_test.go | 67 ++++++++++++++++---- 3 files changed, 82 insertions(+), 37 deletions(-) diff --git a/libraries/shared/storage/utils/decoder.go b/libraries/shared/storage/utils/decoder.go index 31e8caf15..354f4b7d3 100644 --- a/libraries/shared/storage/utils/decoder.go +++ b/libraries/shared/storage/utils/decoder.go @@ -23,14 +23,18 @@ import ( "github.com/ethereum/go-ethereum/common" ) +const ( + bitsPerByte = 8 +) + func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) { switch metadata.Type { case Uint256: - return decodeUint256(row.StorageValue.Bytes()), nil + return decodeInteger(row.StorageValue.Bytes()), nil case Uint48: - return decodeUint48(row.StorageValue.Bytes()), nil + return decodeInteger(row.StorageValue.Bytes()), nil case Uint128: - return decodeUint128(row.StorageValue.Bytes()), nil + return decodeInteger(row.StorageValue.Bytes()), nil case Address: return decodeAddress(row.StorageValue.Bytes()), nil case Bytes32: @@ -42,17 +46,7 @@ func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, err } } -func decodeUint256(raw []byte) string { - n := big.NewInt(0).SetBytes(raw) - return n.String() -} - -func decodeUint128(raw []byte) string { - n := big.NewInt(0).SetBytes(raw) - return n.String() -} - -func decodeUint48(raw []byte) string { +func decodeInteger(raw []byte) string { n := big.NewInt(0).SetBytes(raw) return n.String() } @@ -90,21 +84,20 @@ func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) map[int]string func decodeIndividualItems(itemBytes []byte, valueType ValueType) string { switch valueType { case Uint48: - return decodeUint48(itemBytes) + return decodeInteger(itemBytes) case Uint128: - return decodeUint128(itemBytes) + return decodeInteger(itemBytes) default: panic(fmt.Sprintf("can't decode unknown type: %d", valueType)) } } func getNumberOfBytes(valueType ValueType) int { - // 8 bits per byte switch valueType { case Uint48: - return 48 / 8 + return 48 / bitsPerByte case Uint128: - return 128 / 8 + return 128 / bitsPerByte default: panic(fmt.Sprintf("ValueType %d not recognized", valueType)) } diff --git a/libraries/shared/storage/utils/value.go b/libraries/shared/storage/utils/value.go index 36056255e..c658b55fe 100644 --- a/libraries/shared/storage/utils/value.go +++ b/libraries/shared/storage/utils/value.go @@ -16,6 +16,8 @@ package utils +import "fmt" + type ValueType int const ( @@ -33,8 +35,8 @@ type StorageValueMetadata struct { Name string Keys map[Key]string Type ValueType - PackedNames map[int]string //position in map (zero indexed) => name of packed item - PackedTypes map[int]ValueType //position in map (zero indexed)=> type of packed item + PackedNames map[int]string //zero indexed position in map => name of packed item + PackedTypes map[int]ValueType //zero indexed position in map => type of packed item } func GetStorageValueMetadata(name string, keys map[Key]string, valueType ValueType) StorageValueMetadata { @@ -45,12 +47,23 @@ func GetStorageValueMetadataForPackedSlot(name string, keys map[Key]string, valu return getMetadata(name, keys, valueType, packedNames, packedTypes) } -func getMetadata(name string, keys map[Key]string, t ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata { +func getMetadata(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata { + assertPackedSlotArgs(valueType, packedNames, packedTypes) + return StorageValueMetadata{ Name: name, Keys: keys, - Type: t, + Type: valueType, PackedNames: packedNames, PackedTypes: packedTypes, } } + +func assertPackedSlotArgs(valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) { + if valueType == PackedSlot && (packedTypes == nil || packedNames == nil) { + panic(fmt.Sprintf("ValueType is PackedSlot. Expected PackedNames and PackedTypes to not be nil, but got PackedNames = %v and PackedTypes = %v", packedNames, packedTypes)) + } else if (packedNames != nil && packedTypes != nil) && valueType != PackedSlot { + panic(fmt.Sprintf("PackedNames and PackedTypes passed in. Expected ValueType to equal PackedSlot (%v), but got %v.", PackedSlot, valueType)) + } + +} diff --git a/libraries/shared/storage/utils/value_test.go b/libraries/shared/storage/utils/value_test.go index f07a8fae2..bcfdc3243 100644 --- a/libraries/shared/storage/utils/value_test.go +++ b/libraries/shared/storage/utils/value_test.go @@ -20,20 +20,59 @@ var _ = Describe("Storage value metadata getter", func() { Expect(utils.GetStorageValueMetadata(metadataName, metadataKeys, metadataType)).To(Equal(expectedMetadata)) }) - It("returns metadata for a packed storage slot variables", func() { - metadataName := "fake_name" - metadataKeys := map[utils.Key]string{"key": "value"} - metadataType := utils.Uint256 - metadataPackedNames := map[int]string{0: "name"} - metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48} + Describe("metadata for a packed storaged slot", func() { + It("returns metadata for multiple storage variables", func() { + metadataName := "fake_name" + metadataKeys := map[utils.Key]string{"key": "value"} + metadataType := utils.PackedSlot + metadataPackedNames := map[int]string{0: "name"} + metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48} - expectedMetadata := utils.StorageValueMetadata{ - Name: metadataName, - Keys: metadataKeys, - Type: metadataType, - PackedTypes: metadataPackedTypes, - PackedNames: metadataPackedNames, - } - Expect(utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)).To(Equal(expectedMetadata)) + expectedMetadata := utils.StorageValueMetadata{ + Name: metadataName, + Keys: metadataKeys, + Type: metadataType, + PackedTypes: metadataPackedTypes, + PackedNames: metadataPackedNames, + } + Expect(utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)).To(Equal(expectedMetadata)) + }) + + It("panics if PackedTypes are nil when the type is PackedSlot", func() { + metadataName := "fake_name" + metadataKeys := map[utils.Key]string{"key": "value"} + metadataType := utils.PackedSlot + metadataPackedNames := map[int]string{0: "name"} + + getMetadata := func() { + utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, nil) + } + Expect(getMetadata).To(Panic()) + }) + + It("panics if PackedNames are nil when the type is PackedSlot", func() { + metadataName := "fake_name" + metadataKeys := map[utils.Key]string{"key": "value"} + metadataType := utils.PackedSlot + metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48} + + getMetadata := func() { + utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, nil, metadataPackedTypes) + } + Expect(getMetadata).To(Panic()) + }) + + It("panics if valueType is not PackedSlot if PackedNames is populated", func() { + metadataName := "fake_name" + metadataKeys := map[utils.Key]string{"key": "value"} + metadataType := utils.Uint48 + metadataPackedNames := map[int]string{0: "name"} + metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48} + + getMetadata := func() { + utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes) + } + Expect(getMetadata).To(Panic()) + }) }) })