Merge pull request #5430 from multiversx/intra-shard-mbs-in-outport-b…

…lock

Intra-shard miniblocks in outport

go.mod

@@ -14,9 +14,9 @@ require (
 	github.com/gorilla/websocket v1.5.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/multiversx/mx-chain-communication-go v1.0.4
-	github.com/multiversx/mx-chain-core-go v1.2.9
+	github.com/multiversx/mx-chain-core-go v1.2.11
 	github.com/multiversx/mx-chain-crypto-go v1.2.7
-	github.com/multiversx/mx-chain-es-indexer-go v1.4.6
+	github.com/multiversx/mx-chain-es-indexer-go v1.4.7
 	github.com/multiversx/mx-chain-logger-go v1.0.11
 	github.com/multiversx/mx-chain-scenario-go v1.1.2
 	github.com/multiversx/mx-chain-storage-go v1.0.11

go.sum

@@ -628,13 +628,13 @@ github.com/multiversx/mx-chain-core-go v1.1.30/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZ
 github.com/multiversx/mx-chain-core-go v1.2.1/go.mod h1:8gGEQv6BWuuJwhd25qqhCOZbBSv9mk+hLeKvinSaSMk=
 github.com/multiversx/mx-chain-core-go v1.2.5/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ=
 github.com/multiversx/mx-chain-core-go v1.2.6/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ=
-github.com/multiversx/mx-chain-core-go v1.2.9 h1:uXrmzhKxEe5sztUTs1am3ILrxuBwNQQOESOKBlyrlLE=
-github.com/multiversx/mx-chain-core-go v1.2.9/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ=
+github.com/multiversx/mx-chain-core-go v1.2.11 h1:C5W6P696SwTb8H0wvCUGX/+Ix5hhF4iDAmFXnDM0Jdo=
+github.com/multiversx/mx-chain-core-go v1.2.11/go.mod h1:jzYFSiYBuO0dGpGFXnZWSwcwcKP7Flyn/X41y4zIQrQ=
 github.com/multiversx/mx-chain-crypto-go v1.2.6/go.mod h1:rOj0Rr19HTOYt9YTeym7RKxlHt91NXln3LVKjHKVmA0=
 github.com/multiversx/mx-chain-crypto-go v1.2.7 h1:7OAINQ+zmnuUelJ2G1ETt5vLtBHn46tXmQO+hFXRrmw=
 github.com/multiversx/mx-chain-crypto-go v1.2.7/go.mod h1:YVTxFG1OwtJW3jGOuRewvJKqyqrUzXzR5p/7/+jKlYM=
-github.com/multiversx/mx-chain-es-indexer-go v1.4.6 h1:H49T2Swdk1aGgNBMbpApzx3pmOvqEJkdaxeZfzzTD1k=
-github.com/multiversx/mx-chain-es-indexer-go v1.4.6/go.mod h1:RPQP9B54CYpiqpnlwRlhEUedArc8vPCeyC3iVB2UbZY=
+github.com/multiversx/mx-chain-es-indexer-go v1.4.7 h1:G7EKyJB0NE4hGiIFh5GHglW/Iyh9aMLKUWuKXKUCPso=
+github.com/multiversx/mx-chain-es-indexer-go v1.4.7/go.mod h1:35HNdhfFbQogZ4tm0MdqobDsgM9uLdRpLJC2YzXujUs=
 github.com/multiversx/mx-chain-logger-go v1.0.11 h1:DFsHa+sc5fKwhDR50I8uBM99RTDTEW68ESyr5ALRDwE=
 github.com/multiversx/mx-chain-logger-go v1.0.11/go.mod h1:1srDkP0DQucWQ+rYfaq0BX2qLnULsUdRPADpYUTM6dA=
 github.com/multiversx/mx-chain-scenario-go v1.1.2 h1:6Mok/W48r5lBlqJfriFFhDO5UiYUU1Em+0n0oB0NNGs=

outport/outport.go

@@ -85,10 +85,11 @@ func prepareBlockData(
 	}
 
 	return &outportcore.BlockData{
-		HeaderBytes: headerBytes,
-		HeaderType:  string(headerType),
-		HeaderHash:  headerBodyData.HeaderHash,
-		Body:        body,
+		HeaderBytes:          headerBytes,
+		HeaderType:           string(headerType),
+		HeaderHash:           headerBodyData.HeaderHash,
+		Body:                 body,
+		IntraShardMiniBlocks: headerBodyData.IntraShardMiniBlocks,
 	}, nil
 }
 

outport/process/factory/outportDataProviderFactory.go

@@ -91,5 +91,7 @@ func CreateOutportDataProvider(arg ArgOutportDataProviderFactory) (outport.DataP
 		GasConsumedProvider:      arg.GasConsumedProvider,
 		EconomicsData:            arg.EconomicsData,
 		ExecutionOrderHandler:    executionOrderHandler,
+		Hasher:                   arg.Hasher,
+		Marshaller:               arg.Marshaller,
 	})
 }

outport/process/outportDataProvider.go

@@ -14,6 +14,7 @@ import (
 	"github.com/multiversx/mx-chain-core-go/data/rewardTx"
 	"github.com/multiversx/mx-chain-core-go/data/smartContractResult"
 	"github.com/multiversx/mx-chain-core-go/data/transaction"
+	"github.com/multiversx/mx-chain-core-go/hashing"
 	"github.com/multiversx/mx-chain-core-go/marshal"
 	"github.com/multiversx/mx-chain-go/outport/process/alteredaccounts/shared"
 	"github.com/multiversx/mx-chain-go/process"
@@ -33,6 +34,7 @@ type ArgOutportDataProvider struct {
 	EconomicsData            EconomicsDataHandler
 	ExecutionOrderHandler    ExecutionOrderHandler
 	Marshaller               marshal.Marshalizer
+	Hasher                   hashing.Hasher
 }
 
 // ArgPrepareOutportSaveBlockData holds the arguments needed for prepare outport save block data
@@ -60,6 +62,7 @@ type outportDataProvider struct {
 	economicsData            EconomicsDataHandler
 	executionOrderHandler    ExecutionOrderHandler
 	marshaller               marshal.Marshalizer
+	hasher                   hashing.Hasher
 }
 
 // NewOutportDataProvider will create a new instance of outportDataProvider
@@ -75,6 +78,7 @@ func NewOutportDataProvider(arg ArgOutportDataProvider) (*outportDataProvider, e
 		economicsData:            arg.EconomicsData,
 		executionOrderHandler:    arg.ExecutionOrderHandler,
 		marshaller:               arg.Marshaller,
+		hasher:                   arg.Hasher,
 	}, nil
 }
 
@@ -117,6 +121,11 @@ func (odp *outportDataProvider) PrepareOutportSaveBlockData(arg ArgPrepareOutpor
 		return nil, err
 	}
 
+	intraMiniBlocks, err := odp.getIntraShardMiniBlocks(arg.Body)
+	if err != nil {
+		return nil, err
+	}
+
 	return &outportcore.OutportBlockWithHeaderAndBody{
 		OutportBlock: &outportcore.OutportBlock{
 			BlockData:       nil, // this will be filled with specific data for each driver
@@ -136,9 +145,10 @@ func (odp *outportDataProvider) PrepareOutportSaveBlockData(arg ArgPrepareOutpor
 			HighestFinalBlockHash:  arg.HighestFinalBlockHash,
 		},
 		HeaderDataWithBody: &outportcore.HeaderDataWithBody{
-			Body:       arg.Body,
-			Header:     arg.Header,
-			HeaderHash: arg.HeaderHash,
+			Body:                 arg.Body,
+			Header:               arg.Header,
+			HeaderHash:           arg.HeaderHash,
+			IntraShardMiniBlocks: intraMiniBlocks,
 		},
 	}, nil
 }
@@ -358,3 +368,39 @@ func getLogs(logs []*data.LogData) ([]*outportcore.LogData, error) {
 func (odp *outportDataProvider) IsInterfaceNil() bool {
 	return odp == nil
 }
+
+func (odp *outportDataProvider) getIntraShardMiniBlocks(bodyHandler data.BodyHandler) ([]*block.MiniBlock, error) {
+	body, err := outportcore.GetBody(bodyHandler)
+	if err != nil {
+		return nil, err
+	}
+
+	return odp.filterOutDuplicatedMiniBlocks(body.MiniBlocks, odp.txCoordinator.GetCreatedInShardMiniBlocks())
+}
+
+func (odp *outportDataProvider) filterOutDuplicatedMiniBlocks(miniBlocksFromBody []*block.MiniBlock, intraMiniBlocks []*block.MiniBlock) ([]*block.MiniBlock, error) {
+	filteredMiniBlocks := make([]*block.MiniBlock, 0, len(intraMiniBlocks))
+	mapMiniBlocksFromBody := make(map[string]struct{})
+	for _, mb := range miniBlocksFromBody {
+		mbHash, err := core.CalculateHash(odp.marshaller, odp.hasher, mb)
+		if err != nil {
+			return nil, err
+		}
+		mapMiniBlocksFromBody[string(mbHash)] = struct{}{}
+	}
+
+	for _, mb := range intraMiniBlocks {
+		mbHash, err := core.CalculateHash(odp.marshaller, odp.hasher, mb)
+		if err != nil {
+			return nil, err
+		}
+
+		_, found := mapMiniBlocksFromBody[string(mbHash)]
+		if found {
+			continue
+		}
+		filteredMiniBlocks = append(filteredMiniBlocks, mb)
+	}
+
+	return filteredMiniBlocks, nil
+}

outport/process/outportDataProvider_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/multiversx/mx-chain-go/outport/process/transactionsfee"
 	"github.com/multiversx/mx-chain-go/testscommon"
 	"github.com/multiversx/mx-chain-go/testscommon/genericMocks"
+	"github.com/multiversx/mx-chain-go/testscommon/hashingMocks"
 	"github.com/multiversx/mx-chain-go/testscommon/marshallerMock"
 	"github.com/multiversx/mx-chain-go/testscommon/shardingMocks"
 	"github.com/stretchr/testify/require"
@@ -31,6 +32,7 @@ func createArgOutportDataProvider() ArgOutportDataProvider {
 		ShardCoordinator:         &testscommon.ShardsCoordinatorMock{},
 		ExecutionOrderHandler:    &mock.ExecutionOrderHandlerStub{},
 		Marshaller:               &marshallerMock.MarshalizerMock{},
+		Hasher:                   &hashingMocks.HasherMock{},
 	}
 }
 
@@ -93,3 +95,48 @@ func TestPrepareOutportSaveBlockData(t *testing.T) {
 	require.NotNil(t, res.HeaderGasConsumption)
 	require.NotNil(t, res.TransactionPool)
 }
+
+func TestOutportDataProvider_GetIntraShardMiniBlocks(t *testing.T) {
+	t.Parallel()
+
+	mb1 := &block.MiniBlock{
+		Type:     block.SmartContractResultBlock,
+		TxHashes: [][]byte{[]byte("scr1")},
+	}
+	mb2 := &block.MiniBlock{
+		SenderShardID:   0,
+		ReceiverShardID: 1,
+		Type:            block.SmartContractResultBlock,
+		TxHashes:        [][]byte{[]byte("scr2"), []byte("scr3")},
+	}
+	mb3 := &block.MiniBlock{
+		Type:     block.SmartContractResultBlock,
+		TxHashes: [][]byte{[]byte("scr4"), []byte("scr5")},
+	}
+
+	arg := createArgOutportDataProvider()
+	arg.NodesCoordinator = &shardingMocks.NodesCoordinatorMock{
+		GetValidatorsPublicKeysCalled: func(randomness []byte, round uint64, shardId uint32, epoch uint32) ([]string, error) {
+			return nil, nil
+		},
+		GetValidatorsIndexesCalled: func(publicKeys []string, epoch uint32) ([]uint64, error) {
+			return []uint64{0, 1}, nil
+		},
+	}
+	arg.TxCoordinator = &testscommon.TransactionCoordinatorMock{
+		GetCreatedInShardMiniBlocksCalled: func() []*block.MiniBlock {
+			return []*block.MiniBlock{mb1, mb3}
+		},
+	}
+	outportDataP, _ := NewOutportDataProvider(arg)
+
+	res, err := outportDataP.PrepareOutportSaveBlockData(ArgPrepareOutportSaveBlockData{
+		Header: &block.Header{},
+		Body: &block.Body{
+			MiniBlocks: []*block.MiniBlock{mb1, mb2},
+		},
+		HeaderHash: []byte("something"),
+	})
+	require.Nil(t, err)
+	require.Equal(t, []*block.MiniBlock{mb3}, res.HeaderDataWithBody.IntraShardMiniBlocks)
+}