Merge pull request #399 from multiversx/feat/snapshotless-observer-su…

…pport

Feat/snapshotless observer support

api/groups/baseNetworkGroup.go

@@ -8,7 +8,6 @@ import (
 	"github.com/multiversx/mx-chain-proxy-go/api/errors"
 	"github.com/multiversx/mx-chain-proxy-go/api/shared"
 	"github.com/multiversx/mx-chain-proxy-go/data"
-	"github.com/multiversx/mx-chain-proxy-go/process"
 )
 
 type networkGroup struct {
@@ -55,7 +54,7 @@ func NewNetworkGroup(facadeHandler data.FacadeHandler) (*networkGroup, error) {
 func (group *networkGroup) getNetworkStatusData(c *gin.Context) {
 	shardIDUint, err := shared.FetchShardIDFromRequest(c)
 	if err != nil {
-		shared.RespondWith(c, http.StatusBadRequest, nil, process.ErrInvalidShardId.Error(), data.ReturnCodeRequestError)
+		shared.RespondWith(c, http.StatusBadRequest, nil, errors.ErrInvalidShardIDParam.Error(), data.ReturnCodeRequestError)
 		return
 	}
 
@@ -204,7 +203,7 @@ func (group *networkGroup) getGasConfigs(c *gin.Context) {
 func (group *networkGroup) getTrieStatistics(c *gin.Context) {
 	shardID, err := shared.FetchShardIDFromRequest(c)
 	if err != nil {
-		shared.RespondWith(c, http.StatusBadRequest, nil, process.ErrInvalidShardId.Error(), data.ReturnCodeRequestError)
+		shared.RespondWith(c, http.StatusBadRequest, nil, errors.ErrInvalidShardIDParam.Error(), data.ReturnCodeRequestError)
 		return
 	}
 

cmd/proxy/config/config.toml

@@ -65,9 +65,11 @@
 # List of Observers. If you want to define a metachain observer (needed for validator statistics route) use
 # shard id 4294967295
 # Fallback observers which are only used when regular ones are offline should have IsFallback = true
+# Snapshotless observers are observers that can only respond to real-time requests, such as vm queries. They should have IsSnapshotless = true
 [[Observers]]
    ShardId = 0
    Address = "http://127.0.0.1:8081"
+   IsSnapshotless = false
 
 [[Observers]]
    ShardId = 1
@@ -77,3 +79,4 @@
    ShardId = 4294967295
    Address = "http://127.0.0.1:8083"
    IsFallback = false
+

common/options.go

@@ -111,6 +111,15 @@ type AccountQueryOptions struct {
 	HintEpoch      core.OptionalUint32
 }
 
+// AreHistoricalCoordinatesSet returns true if historical block coordinates are set
+func (a AccountQueryOptions) AreHistoricalCoordinatesSet() bool {
+	return a.BlockNonce.HasValue ||
+		a.OnStartOfEpoch.HasValue ||
+		a.HintEpoch.HasValue ||
+		len(a.BlockHash) > 0 ||
+		len(a.BlockRootHash) > 0
+}
+
 // BuildUrlWithAccountQueryOptions builds an URL with block query parameters
 func BuildUrlWithAccountQueryOptions(path string, options AccountQueryOptions) string {
 	u := url.URL{Path: path}

common/options_test.go

@@ -9,6 +9,8 @@ import (
 )
 
 func TestBuildUrlWithBlockQueryOptions_ShouldWork(t *testing.T) {
+	t.Parallel()
+
 	builtUrl := BuildUrlWithBlockQueryOptions("/block/by-nonce/15", BlockQueryOptions{})
 	require.Equal(t, "/block/by-nonce/15", builtUrl)
 
@@ -29,6 +31,8 @@ func TestBuildUrlWithBlockQueryOptions_ShouldWork(t *testing.T) {
 }
 
 func TestBuildUrlWithAccountQueryOptions_ShouldWork(t *testing.T) {
+	t.Parallel()
+
 	builtUrl := BuildUrlWithAccountQueryOptions("/address/erd1alice", AccountQueryOptions{})
 	require.Equal(t, "/address/erd1alice", builtUrl)
 
@@ -65,6 +69,8 @@ func TestBuildUrlWithAccountQueryOptions_ShouldWork(t *testing.T) {
 }
 
 func TestBuildUrlWithAlteredAccountsQueryOptions(t *testing.T) {
+	t.Parallel()
+
 	resultedUrl := BuildUrlWithAlteredAccountsQueryOptions("path", GetAlteredAccountsForBlockOptions{})
 	require.Equal(t, "path", resultedUrl)
 
@@ -74,3 +80,35 @@ func TestBuildUrlWithAlteredAccountsQueryOptions(t *testing.T) {
 	// 2C is the ascii hex encoding of (,)
 	require.Equal(t, "path?tokens=token1%2Ctoken2%2Ctoken3", resultedUrl)
 }
+
+func TestAccountQueryOptions_AreHistoricalCoordinatesSet(t *testing.T) {
+	t.Parallel()
+
+	emptyQuery := AccountQueryOptions{}
+	require.False(t, emptyQuery.AreHistoricalCoordinatesSet())
+
+	queryWithNonce := AccountQueryOptions{
+		BlockNonce: core.OptionalUint64{HasValue: true, Value: 37},
+	}
+	require.True(t, queryWithNonce.AreHistoricalCoordinatesSet())
+
+	queryWithBlockHash := AccountQueryOptions{
+		BlockHash: []byte("hash"),
+	}
+	require.True(t, queryWithBlockHash.AreHistoricalCoordinatesSet())
+
+	queryWithBlockRootHash := AccountQueryOptions{
+		BlockRootHash: []byte("rootHash"),
+	}
+	require.True(t, queryWithBlockRootHash.AreHistoricalCoordinatesSet())
+
+	queryWithEpochStart := AccountQueryOptions{
+		OnStartOfEpoch: core.OptionalUint32{HasValue: true, Value: 37},
+	}
+	require.True(t, queryWithEpochStart.AreHistoricalCoordinatesSet())
+
+	queryWithHintEpoch := AccountQueryOptions{
+		HintEpoch: core.OptionalUint32{HasValue: true, Value: 37},
+	}
+	require.True(t, queryWithHintEpoch.AreHistoricalCoordinatesSet())
+}

data/observer.go

@@ -2,10 +2,11 @@ package data
 
 // NodeData holds an observer data
 type NodeData struct {
-	ShardId    uint32
-	Address    string
-	IsSynced   bool
-	IsFallback bool
+	ShardId        uint32
+	Address        string
+	IsSynced       bool
+	IsFallback     bool
+	IsSnapshotless bool
 }
 
 // NodesReloadResponse is a DTO that holds details about nodes reloading
@@ -25,3 +26,14 @@ const (
 	// FullHistoryNode identifier a node that has full history mode enabled
 	FullHistoryNode NodeType = "full history"
 )
+
+// ObserverDataAvailabilityType represents the type to be used for the observers' data availability
+type ObserverDataAvailabilityType string
+
+const (
+	// AvailabilityAll mean that the observer can be used for both real-time and historical requests
+	AvailabilityAll ObserverDataAvailabilityType = "all"
+
+	// AvailabilityRecent means that the observer can be used only for recent data
+	AvailabilityRecent ObserverDataAvailabilityType = "recent"
+)

observer/availabilityCommon/availabilityProvider.go

@@ -0,0 +1,53 @@
+package availabilityCommon
+
+import (
+	"github.com/multiversx/mx-chain-proxy-go/common"
+	"github.com/multiversx/mx-chain-proxy-go/data"
+)
+
+// AvailabilityProvider is a stateless component that aims to group common operations regarding observers' data availability
+type AvailabilityProvider struct {
+}
+
+// AvailabilityForAccountQueryOptions returns the availability needed for the provided query options
+func (ap *AvailabilityProvider) AvailabilityForAccountQueryOptions(options common.AccountQueryOptions) data.ObserverDataAvailabilityType {
+	availability := data.AvailabilityRecent
+	if options.AreHistoricalCoordinatesSet() {
+		availability = data.AvailabilityAll
+	}
+	return availability
+}
+
+// AvailabilityForVmQuery returns the availability needed for the provided query options
+func (ap *AvailabilityProvider) AvailabilityForVmQuery(query *data.SCQuery) data.ObserverDataAvailabilityType {
+	availability := data.AvailabilityRecent
+	if query.BlockNonce.HasValue || len(query.BlockHash) > 0 {
+		availability = data.AvailabilityAll
+	}
+	return availability
+}
+
+// IsNodeValid returns true if the provided node is valid based on the availability
+func (ap *AvailabilityProvider) IsNodeValid(node *data.NodeData, availability data.ObserverDataAvailabilityType) bool {
+	isInvalidSnapshotlessNode := availability == data.AvailabilityRecent && !node.IsSnapshotless
+	isInvalidRegularNode := availability == data.AvailabilityAll && node.IsSnapshotless
+	isInvalidNode := isInvalidSnapshotlessNode || isInvalidRegularNode
+	return !isInvalidNode
+}
+
+// GetDescriptionForAvailability returns a short description string about the provided availability
+func (ap *AvailabilityProvider) GetDescriptionForAvailability(availability data.ObserverDataAvailabilityType) string {
+	switch availability {
+	case data.AvailabilityAll:
+		return "regular nodes"
+	case data.AvailabilityRecent:
+		return "snapshotless nodes"
+	default:
+		return "N/A"
+	}
+}
+
+// GetAllAvailabilityTypes returns all data availability types
+func (ap *AvailabilityProvider) GetAllAvailabilityTypes() []data.ObserverDataAvailabilityType {
+	return []data.ObserverDataAvailabilityType{data.AvailabilityAll, data.AvailabilityRecent}
+}

observer/availabilityCommon/availabilityProvider_test.go

@@ -0,0 +1,81 @@
+package availabilityCommon
+
+import (
+	"testing"
+
+	"github.com/multiversx/mx-chain-core-go/core"
+	"github.com/multiversx/mx-chain-proxy-go/common"
+	"github.com/multiversx/mx-chain-proxy-go/data"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAvailabilityForAccountQueryOptions(t *testing.T) {
+	t.Parallel()
+
+	ap := &AvailabilityProvider{}
+
+	// Test with historical coordinates set
+	options := common.AccountQueryOptions{BlockHash: []byte("hash")}
+	require.Equal(t, data.AvailabilityAll, ap.AvailabilityForAccountQueryOptions(options))
+
+	// Test without historical coordinates set
+	options = common.AccountQueryOptions{}
+	require.Equal(t, data.AvailabilityRecent, ap.AvailabilityForAccountQueryOptions(options))
+}
+
+func TestAvailabilityForVmQuery(t *testing.T) {
+	t.Parallel()
+
+	ap := &AvailabilityProvider{}
+
+	// Test with BlockNonce set
+	query := &data.SCQuery{BlockNonce: core.OptionalUint64{HasValue: true, Value: 37}}
+	require.Equal(t, data.AvailabilityAll, ap.AvailabilityForVmQuery(query))
+
+	// Test without BlockNonce set but with BlockHash
+	query = &data.SCQuery{BlockHash: []byte("hash")}
+	require.Equal(t, data.AvailabilityAll, ap.AvailabilityForVmQuery(query))
+
+	// Test without BlockNonce and BlockHash
+	query = &data.SCQuery{}
+	require.Equal(t, data.AvailabilityRecent, ap.AvailabilityForVmQuery(query))
+}
+
+func TestIsNodeValid(t *testing.T) {
+	t.Parallel()
+
+	ap := &AvailabilityProvider{}
+
+	// Test with AvailabilityRecent and snapshotless node
+	node := &data.NodeData{IsSnapshotless: true}
+	require.True(t, ap.IsNodeValid(node, data.AvailabilityRecent))
+
+	// Test with AvailabilityRecent and regular node
+	node = &data.NodeData{}
+	require.False(t, ap.IsNodeValid(node, data.AvailabilityRecent))
+
+	// Test with AvailabilityAll and regular node
+	node = &data.NodeData{}
+	require.True(t, ap.IsNodeValid(node, data.AvailabilityAll))
+
+	// Test with AvailabilityAll and Snapshotless node
+	node = &data.NodeData{IsSnapshotless: true}
+	require.False(t, ap.IsNodeValid(node, data.AvailabilityAll))
+}
+
+func TestGetDescriptionForAvailability(t *testing.T) {
+	t.Parallel()
+
+	ap := &AvailabilityProvider{}
+
+	require.Equal(t, "regular nodes", ap.GetDescriptionForAvailability(data.AvailabilityAll))
+	require.Equal(t, "snapshotless nodes", ap.GetDescriptionForAvailability(data.AvailabilityRecent))
+	require.Equal(t, "N/A", ap.GetDescriptionForAvailability("invalid")) // Invalid value
+}
+
+func TestAvailabilityProvider_GetAllAvailabilityTypes(t *testing.T) {
+	t.Parallel()
+
+	ap := &AvailabilityProvider{}
+	require.Equal(t, []data.ObserverDataAvailabilityType{data.AvailabilityAll, data.AvailabilityRecent}, ap.GetAllAvailabilityTypes())
+}