Merge pull request #5707 from multiversx/prometheus_dashboards

Added new flag + readme for starting p2p prometheus dashboards

README.md

@@ -83,6 +83,36 @@ sudo cp protoc-gen-gogoslick /usr/bin/
 
 Done
 
+## Running p2p Prometheus dashboards
+1. Start the node with `--p2p-prometheus-metrics` flag. This exposes a metrics collection at http://localhost:8080/debug/metrics/prometheus (port defined by -rest-api-interface flag, default 8080)
+2. Clone libp2p repository: `git clone https://github.com/libp2p/go-libp2p`
+3. `cd go-libp2p/dasboards/swarm` and under the 
+```  
+"templating": {
+   "list": [
+```
+section, add the following lines:
+```
+{
+  "hide": 0,
+  "label": "datasource",
+  "name": "DS_PROMETHEUS",
+  "options": [],
+  "query": "prometheus",
+  "refresh": 1,
+  "regex": "",
+  "type": "datasource"
+},
+```
+(this step will be removed once it will be fixed on libp2p)
+4. `cd ..` to dashboards directory and update the port of `host.docker.internal` from `prometheus.yml` to node's Rest API port(default `8080`)
+5. From this directory, run the following docker compose command: 
+```
+sudo docker compose -f docker-compose.base.yml -f docker-compose-linux.yml up --force-recreate
+```
+**Note:** If you choose to install the new Docker version manually, please make sure that installation is done for all users of the system. Otherwise, the docker command will fail because it needs the super-user privileges.
+6. The preconfigured dashboards should be now available on Grafana at http://localhost:3000/dashboards
+
 ## Progress
 
 ### Done

api/gin/common_test.go

@@ -22,7 +22,12 @@ func TestCommon_checkArgs(t *testing.T) {
 	err := checkArgs(args)
 	require.True(t, errors.Is(err, apiErrors.ErrCannotCreateGinWebServer))
 
-	args.Facade, err = initial.NewInitialNodeFacade("api interface", false, &testscommon.StatusMetricsStub{})
+	args.Facade, err = initial.NewInitialNodeFacade(initial.ArgInitialNodeFacade{
+		ApiInterface:                "api interface",
+		PprofEnabled:                false,
+		P2PPrometheusMetricsEnabled: false,
+		StatusMetricsHandler:        &testscommon.StatusMetricsStub{},
+	})
 	require.NoError(t, err)
 	err = checkArgs(args)
 	require.NoError(t, err)

api/gin/webServer.go

@@ -19,10 +19,13 @@ import (
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/facade"
 	logger "github.com/multiversx/mx-chain-logger-go"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var log = logger.GetOrCreate("api/gin")
 
+const prometheusMetricsRoute = "/debug/metrics/prometheus"
+
 // ArgsNewWebServer holds the arguments needed to create a new instance of webServer
 type ArgsNewWebServer struct {
 	Facade          shared.FacadeHandler
@@ -227,6 +230,10 @@ func (ws *webServer) registerRoutes(ginRouter *gin.Engine) {
 	if ws.facade.PprofEnabled() {
 		pprof.Register(ginRouter)
 	}
+
+	if ws.facade.P2PPrometheusMetricsEnabled() {
+		ginRouter.GET(prometheusMetricsRoute, gin.WrapH(promhttp.Handler()))
+	}
 }
 
 func (ws *webServer) createMiddlewareLimiters() ([]shared.MiddlewareProcessor, error) {

api/mock/facadeStub.go

@@ -93,6 +93,7 @@ type FacadeStub struct {
 	GetManagedKeysCalled                        func() []string
 	GetEligibleManagedKeysCalled                func() ([]string, error)
 	GetWaitingManagedKeysCalled                 func() ([]string, error)
+	P2PPrometheusMetricsEnabledCalled           func() bool
 }
 
 // GetTokenSupply -
@@ -610,6 +611,14 @@ func (f *FacadeStub) GetWaitingManagedKeys() ([]string, error) {
 	return make([]string, 0), nil
 }
 
+// P2PPrometheusMetricsEnabled -
+func (f *FacadeStub) P2PPrometheusMetricsEnabled() bool {
+	if f.P2PPrometheusMetricsEnabledCalled != nil {
+		return f.P2PPrometheusMetricsEnabledCalled()
+	}
+	return false
+}
+
 // Close -
 func (f *FacadeStub) Close() error {
 	return nil

api/shared/interface.go

@@ -132,5 +132,6 @@ type FacadeHandler interface {
 	GetManagedKeys() []string
 	GetEligibleManagedKeys() ([]string, error)
 	GetWaitingManagedKeys() ([]string, error)
+	P2PPrometheusMetricsEnabled() bool
 	IsInterfaceNil() bool
 }

cmd/node/CLI.md

@@ -73,6 +73,7 @@ GLOBAL OPTIONS:
    --logs-path directory                     This flag specifies the directory where the node will store logs.
    --operation-mode operation mode           String flag for specifying the desired operation mode(s) of the node, resulting in altering some configuration values accordingly. Possible values are: snapshotless-observer, full-archive, db-lookup-extension, historical-balances or `""` (empty). Multiple values can be separated via ,
    --repopulate-tokens-supplies              Boolean flag for repopulating the tokens supplies database. It will delete the current data, iterate over the entire trie and add he new obtained supplies
+   --p2p-prometheus-metrics                  Boolean option for enabling the /debug/metrics/prometheus route for p2p prometheus metrics
    --help, -h                                show help
    --version, -v                             print the version
    

cmd/node/flags.go

@@ -407,6 +407,13 @@ var (
 		Name:  "repopulate-tokens-supplies",
 		Usage: "Boolean flag for repopulating the tokens supplies database. It will delete the current data, iterate over the entire trie and add he new obtained supplies",
 	}
+
+	// p2pPrometheusMetrics defines a flag for p2p prometheus metrics
+	// If enabled, it will open a new route, /debug/metrics/prometheus, where p2p prometheus metrics will be available
+	p2pPrometheusMetrics = cli.BoolFlag{
+		Name:  "p2p-prometheus-metrics",
+		Usage: "Boolean option for enabling the /debug/metrics/prometheus route for p2p prometheus metrics",
+	}
 )
 
 func getFlags() []cli.Flag {
@@ -469,6 +476,7 @@ func getFlags() []cli.Flag {
 		logsDirectory,
 		operationMode,
 		repopulateTokensSupplies,
+		p2pPrometheusMetrics,
 	}
 }
 
@@ -497,6 +505,7 @@ func getFlagsConfig(ctx *cli.Context, log logger.Logger) *config.ContextFlagsCon
 	flagsConfig.SerializeSnapshots = ctx.GlobalBool(serializeSnapshots.Name)
 	flagsConfig.OperationMode = ctx.GlobalString(operationMode.Name)
 	flagsConfig.RepopulateTokensSupplies = ctx.GlobalBool(repopulateTokensSupplies.Name)
+	flagsConfig.P2PPrometheusMetricsEnabled = ctx.GlobalBool(p2pPrometheusMetrics.Name)
 
 	if ctx.GlobalBool(noKey.Name) {
 		log.Warn("the provided -no-key option is deprecated and will soon be removed. To start a node without " +

cmd/seednode/CLI.md

@@ -21,6 +21,7 @@ GLOBAL OPTIONS:
    --log-save                             Boolean option for enabling log saving. If set, it will automatically save all the logs into a file.
    --config [path]                        The [path] for the main configuration file. This TOML file contain the main configurations such as the marshalizer type (default: "./config/config.toml")
    --p2p-key-pem-file filepath            The filepath for the PEM file which contains the secret keys for the p2p key. If this is not specified a new key will be generated (internally) by default. (default: "./config/p2pKey.pem")
+   --p2p-prometheus-metrics               Boolean option for enabling the /debug/metrics/prometheus route for p2p prometheus metrics
    --help, -h                             show help
    --version, -v                          print the version
    

cmd/seednode/api/api.go

@@ -9,25 +9,26 @@ import (
 	"github.com/multiversx/mx-chain-core-go/marshal"
 	"github.com/multiversx/mx-chain-go/api/logs"
 	logger "github.com/multiversx/mx-chain-logger-go"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var log = logger.GetOrCreate("seednode/api")
 
 // Start will boot up the api and appropriate routes, handlers and validators
-func Start(restApiInterface string, marshalizer marshal.Marshalizer) error {
+func Start(restApiInterface string, marshalizer marshal.Marshalizer, p2pPrometheusMetricsEnabled bool) error {
 	ws := gin.Default()
 	ws.Use(cors.Default())
 
-	registerRoutes(ws, marshalizer)
+	registerRoutes(ws, marshalizer, p2pPrometheusMetricsEnabled)
 
 	return ws.Run(restApiInterface)
 }
 
-func registerRoutes(ws *gin.Engine, marshalizer marshal.Marshalizer) {
-	registerLoggerWsRoute(ws, marshalizer)
+func registerRoutes(ws *gin.Engine, marshalizer marshal.Marshalizer, p2pPrometheusMetricsEnabled bool) {
+	registerLoggerWsRoute(ws, marshalizer, p2pPrometheusMetricsEnabled)
 }
 
-func registerLoggerWsRoute(ws *gin.Engine, marshalizer marshal.Marshalizer) {
+func registerLoggerWsRoute(ws *gin.Engine, marshalizer marshal.Marshalizer, p2pPrometheusMetricsEnabled bool) {
 	upgrader := websocket.Upgrader{}
 
 	ws.GET("/log", func(c *gin.Context) {
@@ -49,4 +50,8 @@ func registerLoggerWsRoute(ws *gin.Engine, marshalizer marshal.Marshalizer) {
 
 		ls.StartSendingBlocking()
 	})
+
+	if p2pPrometheusMetricsEnabled {
+		ws.GET("/debug/metrics/prometheus", gin.WrapH(promhttp.Handler()))
+	}
 }

cmd/seednode/main.go

@@ -98,6 +98,13 @@ VERSION:
 	}
 
 	p2pConfigurationFile = "./config/p2p.toml"
+
+	// p2pPrometheusMetrics defines a flag for p2p prometheus metrics
+	// If enabled, it will open a new route, /debug/metrics/prometheus, where p2p prometheus metrics will be available
+	p2pPrometheusMetrics = cli.BoolFlag{
+		Name:  "p2p-prometheus-metrics",
+		Usage: "Boolean option for enabling the /debug/metrics/prometheus route for p2p prometheus metrics",
+	}
 )
 
 var log = logger.GetOrCreate("main")
@@ -114,6 +121,7 @@ func main() {
 		logSaveFile,
 		configurationFile,
 		p2pKeyPemFile,
+		p2pPrometheusMetrics,
 	}
 	app.Version = "v0.0.1"
 	app.Authors = []cli.Author{
@@ -350,14 +358,15 @@ func checkExpectedPeerCount(p2pConfig p2pConfig.P2PConfig) error {
 func startRestServices(ctx *cli.Context, marshalizer marshal.Marshalizer) {
 	restApiInterface := ctx.GlobalString(restApiInterfaceFlag.Name)
 	if restApiInterface != facade.DefaultRestPortOff {
-		go startGinServer(restApiInterface, marshalizer)
+		p2pPrometheusMetricsEnabled := ctx.GlobalBool(p2pPrometheusMetrics.Name)
+		go startGinServer(restApiInterface, marshalizer, p2pPrometheusMetricsEnabled)
 	} else {
 		log.Info("rest api is disabled")
 	}
 }
 
-func startGinServer(restApiInterface string, marshalizer marshal.Marshalizer) {
-	err := api.Start(restApiInterface, marshalizer)
+func startGinServer(restApiInterface string, marshalizer marshal.Marshalizer, p2pPrometheusMetricsEnabled bool) {
+	err := api.Start(restApiInterface, marshalizer, p2pPrometheusMetricsEnabled)
 	if err != nil {
 		log.LogIfError(err)
 	}

config/config.go

@@ -285,8 +285,9 @@ type GeneralSettingsConfig struct {
 
 // FacadeConfig will hold different configuration option that will be passed to the node facade
 type FacadeConfig struct {
-	RestApiInterface string
-	PprofEnabled     bool
+	RestApiInterface            string
+	PprofEnabled                bool
+	P2PPrometheusMetricsEnabled bool
 }
 
 // StateTriesConfig will hold information about state tries

config/contextFlagsConfig.go

@@ -27,6 +27,7 @@ type ContextFlagsConfig struct {
 	SerializeSnapshots           bool
 	OperationMode                string
 	RepopulateTokensSupplies     bool
+	P2PPrometheusMetricsEnabled  bool
 }
 
 // ImportDbConfig will hold the import-db parameters

facade/initial/initialNodeFacade.go

@@ -26,28 +26,38 @@ import (
 var errNodeStarting = errors.New("node is starting")
 var emptyString = ""
 
+// ArgInitialNodeFacade is the DTO used to create a new instance of initialNodeFacade
+type ArgInitialNodeFacade struct {
+	ApiInterface                string
+	PprofEnabled                bool
+	P2PPrometheusMetricsEnabled bool
+	StatusMetricsHandler        external.StatusMetricsHandler
+}
+
 // initialNodeFacade represents a facade with no functionality
 type initialNodeFacade struct {
-	apiInterface         string
-	statusMetricsHandler external.StatusMetricsHandler
-	pprofEnabled         bool
+	apiInterface                string
+	statusMetricsHandler        external.StatusMetricsHandler
+	pprofEnabled                bool
+	p2pPrometheusMetricsEnabled bool
 }
 
 // NewInitialNodeFacade is the initial implementation of the facade interface
-func NewInitialNodeFacade(apiInterface string, pprofEnabled bool, statusMetricsHandler external.StatusMetricsHandler) (*initialNodeFacade, error) {
-	if check.IfNil(statusMetricsHandler) {
+func NewInitialNodeFacade(args ArgInitialNodeFacade) (*initialNodeFacade, error) {
+	if check.IfNil(args.StatusMetricsHandler) {
 		return nil, facade.ErrNilStatusMetrics
 	}
 
-	initialStatusMetrics, err := NewInitialStatusMetricsProvider(statusMetricsHandler)
+	initialStatusMetrics, err := NewInitialStatusMetricsProvider(args.StatusMetricsHandler)
 	if err != nil {
 		return nil, err
 	}
 
 	return &initialNodeFacade{
-		apiInterface:         apiInterface,
-		statusMetricsHandler: initialStatusMetrics,
-		pprofEnabled:         pprofEnabled,
+		apiInterface:                args.ApiInterface,
+		statusMetricsHandler:        initialStatusMetrics,
+		pprofEnabled:                args.PprofEnabled,
+		p2pPrometheusMetricsEnabled: args.P2PPrometheusMetricsEnabled,
 	}, nil
 }
 
@@ -76,7 +86,7 @@ func (inf *initialNodeFacade) SetSyncer(_ ntp.SyncTimer) {
 }
 
 // RestAPIServerDebugMode returns false
-//TODO: remove in the future
+// TODO: remove in the future
 func (inf *initialNodeFacade) RestAPIServerDebugMode() bool {
 	return false
 }
@@ -426,6 +436,11 @@ func (inf *initialNodeFacade) GetWaitingManagedKeys() ([]string, error) {
 	return nil, errNodeStarting
 }
 
+// P2PPrometheusMetricsEnabled returns either the p2p prometheus metrics are enabled or not
+func (inf *initialNodeFacade) P2PPrometheusMetricsEnabled() bool {
+	return inf.p2pPrometheusMetricsEnabled
+}
+
 // IsInterfaceNil returns true if there is no value under the interface
 func (inf *initialNodeFacade) IsInterfaceNil() bool {
 	return inf == nil

facade/initial/initialNodeFacade_test.go

@@ -11,20 +11,31 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func createInitialNodeFacadeArgs() ArgInitialNodeFacade {
+	return ArgInitialNodeFacade{
+		ApiInterface:                "127.0.0.1:8080",
+		PprofEnabled:                true,
+		P2PPrometheusMetricsEnabled: false,
+		StatusMetricsHandler:        &testscommon.StatusMetricsStub{},
+	}
+}
+
 func TestInitialNodeFacade(t *testing.T) {
 	t.Parallel()
 
 	t.Run("nil status metrics should error", func(t *testing.T) {
 		t.Parallel()
 
-		inf, err := NewInitialNodeFacade("127.0.0.1:8080", true, nil)
+		args := createInitialNodeFacadeArgs()
+		args.StatusMetricsHandler = nil
+		inf, err := NewInitialNodeFacade(args)
 		assert.Equal(t, facade.ErrNilStatusMetrics, err)
 		assert.Nil(t, inf)
 	})
 	t.Run("should work", func(t *testing.T) {
 		t.Parallel()
 
-		inf, err := NewInitialNodeFacade("127.0.0.1:8080", true, &testscommon.StatusMetricsStub{})
+		inf, err := NewInitialNodeFacade(createInitialNodeFacadeArgs())
 		assert.Nil(t, err)
 		assert.NotNil(t, inf)
 	})
@@ -40,7 +51,9 @@ func TestInitialNodeFacade_AllMethodsShouldNotPanic(t *testing.T) {
 	}()
 
 	apiInterface := "127.0.0.1:7799"
-	inf, err := NewInitialNodeFacade(apiInterface, true, &testscommon.StatusMetricsStub{})
+	args := createInitialNodeFacadeArgs()
+	args.ApiInterface = apiInterface
+	inf, err := NewInitialNodeFacade(args)
 	assert.Nil(t, err)
 
 	inf.SetSyncer(nil)
@@ -325,6 +338,6 @@ func TestInitialNodeFacade_IsInterfaceNil(t *testing.T) {
 	var inf *initialNodeFacade
 	assert.True(t, inf.IsInterfaceNil())
 
-	inf, _ = NewInitialNodeFacade("127.0.0.1:7799", true, &testscommon.StatusMetricsStub{})
+	inf, _ = NewInitialNodeFacade(createInitialNodeFacadeArgs())
 	assert.False(t, inf.IsInterfaceNil())
 }