expose prometheus metrics (#24)

Dockerfile

@@ -1,6 +1,7 @@
 FROM golang:1.16 as build
 
 COPY ./cmd /usr/src/envoy-control-plane/cmd
+COPY ./pkg /usr/src/envoy-control-plane/pkg
 COPY go.* /usr/src/envoy-control-plane/
 COPY .git /usr/src/envoy-control-plane/
 

Makefile

@@ -10,11 +10,11 @@ testChart:
 	helm lint --strict ./chart/envoy-control-plane
 	helm template ./chart/envoy-control-plane | kubectl apply --dry-run=client --validate -f -
 build:
-	docker build . -t paskalmaksim/envoy-control-plane:dev
+	docker build --pull . -t paskalmaksim/envoy-control-plane:dev
 build-envoy:
 	docker-compose build envoy-test1
 buildEnvoy:
-	docker build ./envoy -t paskalmaksim/envoy-docker-image:dev
+	docker build --pull ./envoy -t paskalmaksim/envoy-docker-image:dev
 build-cli:
 	./scripts/build-cli.sh
 push:

README.md

@@ -65,4 +65,23 @@ helm uninstall envoy-control-plane -n envoy-control-plane
   - containerPort: 18000  # envoy admin
 ```
 ### Configurate your envoy sidecars with with ConfigMap
-sample configuration here https://github.com/maksim-paskal/envoy-control-plane/blob/main/chart/envoy-control-plane/templates/envoy-test1-id.yaml
+sample configuration here https://github.com/maksim-paskal/envoy-control-plane/blob/main/chart/envoy-control-plane/templates/envoy-test1-id.yaml
+
+### Prometheus metrics
+envoy-control-plane expose metrics on `/api/metrics` endpoint in web interface - for static configuration use this scrape config:
+```
+scrape_configs:
+- job_name: envoy-control-plane
+  scrape_interval: 1s
+  metrics_path: /api/metrics
+  static_configs:
+  - targets:
+    - <envoy-control-plane-ip>:18081
+```
+or just add pod annotation
+```
+annotations:
+  prometheus.io/path: '/api/metrics'
+  prometheus.io/scrape: 'true'
+  prometheus.io/port: '18081'
+```

chart/envoy-control-plane/templates/control-plane.yaml

@@ -63,6 +63,10 @@ spec:
   replicas: 1
   template:
     metadata:
+      annotations:
+        prometheus.io/path: '/api/metrics'
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: '18081'
       labels:
         app: envoy-control-plane
     spec:

cmd/main/planeCallbacks.go

@@ -17,6 +17,7 @@ import (
 	"sync"
 
 	discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/maksim-paskal/envoy-control-plane/pkg/metrics"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/encoding/protojson"
 )
@@ -38,6 +39,8 @@ func (cb *callbacks) Report() {
 }
 
 func (cb *callbacks) OnStreamOpen(ctx context.Context, id int64, typ string) error {
+	metrics.GrpcOnStreamOpen.Inc()
+
 	if log.GetLevel() >= log.DebugLevel || *appConfig.LogAccess {
 		log.Debugf("OnStreamOpen %d open for %s", id, typ)
 	}
@@ -46,12 +49,16 @@ func (cb *callbacks) OnStreamOpen(ctx context.Context, id int64, typ string) err
 }
 
 func (cb *callbacks) OnStreamClosed(id int64) {
+	metrics.GrpcOnStreamClosed.Inc()
+
 	if log.GetLevel() >= log.DebugLevel || *appConfig.LogAccess {
 		log.Debugf("OnStreamClosed %d closed", id)
 	}
 }
 
 func (cb *callbacks) OnStreamRequest(id int64, r *discovery.DiscoveryRequest) error {
+	metrics.GrpcOnStreamRequest.Inc()
+
 	if log.GetLevel() >= log.DebugLevel || *appConfig.LogAccess {
 		log.Debugf("OnStreamRequest")
 	}
@@ -69,6 +76,8 @@ func (cb *callbacks) OnStreamRequest(id int64, r *discovery.DiscoveryRequest) er
 }
 
 func (cb *callbacks) OnStreamResponse(id int64, r *discovery.DiscoveryRequest, w *discovery.DiscoveryResponse) {
+	metrics.GrpcOnStreamResponse.Inc()
+
 	if log.GetLevel() >= log.DebugLevel || *appConfig.LogAccess {
 		json, _ := protojson.Marshal(r)
 		log.Debugf("DiscoveryRequest=>\n%s\n", string(json))
@@ -81,6 +90,8 @@ func (cb *callbacks) OnStreamResponse(id int64, r *discovery.DiscoveryRequest, w
 }
 
 func (cb *callbacks) OnFetchRequest(ctx context.Context, req *discovery.DiscoveryRequest) error {
+	metrics.GrpcOnFetchRequest.Inc()
+
 	if log.GetLevel() >= log.DebugLevel || *appConfig.LogAccess {
 		log.Debugf("OnFetchRequest...")
 	}
@@ -98,6 +109,8 @@ func (cb *callbacks) OnFetchRequest(ctx context.Context, req *discovery.Discover
 }
 
 func (cb *callbacks) OnFetchResponse(r *discovery.DiscoveryRequest, w *discovery.DiscoveryResponse) {
+	metrics.GrpcOnFetchResponse.Inc()
+
 	if log.GetLevel() >= log.DebugLevel || *appConfig.LogAccess {
 		json, _ := protojson.Marshal(r)
 		log.Debugf("DiscoveryRequest=>\n%s\n", string(json))
@@ -108,6 +121,8 @@ func (cb *callbacks) OnFetchResponse(r *discovery.DiscoveryRequest, w *discovery
 }
 
 func (cb *callbacks) OnStreamDeltaRequest(streamID int64, req *discovery.DeltaDiscoveryRequest) error {
+	metrics.GrpcOnStreamDeltaRequest.Inc()
+
 	if log.GetLevel() >= log.DebugLevel || *appConfig.LogAccess {
 		log := log.WithField("streamID", streamID)
 
@@ -119,6 +134,8 @@ func (cb *callbacks) OnStreamDeltaRequest(streamID int64, req *discovery.DeltaDi
 }
 
 func (cb *callbacks) OnStreamDeltaResponse(streamID int64, req *discovery.DeltaDiscoveryRequest, resp *discovery.DeltaDiscoveryResponse) { //nolint:lll
+	metrics.GrpcOnStreamDeltaResponse.Inc()
+
 	if log.GetLevel() >= log.DebugLevel || *appConfig.LogAccess {
 		log := log.WithField("streamID", streamID)
 
@@ -131,6 +148,8 @@ func (cb *callbacks) OnStreamDeltaResponse(streamID int64, req *discovery.DeltaD
 }
 
 func (cb *callbacks) OnStreamDeltaRequestOnStreamDeltaRequest(streamID int64, req *discovery.DeltaDiscoveryRequest) error { //nolint:lll,unparam
+	metrics.GrpcOnStreamDeltaRequestOnStreamDeltaRequest.Inc()
+
 	if log.GetLevel() >= log.DebugLevel || *appConfig.LogAccess {
 		log := log.WithField("streamID", streamID)
 
@@ -142,6 +161,8 @@ func (cb *callbacks) OnStreamDeltaRequestOnStreamDeltaRequest(streamID int64, re
 }
 
 func (cb *callbacks) OnDeltaStreamOpen(ctx context.Context, streamID int64, typeURL string) error {
+	metrics.GrpcOnDeltaStreamOpen.Inc()
+
 	if log.GetLevel() >= log.DebugLevel || *appConfig.LogAccess {
 		log := log.WithField("streamID", streamID)
 
@@ -152,6 +173,8 @@ func (cb *callbacks) OnDeltaStreamOpen(ctx context.Context, streamID int64, type
 }
 
 func (cb *callbacks) OnDeltaStreamClosed(streamID int64) {
+	metrics.GrpcOnDeltaStreamClosed.Inc()
+
 	if log.GetLevel() >= log.DebugLevel || *appConfig.LogAccess {
 		log := log.WithField("streamID", streamID)
 

cmd/main/webServer.go

@@ -26,6 +26,7 @@ import (
 	"sync"
 
 	"github.com/envoyproxy/go-control-plane/pkg/cache/v3"
+	"github.com/maksim-paskal/envoy-control-plane/pkg/metrics"
 	logrushooksentry "github.com/maksim-paskal/logrus-hook-sentry"
 	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -95,11 +96,15 @@ func newWebServer(clientset *kubernetes.Clientset, configStore *sync.Map) *WebSe
 		handler:     ws.handlerVersion,
 	})
 
+	m := metrics.NewMetrics()
+
 	go func() {
 		for _, route := range ws.routes {
 			http.HandleFunc(route.path, route.handler)
 		}
 
+		http.Handle("/api/metrics", m.Handler())
+
 		ws.log.Info("http.port=", *appConfig.WebAddress)
 
 		if err := http.ListenAndServe(*appConfig.WebAddress, nil); err != nil {

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/maksim-paskal/utils-go v0.0.5
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.11.0
 	github.com/sirupsen/logrus v1.8.1
 	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
 	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect