Merge pull request #3600 from weaveworks/expose-probe-metrics

Expose probe metrics to Prometheus

probe/appclient/app_client.go

@@ -11,6 +11,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/armon/go-metrics"
 	"github.com/gorilla/websocket"
 	"github.com/hashicorp/go-cleanhttp"
 	log "github.com/sirupsen/logrus"
@@ -299,6 +300,10 @@ func (c *appClient) publish(r io.Reader) error {
 	}
 	defer resp.Body.Close()
 
+	metrics.IncrCounterWithLabels([]string{"publishes"}, 1, []metrics.Label{
+		{Name: "destination", Value: req.Host},
+		{Name: "status", Value: fmt.Sprint(resp.StatusCode)},
+	})
 	if resp.StatusCode != http.StatusOK {
 		text, _ := ioutil.ReadAll(resp.Body)
 		return fmt.Errorf(resp.Status + ": " + string(text))

probe/endpoint/conntrack.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/armon/go-metrics"
 	log "github.com/sirupsen/logrus"
 	"github.com/typetypetype/conntrack"
 )
@@ -152,6 +153,7 @@ func (c *conntrackWalker) run() {
 				return
 			}
 			if f.Err != nil {
+				metrics.IncrCounter([]string{"conntrack", "errors"}, 1)
 				log.Errorf("conntrack event error: %v", f.Err)
 				stop()
 				return

probe/endpoint/ebpf.go

@@ -13,6 +13,7 @@ import (
 	"sync"
 	"syscall"
 
+	"github.com/armon/go-metrics"
 	log "github.com/sirupsen/logrus"
 	"github.com/weaveworks/common/fs"
 	"github.com/weaveworks/scope/probe/endpoint/procspy"
@@ -160,6 +161,9 @@ func (t *EbpfTracker) TCPEventV4(e tracer.TcpV4) {
 		// https://github.com/weaveworks/scope/issues/2334
 		log.Errorf("tcp tracer received event with timestamp %v even though the last timestamp was %v. Stopping the eBPF tracker.", e.Timestamp, t.lastTimestampV4)
 		t.stop()
+		metrics.IncrCounterWithLabels([]string{"ebpf", "errors"}, 1, []metrics.Label{
+			{Name: "kind", Value: "timestamp-out-of-order"},
+		})
 		return
 	}
 
@@ -182,6 +186,9 @@ func (t *EbpfTracker) TCPEventV6(e tracer.TcpV6) {
 // LostV4 handles IPv4 TCP event misses from the eBPF tracer.
 func (t *EbpfTracker) LostV4(count uint64) {
 	log.Errorf("tcp tracer lost %d events. Stopping the eBPF tracker", count)
+	metrics.IncrCounterWithLabels([]string{"ebpf", "errors"}, 1, []metrics.Label{
+		{Name: "kind", Value: "lost-events"},
+	})
 	t.stop()
 }
 

probe/probe.go

@@ -135,12 +135,10 @@ func (p *Probe) spyLoop() {
 	for {
 		select {
 		case <-spyTick:
-			t := time.Now()
 			p.tick()
 			rpt := p.report()
 			rpt = p.tag(rpt)
 			p.spiedReports <- rpt
-			metrics.MeasureSince([]string{"Report Generaton"}, t)
 		case <-p.quit:
 			return
 		}
@@ -151,7 +149,10 @@ func (p *Probe) tick() {
 	for _, ticker := range p.tickers {
 		t := time.Now()
 		err := ticker.Tick()
-		metrics.MeasureSince([]string{ticker.Name(), "ticker"}, t)
+		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
+			{Name: "operation", Value: "ticker"},
+			{Name: "module", Value: ticker.Name()},
+		})
 		if err != nil {
 			log.Errorf("Error doing ticker: %v", err)
 		}
@@ -168,7 +169,10 @@ func (p *Probe) report() report.Report {
 			if !timer.Stop() {
 				log.Warningf("%v reporter took %v (longer than %v)", rep.Name(), time.Now().Sub(t), p.spyInterval)
 			}
-			metrics.MeasureSince([]string{rep.Name(), "reporter"}, t)
+			metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
+				{Name: "operation", Value: "reporter"},
+				{Name: "module", Value: rep.Name()},
+			})
 			if err != nil {
 				log.Errorf("Error generating %s report: %v", rep.Name(), err)
 				newReport = report.MakeReport() // empty is OK to merge
@@ -193,7 +197,10 @@ func (p *Probe) tag(r report.Report) report.Report {
 		if !timer.Stop() {
 			log.Warningf("%v tagger took %v (longer than %v)", tagger.Name(), time.Now().Sub(t), p.spyInterval)
 		}
-		metrics.MeasureSince([]string{tagger.Name(), "tagger"}, t)
+		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
+			{Name: "operation", Value: "tagger"},
+			{Name: "module", Value: tagger.Name()},
+		})
 		if err != nil {
 			log.Errorf("Error applying tagger: %v", err)
 		}

prog/probe.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/armon/go-metrics"
+	metrics_prom "github.com/armon/go-metrics/prometheus"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 
@@ -108,11 +109,24 @@ func probeMain(flags probeFlags, targets []appclient.Target) {
 	traceCloser := tracing.NewFromEnv("scope-probe")
 	defer traceCloser.Close()
 
-	// Setup in memory metrics sink
-	inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
-	sig := metrics.DefaultInmemSignal(inm)
-	defer sig.Stop()
-	metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)
+	cfg := &metrics.Config{
+		ServiceName:      "scope-probe",
+		TimerGranularity: time.Second,
+		FilterDefault:    true, // Don't filter metrics by default
+	}
+	if flags.httpListen == "" {
+		// Setup in memory metrics sink
+		inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
+		sig := metrics.DefaultInmemSignal(inm)
+		defer sig.Stop()
+		metrics.NewGlobal(cfg, inm)
+	} else {
+		sink, err := metrics_prom.NewPrometheusSink()
+		if err != nil {
+			log.Fatalf("Failed to create Prometheus metrics sink: %v", err)
+		}
+		metrics.NewGlobal(cfg, sink)
+	}
 	logCensoredArgs()
 	defer log.Info("probe exiting")
 

site/building.md

@@ -62,6 +62,8 @@ The Scope Probe is instrumented with various counters and timers. To have it dum
     kill -USR1 $(pgrep -f scope-probe)
     docker logs weavescope
 
+If you run with `--probe.http.listen` enabled, these are exposed as Prometheus metrics instead, via http at `/metrics`.
+
 ## <a name="profiling"></a>Profiling
 
 Both the Scope App and the Scope Probe offer [HTTP endpoints with profiling information](https://golang.org/pkg/net/http/pprof/).