Added Invariant Execution Error Metric Gauge (#112)

internal/engine/engine.go

@@ -6,6 +6,8 @@ import (
 	"github.com/base-org/pessimism/internal/core"
 	"github.com/base-org/pessimism/internal/engine/invariant"
 	"github.com/base-org/pessimism/internal/logging"
+	"github.com/base-org/pessimism/internal/metrics"
+
 	"go.uber.org/zap"
 )
 
@@ -51,6 +53,10 @@ func (e *hardCodedEngine) Execute(ctx context.Context, data core.TransitData,
 	outcome, invalid, err := inv.Invalidate(data)
 	if err != nil {
 		logger.Error("Failed to perform invalidation option for invariant", zap.Error(err))
+
+		metrics.WithContext(ctx).
+			RecordInvExecutionError(inv)
+
 		return nil, false
 	}
 

internal/metrics/metrics.go

@@ -38,6 +38,7 @@ type Metricer interface {
 	RecordAlertGenerated(alert core.Alert)
 	RecordNodeError(network core.Network)
 	RecordPipelineLatency(pUUID core.PUUID, latency float64)
+	RecordInvExecutionError(inv invariant.Invariant)
 	RecordInvExecutionTime(inv invariant.Invariant, latency float64)
 	RecordUp()
 	Start()
@@ -55,6 +56,7 @@ type Metrics struct {
 	BlockLatency     *prometheus.GaugeVec
 	PipelineLatency  *prometheus.GaugeVec
 	InvExecutionTime *prometheus.GaugeVec
+	InvariantErrors  *prometheus.CounterVec
 
 	registry *prometheus.Registry
 	factory  Factory
@@ -138,6 +140,11 @@ func New(ctx context.Context, cfg *Config) (Metricer, func(), error) {
 			Help:      "Nanosecond time of invariant execution",
 			Namespace: metricsNamespace,
 		}, []string{"invariant"}),
+		InvariantErrors: factory.NewCounterVec(prometheus.CounterOpts{
+			Name:      "invariant_errors_total",
+			Help:      "Number of errors generated by invariant executions",
+			Namespace: metricsNamespace,
+		}, []string{"invariant"}),
 
 		registry: registry,
 		factory:  factory,
@@ -163,6 +170,13 @@ func (m *Metrics) RecordUp() {
 	m.Up.Set(1)
 }
 
+// RecordInvExecutionError ... Increments the number of errors generated by invariant executions
+func (m *Metrics) RecordInvExecutionError(inv invariant.Invariant) {
+	invType := inv.SUUID().PID.InvType().String()
+	m.InvariantErrors.WithLabelValues(invType).Inc()
+}
+
+// RecordInvExecutionTime ... Records the time it took to execute an invariant
 func (m *Metrics) RecordInvExecutionTime(inv invariant.Invariant, latency float64) {
 	invType := inv.SUUID().PID.InvType().String()
 	m.InvExecutionTime.WithLabelValues(invType).Set(latency)
@@ -240,6 +254,7 @@ func (n *noopMetricer) RecordAlertGenerated(_ core.Alert)
 func (n *noopMetricer) RecordNodeError(_ core.Network)                          {}
 func (n *noopMetricer) RecordBlockLatency(_ core.Network, _ float64)            {}
 func (n *noopMetricer) RecordPipelineLatency(_ core.PUUID, _ float64)           {}
+func (n *noopMetricer) RecordInvExecutionError(_ invariant.Invariant)           {}
 
 func (n *noopMetricer) Shutdown(_ context.Context) error {
 	return nil