Initialize metrics with labels

go.mod

@@ -29,7 +29,6 @@ require (
 	github.com/mennanov/fieldmask-utils v1.1.0
 	github.com/opencontainers/runtime-spec v1.2.0
 	github.com/pelletier/go-toml v1.9.5
-	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.19.0
 	github.com/prometheus/client_model v0.6.0
 	github.com/sirupsen/logrus v1.9.3
@@ -149,6 +148,7 @@ require (
 	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/common v0.48.0 // indirect

pkg/api/ops/ops.go

@@ -3,6 +3,12 @@
 
 package ops
 
+import (
+	"fmt"
+
+	"github.com/cilium/tetragon/pkg/logger"
+)
+
 const (
 	MSG_OP_UNDEF = 0
 	// MSG_OP_EXECVE event indicates a process was created. The 'PID'
@@ -65,21 +71,36 @@ const (
 	MsgOpKfreeSkb           = 11
 	MsgOpGenericKprobe      = 13
 	MsgOpGeneric_Tracepoint = 14
+	MsgOpGenericUprobe      = 15
+	MsgOpClone              = 23
+	MsgOpData               = 24
+	MsgOpCgroup             = 25
+	MsgOpLoader             = 26
 	MsgOpTest               = 254
 )
 
+var OpCodeStrings = map[OpCode]string{
+	MsgOpUndef:              "Undef",
+	MsgOpExecve:             "Execve",
+	MsgOpExit:               "Exit",
+	MsgOpKfreeSkb:           "KfreeSkb",
+	MsgOpGenericKprobe:      "GenericKprobe",
+	MsgOpGeneric_Tracepoint: "GenericTracepoint",
+	MsgOpGenericUprobe:      "GenericUprobe",
+	MsgOpClone:              "Clone",
+	MsgOpData:               "Data",
+	MsgOpCgroup:             "Cgroup",
+	MsgOpLoader:             "Loader",
+	MsgOpTest:               "Test",
+}
+
 func (op OpCode) String() string {
-	return [...]string{
-		0:   "Undef",
-		5:   "Execve",
-		7:   "Exit",
-		13:  "GenericKprobe",
-		14:  "GenericTracepoint",
-		23:  "Clone",
-		24:  "Data",
-		25:  "Cgroup",
-		254: "Test",
-	}[op]
+	s, ok := OpCodeStrings[op]
+	if !ok {
+		logger.GetLogger().WithField("opcode", op).Info("Unknown OpCode. This is a bug, please report it to Tetragon developers.")
+		return fmt.Sprintf("Unknown(%d)", op)
+	}
+	return s
 }
 
 func (op CgroupOpCode) String() string {

pkg/eventcache/eventcache.go

@@ -141,11 +141,11 @@ func (ec *Cache) handleEvents() {
 				continue
 			}
 			if errors.Is(err, ErrFailedToGetParentInfo) {
-				eventcachemetrics.ParentInfoError(notify.EventTypeString(event.event)).Inc()
+				eventcachemetrics.ParentInfoError(notify.EventType(event.event)).Inc()
 			} else if errors.Is(err, ErrFailedToGetProcessInfo) {
-				eventcachemetrics.ProcessInfoError(notify.EventTypeString(event.event)).Inc()
+				eventcachemetrics.ProcessInfoError(notify.EventType(event.event)).Inc()
 			} else if errors.Is(err, ErrFailedToGetPodInfo) {
-				eventcachemetrics.PodInfoError(notify.EventTypeString(event.event)).Inc()
+				eventcachemetrics.PodInfoError(notify.EventType(event.event)).Inc()
 			}
 		}
 

pkg/grpc/exec/exec.go

@@ -70,7 +70,7 @@ func GetProcessExec(event *MsgExecveEventUnix, useCache bool) *tetragon.ProcessE
 	}
 
 	if tetragonProcess.Pid == nil {
-		eventcachemetrics.EventCacheError("GetProcessExec: nil Process.Pid").Inc()
+		eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(tetragonEvent)).Inc()
 		return nil
 	}
 
@@ -394,7 +394,7 @@ func GetProcessExit(event *MsgExitEventUnix) *tetragon.ProcessExit {
 	}
 
 	if tetragonProcess.Pid == nil {
-		eventcachemetrics.EventCacheError("GetProcessExit: nil Process.Pid").Inc()
+		eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(tetragonEvent)).Inc()
 		return nil
 	}
 

pkg/grpc/tracing/stats.go

@@ -19,6 +19,14 @@ var (
 
 func InitMetrics(registry *prometheus.Registry) {
 	registry.MustRegister(LoaderStats)
+
+	// Initialize metrics with labels
+	for _, ty := range LoaderTypeStrings {
+		LoaderStats.WithLabelValues(ty).Add(0)
+	}
+
+	// NOTES:
+	// * Rename process_loader_stats metric (to e.g. process_loader_events_total) and count label (to e.g. event)?
 }
 
 type LoaderType int

pkg/grpc/tracing/tracing.go

@@ -335,7 +335,7 @@ func GetProcessKprobe(event *MsgGenericKprobeUnix) *tetragon.ProcessKprobe {
 	}
 
 	if tetragonProcess.Pid == nil {
-		eventcachemetrics.EventCacheError("GetProcessKprobe: nil Process.Pid").Inc()
+		eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(tetragonEvent)).Inc()
 		return nil
 	}
 
@@ -497,7 +497,7 @@ func (msg *MsgGenericTracepointUnix) HandleMessage() *tetragon.GetEventsResponse
 	}
 
 	if tetragonProcess.Pid == nil {
-		eventcachemetrics.EventCacheError("GetProcessTracepoint: nil Process.Pid").Inc()
+		eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(tetragonEvent)).Inc()
 		return nil
 	}
 
@@ -615,28 +615,26 @@ func GetProcessLoader(msg *MsgProcessLoaderUnix) *tetragon.ProcessLoader {
 		tetragonProcess = process.UnsafeGetProcess()
 	}
 
+	notifyEvent := &ProcessLoaderNotify{
+		ProcessLoader: tetragon.ProcessLoader{
+			Process: tetragonProcess,
+			Path:    msg.Path,
+			Buildid: msg.Buildid,
+		},
+	}
+
 	if tetragonProcess.Pid == nil {
-		eventcachemetrics.EventCacheError("GetProcessLoader: nil Process.Pid").Inc()
+		eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(notifyEvent)).Inc()
 		return nil
 	}
 
 	if ec := eventcache.Get(); ec != nil &&
 		(ec.Needed(tetragonProcess) || (tetragonProcess.Pid.Value > 1)) {
-		tetragonEvent := &ProcessLoaderNotify{}
-		tetragonEvent.Process = tetragonProcess
-		tetragonEvent.Path = msg.Path
-		tetragonEvent.Buildid = msg.Buildid
-		ec.Add(nil, tetragonEvent, msg.Msg.Common.Ktime, msg.Msg.ProcessKey.Ktime, msg)
+		ec.Add(nil, notifyEvent, msg.Msg.Common.Ktime, msg.Msg.ProcessKey.Ktime, msg)
 		return nil
 	}
 
-	tetragonEvent := &tetragon.ProcessLoader{
-		Process: tetragonProcess,
-		Path:    msg.Path,
-		Buildid: msg.Buildid,
-	}
-
-	return tetragonEvent
+	return &notifyEvent.ProcessLoader
 }
 
 func (msg *MsgProcessLoaderUnix) Notify() bool {
@@ -735,7 +733,7 @@ func GetProcessUprobe(event *MsgGenericUprobeUnix) *tetragon.ProcessUprobe {
 	}
 
 	if tetragonProcess.Pid == nil {
-		eventcachemetrics.EventCacheError("GetProcessUprobe: nil Process.Pid").Inc()
+		eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(tetragonEvent)).Inc()
 		return nil
 	}
 

pkg/metrics/errormetrics/errormetrics.go

@@ -5,32 +5,62 @@ package errormetrics
 
 import (
 	"fmt"
-	"strings"
 
+	"github.com/cilium/tetragon/pkg/api/ops"
 	"github.com/cilium/tetragon/pkg/metrics/consts"
-	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-type ErrorType string
+type ErrorType int
 
-var (
+const (
 	// Process not found on get() call.
-	ProcessCacheMissOnGet ErrorType = "process_cache_miss_on_get"
+	ProcessCacheMissOnGet ErrorType = iota
 	// Process evicted from the cache.
-	ProcessCacheEvicted ErrorType = "process_cache_evicted"
+	ProcessCacheEvicted
 	// Process not found on remove() call.
-	ProcessCacheMissOnRemove ErrorType = "process_cache_miss_on_remove"
+	ProcessCacheMissOnRemove
 	// Tid and Pid mismatch that could affect BPF and user space caching logic
-	ProcessPidTidMismatch ErrorType = "process_pid_tid_mismatch"
+	ProcessPidTidMismatch
 	// An event is missing process info.
-	EventMissingProcessInfo ErrorType = "event_missing_process_info"
+	EventMissingProcessInfo
 	// An error occurred in an event handler.
-	HandlerError ErrorType = "handler_error"
+	HandlerError
 	// An event finalizer on Process failed
-	EventFinalizeProcessInfoFailed ErrorType = "event_finalize_process_info_failed"
+	EventFinalizeProcessInfoFailed
+)
+
+var errorTypeLabelValues = map[ErrorType]string{
+	ProcessCacheMissOnGet:          "process_cache_miss_on_get",
+	ProcessCacheEvicted:            "process_cache_evicted",
+	ProcessCacheMissOnRemove:       "process_cache_miss_on_remove",
+	ProcessPidTidMismatch:          "process_pid_tid_mismatch",
+	EventMissingProcessInfo:        "event_missing_process_info",
+	HandlerError:                   "handler_error",
+	EventFinalizeProcessInfoFailed: "event_finalize_process_info_failed",
+}
+
+func (e ErrorType) String() string {
+	return errorTypeLabelValues[e]
+}
+
+type EventHandlerError int
+
+// TODO: Recognize different errors returned by individual handlers
+const (
+	HandlePerfUnknownOp EventHandlerError = iota
+	HandlePerfHandlerError
 )
 
+var eventHandlerErrorLabelValues = map[EventHandlerError]string{
+	HandlePerfUnknownOp:    "unknown_opcode",
+	HandlePerfHandlerError: "event_handler_failed",
+}
+
+func (e EventHandlerError) String() string {
+	return eventHandlerErrorLabelValues[e]
+}
+
 var (
 	ErrorTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace:   consts.MetricsNamespace,
@@ -50,24 +80,44 @@ var (
 func InitMetrics(registry *prometheus.Registry) {
 	registry.MustRegister(ErrorTotal)
 	registry.MustRegister(HandlerErrors)
+
+	// Initialize metrics with labels
+	for er := range errorTypeLabelValues {
+		GetErrorTotal(er).Add(0)
+	}
+	for opcode := range ops.OpCodeStrings {
+		if opcode != ops.MsgOpUndef && opcode != ops.MsgOpTest {
+			GetHandlerErrors(opcode, HandlePerfHandlerError).Add(0)
+		}
+	}
+	// NB: We initialize only ops.MsgOpUndef here, but unknown_opcode can occur for any opcode
+	// that is not explicitly handled.
+	GetHandlerErrors(ops.MsgOpUndef, HandlePerfUnknownOp).Add(0)
+
+	// NOTES:
+	// * op, msg_op, opcode - standardize on a label (+ add human-readable label)
+	// * error, error_type, type - standardize on a label
+	// * Delete errors_total{type="handler_error"} - it duplicates handler_errors_total
+	// * Consider further splitting errors_total
+	// * Rename handler_errors_total to event_handler_errors_total?
 }
 
 // Get a new handle on an ErrorTotal metric for an ErrorType
-func GetErrorTotal(t ErrorType) prometheus.Counter {
-	return ErrorTotal.WithLabelValues(string(t))
+func GetErrorTotal(er ErrorType) prometheus.Counter {
+	return ErrorTotal.WithLabelValues(er.String())
 }
 
 // Increment an ErrorTotal for an ErrorType
-func ErrorTotalInc(t ErrorType) {
-	GetErrorTotal(t).Inc()
+func ErrorTotalInc(er ErrorType) {
+	GetErrorTotal(er).Inc()
 }
 
 // Get a new handle on the HandlerErrors metric
-func GetHandlerErrors(opcode int, err error) prometheus.Counter {
-	return HandlerErrors.WithLabelValues(fmt.Sprint(opcode), strings.ReplaceAll(fmt.Sprintf("%T", errors.Cause(err)), "*", ""))
+func GetHandlerErrors(opcode ops.OpCode, er EventHandlerError) prometheus.Counter {
+	return HandlerErrors.WithLabelValues(fmt.Sprint(int32(opcode)), er.String())
 }
 
 // Increment the HandlerErrors metric
-func HandlerErrorsInc(opcode int, err error) {
-	GetHandlerErrors(opcode, err).Inc()
+func HandlerErrorsInc(opcode ops.OpCode, er EventHandlerError) {
+	GetHandlerErrors(opcode, er).Inc()
 }