[CLOB-1001] Add flag to format tag values for datadog error tracking (#…

…858)

* WIP

* feed in flags to our docker env

* truncate stack, remove makefile diff

* remove prints'

* fix merge conflict

* sync once and fix merge

* add TODO

* remove println

protocol/app/datadog.go

@@ -6,10 +6,14 @@ import (
 	"net"
 	"os"
 	"strconv"
+	"sync"
 
 	"github.com/cometbft/cometbft/libs/log"
 	"github.com/cosmos/cosmos-sdk/version"
 	"github.com/dydxprotocol/v4-chain/protocol/app/flags"
+	errorspkg "github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/pkgerrors"
 	"gopkg.in/DataDog/dd-trace-go.v1/profiler"
 )
 
@@ -111,3 +115,43 @@ func initDatadogProfiler(logger log.Logger, ddAgentHost string, ddAgentPort uint
 		panic(err)
 	}
 }
+
+type DatadogErrorTrackingObject struct {
+	Stack   []map[string]string
+	Message string
+	Kind    string
+}
+
+func (obj DatadogErrorTrackingObject) MarshalZerologObject(e *zerolog.Event) {
+	e.Interface("stack", obj.Stack).
+		Str("message", obj.Message).
+		Str("kind", obj.Kind)
+}
+
+var (
+	zerologFormatterOnce sync.Once
+)
+
+// SetZerologDatadogErrorTrackingFormat sets custom error formatting for log tag
+// values that are errors for the zerolog library. Converts them to a format that
+// is compatible with datadog error tracking.
+func SetZerologDatadogErrorTrackingFormat() {
+	zerologFormatterOnce.Do(func() {
+		// Error fields are default set under `error`
+		// Extract + add the kind and message field
+		zerolog.ErrorMarshalFunc = func(err error) interface{} {
+			stackArr, ok := pkgerrors.MarshalStack(errorspkg.WithStack(err)).([]map[string]string)
+			if !ok {
+				return struct{}{}
+			}
+			objectToReturn := DatadogErrorTrackingObject{
+				// Discard common stack prefixes
+				// TODO(CLOB-1049) Write test for common stack prefix truncation
+				Stack:   stackArr[5:],
+				Kind:    "Exception",
+				Message: err.Error(),
+			}
+			return objectToReturn
+		}
+	})
+}

protocol/app/flags/flags.go

@@ -13,6 +13,7 @@ type Flags struct {
 	DdAgentHost           string
 	DdTraceAgentPort      uint16
 	NonValidatingFullNode bool
+	DdErrorTrackingFormat bool
 
 	// Existing flags
 	GrpcAddress string
@@ -24,6 +25,7 @@ const (
 	DdAgentHost               = "dd-agent-host"
 	DdTraceAgentPort          = "dd-trace-agent-port"
 	NonValidatingFullNodeFlag = "non-validating-full-node"
+	DdErrorTrackingFormat     = "dd-error-tracking-format"
 
 	// Cosmos flags below. These config values can be set as flags or in config.toml.
 	GrpcAddress = "grpc.address"
@@ -35,6 +37,7 @@ const (
 	DefaultDdAgentHost           = ""
 	DefaultDdTraceAgentPort      = 8126
 	DefaultNonValidatingFullNode = false
+	DefaultDdErrorTrackingFormat = false
 )
 
 // AddFlagsToCmd adds flags to app initialization.
@@ -58,6 +61,11 @@ func AddFlagsToCmd(cmd *cobra.Command) {
 		DefaultDdTraceAgentPort,
 		"Sets the Datadog Agent port.",
 	)
+	cmd.Flags().Bool(
+		DdErrorTrackingFormat,
+		DefaultDdErrorTrackingFormat,
+		"Enable formatting of log error tags to datadog error tracking format",
+	)
 }
 
 // Validate checks that the flags are valid.
@@ -79,6 +87,7 @@ func GetFlagValuesFromOptions(
 		NonValidatingFullNode: DefaultNonValidatingFullNode,
 		DdAgentHost:           DefaultDdAgentHost,
 		DdTraceAgentPort:      DefaultDdTraceAgentPort,
+		DdErrorTrackingFormat: DefaultDdErrorTrackingFormat,
 
 		// These are the default values from the Cosmos flags.
 		GrpcAddress: config.DefaultGRPCAddress,
@@ -104,6 +113,12 @@ func GetFlagValuesFromOptions(
 		}
 	}
 
+	if option := appOpts.Get(DdErrorTrackingFormat); option != nil {
+		if v, err := cast.ToBoolE(option); err == nil {
+			result.DdErrorTrackingFormat = v
+		}
+	}
+
 	if option := appOpts.Get(GrpcAddress); option != nil {
 		if v, err := cast.ToStringE(option); err == nil {
 			result.GrpcAddress = v

protocol/cmd/dydxprotocold/cmd/root.go

@@ -34,6 +34,8 @@ import (
 	genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
 	dydxapp "github.com/dydxprotocol/v4-chain/protocol/app"
 	"github.com/dydxprotocol/v4-chain/protocol/app/basic_manager"
+	protocolflags "github.com/dydxprotocol/v4-chain/protocol/app/flags"
+
 	"github.com/spf13/cast"
 	"github.com/spf13/cobra"
 
@@ -88,7 +90,7 @@ func NewRootCmdWithInterceptors(
 	rootCmd := &cobra.Command{
 		Use:   dydxapp.AppDaemonName,
 		Short: "Start dydxprotocol app",
-		PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			// set the default command outputs
 			cmd.SetOut(cmd.OutOrStdout())
 			cmd.SetErr(cmd.ErrOrStderr())
@@ -119,7 +121,16 @@ func NewRootCmdWithInterceptors(
 				return err
 			}
 
-			serverCtxInterceptor(server.GetServerContextFromCmd(cmd))
+			serverCtx := server.GetServerContextFromCmd(cmd)
+
+			// Format logs for error tracking if it is enabled via flags.
+			if ddErrorTrackingFormatterEnabled :=
+				serverCtx.Viper.Get(protocolflags.DdErrorTrackingFormat); ddErrorTrackingFormatterEnabled != nil {
+				if enabled, err := cast.ToBoolE(ddErrorTrackingFormatterEnabled); err == nil && enabled {
+					dydxapp.SetZerologDatadogErrorTrackingFormat()
+				}
+			}
+			serverCtxInterceptor(serverCtx)
 
 			return nil
 		},

protocol/docker-compose.yml

@@ -17,6 +17,8 @@ services:
       - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,b69182310be02559483e42c77b7b104352713166@dydxprotocold1:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656"
       - --bridge-daemon-eth-rpc-endpoint
       - "${ETH_RPC_ENDPOINT}"
+      - --dd-error-tracking-format
+      - "true"
       - --max-daemon-unhealthy-seconds
       - "4294967295" # Effectively disable the daemon monitor because bridge daemon is flaky in localnet.
     environment:
@@ -45,6 +47,8 @@ services:
        - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,b69182310be02559483e42c77b7b104352713166@dydxprotocold1:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656"
        - --bridge-daemon-eth-rpc-endpoint
        - "${ETH_RPC_ENDPOINT}"
+       - --dd-error-tracking-format
+       - "true"
        - --max-daemon-unhealthy-seconds
        - "4294967295"
     environment:
@@ -71,6 +75,8 @@ services:
        - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,b69182310be02559483e42c77b7b104352713166@dydxprotocold1:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656"
        - --bridge-daemon-eth-rpc-endpoint
        - "${ETH_RPC_ENDPOINT}"
+       - --dd-error-tracking-format
+       - "true"
        - --max-daemon-unhealthy-seconds
        - "4294967295"
     environment:
@@ -95,6 +101,8 @@ services:
        - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,b69182310be02559483e42c77b7b104352713166@dydxprotocold1:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656"
        - --bridge-daemon-eth-rpc-endpoint
        - "${ETH_RPC_ENDPOINT}"
+       - --dd-error-tracking-format
+       - "true"
        - --max-daemon-unhealthy-seconds
        - "4294967295"
     environment:

protocol/go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/deckarep/golang-set/v2 v2.3.0
 	github.com/ethereum/go-ethereum v1.12.0
 	github.com/ory/dockertest/v3 v3.10.0
+	github.com/rs/zerolog v1.29.1
 	github.com/shopspring/decimal v1.3.1
 	google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529
 )
@@ -285,7 +286,6 @@ require (
 	github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rs/cors v1.8.3 // indirect
-	github.com/rs/zerolog v1.29.1 // indirect
 	github.com/ryancurrah/gomodguard v1.3.0 // indirect
 	github.com/ryanrolds/sqlclosecheck v0.4.0 // indirect
 	github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect