From fb83527fbf51f7627e0033a2e12d7e69575cffd5 Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Tue, 12 Dec 2023 02:14:43 +0800 Subject: [PATCH] [CLOB-1001] Add flag to format tag values for datadog error tracking (#858) * WIP * feed in flags to our docker env * truncate stack, remove makefile diff * remove prints' * fix merge conflict * sync once and fix merge * add TODO * remove println --- protocol/app/datadog.go | 44 ++++++++++++++++++++++++++ protocol/app/flags/flags.go | 15 +++++++++ protocol/cmd/dydxprotocold/cmd/root.go | 15 +++++++-- protocol/docker-compose.yml | 8 +++++ protocol/go.mod | 2 +- 5 files changed, 81 insertions(+), 3 deletions(-) diff --git a/protocol/app/datadog.go b/protocol/app/datadog.go index 220e784c47..dbe66c8245 100644 --- a/protocol/app/datadog.go +++ b/protocol/app/datadog.go @@ -6,10 +6,14 @@ import ( "net" "os" "strconv" + "sync" "github.com/cometbft/cometbft/libs/log" "github.com/cosmos/cosmos-sdk/version" "github.com/dydxprotocol/v4-chain/protocol/app/flags" + errorspkg "github.com/pkg/errors" + "github.com/rs/zerolog" + "github.com/rs/zerolog/pkgerrors" "gopkg.in/DataDog/dd-trace-go.v1/profiler" ) @@ -111,3 +115,43 @@ func initDatadogProfiler(logger log.Logger, ddAgentHost string, ddAgentPort uint panic(err) } } + +type DatadogErrorTrackingObject struct { + Stack []map[string]string + Message string + Kind string +} + +func (obj DatadogErrorTrackingObject) MarshalZerologObject(e *zerolog.Event) { + e.Interface("stack", obj.Stack). + Str("message", obj.Message). + Str("kind", obj.Kind) +} + +var ( + zerologFormatterOnce sync.Once +) + +// SetZerologDatadogErrorTrackingFormat sets custom error formatting for log tag +// values that are errors for the zerolog library. Converts them to a format that +// is compatible with datadog error tracking. +func SetZerologDatadogErrorTrackingFormat() { + zerologFormatterOnce.Do(func() { + // Error fields are default set under `error` + // Extract + add the kind and message field + zerolog.ErrorMarshalFunc = func(err error) interface{} { + stackArr, ok := pkgerrors.MarshalStack(errorspkg.WithStack(err)).([]map[string]string) + if !ok { + return struct{}{} + } + objectToReturn := DatadogErrorTrackingObject{ + // Discard common stack prefixes + // TODO(CLOB-1049) Write test for common stack prefix truncation + Stack: stackArr[5:], + Kind: "Exception", + Message: err.Error(), + } + return objectToReturn + } + }) +} diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index 0603997801..b7a7780cfe 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -13,6 +13,7 @@ type Flags struct { DdAgentHost string DdTraceAgentPort uint16 NonValidatingFullNode bool + DdErrorTrackingFormat bool // Existing flags GrpcAddress string @@ -24,6 +25,7 @@ const ( DdAgentHost = "dd-agent-host" DdTraceAgentPort = "dd-trace-agent-port" NonValidatingFullNodeFlag = "non-validating-full-node" + DdErrorTrackingFormat = "dd-error-tracking-format" // Cosmos flags below. These config values can be set as flags or in config.toml. GrpcAddress = "grpc.address" @@ -35,6 +37,7 @@ const ( DefaultDdAgentHost = "" DefaultDdTraceAgentPort = 8126 DefaultNonValidatingFullNode = false + DefaultDdErrorTrackingFormat = false ) // AddFlagsToCmd adds flags to app initialization. @@ -58,6 +61,11 @@ func AddFlagsToCmd(cmd *cobra.Command) { DefaultDdTraceAgentPort, "Sets the Datadog Agent port.", ) + cmd.Flags().Bool( + DdErrorTrackingFormat, + DefaultDdErrorTrackingFormat, + "Enable formatting of log error tags to datadog error tracking format", + ) } // Validate checks that the flags are valid. @@ -79,6 +87,7 @@ func GetFlagValuesFromOptions( NonValidatingFullNode: DefaultNonValidatingFullNode, DdAgentHost: DefaultDdAgentHost, DdTraceAgentPort: DefaultDdTraceAgentPort, + DdErrorTrackingFormat: DefaultDdErrorTrackingFormat, // These are the default values from the Cosmos flags. GrpcAddress: config.DefaultGRPCAddress, @@ -104,6 +113,12 @@ func GetFlagValuesFromOptions( } } + if option := appOpts.Get(DdErrorTrackingFormat); option != nil { + if v, err := cast.ToBoolE(option); err == nil { + result.DdErrorTrackingFormat = v + } + } + if option := appOpts.Get(GrpcAddress); option != nil { if v, err := cast.ToStringE(option); err == nil { result.GrpcAddress = v diff --git a/protocol/cmd/dydxprotocold/cmd/root.go b/protocol/cmd/dydxprotocold/cmd/root.go index 6ed390a939..f83560f5bc 100644 --- a/protocol/cmd/dydxprotocold/cmd/root.go +++ b/protocol/cmd/dydxprotocold/cmd/root.go @@ -34,6 +34,8 @@ import ( genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types" dydxapp "github.com/dydxprotocol/v4-chain/protocol/app" "github.com/dydxprotocol/v4-chain/protocol/app/basic_manager" + protocolflags "github.com/dydxprotocol/v4-chain/protocol/app/flags" + "github.com/spf13/cast" "github.com/spf13/cobra" @@ -88,7 +90,7 @@ func NewRootCmdWithInterceptors( rootCmd := &cobra.Command{ Use: dydxapp.AppDaemonName, Short: "Start dydxprotocol app", - PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { // set the default command outputs cmd.SetOut(cmd.OutOrStdout()) cmd.SetErr(cmd.ErrOrStderr()) @@ -119,7 +121,16 @@ func NewRootCmdWithInterceptors( return err } - serverCtxInterceptor(server.GetServerContextFromCmd(cmd)) + serverCtx := server.GetServerContextFromCmd(cmd) + + // Format logs for error tracking if it is enabled via flags. + if ddErrorTrackingFormatterEnabled := + serverCtx.Viper.Get(protocolflags.DdErrorTrackingFormat); ddErrorTrackingFormatterEnabled != nil { + if enabled, err := cast.ToBoolE(ddErrorTrackingFormatterEnabled); err == nil && enabled { + dydxapp.SetZerologDatadogErrorTrackingFormat() + } + } + serverCtxInterceptor(serverCtx) return nil }, diff --git a/protocol/docker-compose.yml b/protocol/docker-compose.yml index c303801158..b9041c9781 100644 --- a/protocol/docker-compose.yml +++ b/protocol/docker-compose.yml @@ -17,6 +17,8 @@ services: - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,b69182310be02559483e42c77b7b104352713166@dydxprotocold1:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656" - --bridge-daemon-eth-rpc-endpoint - "${ETH_RPC_ENDPOINT}" + - --dd-error-tracking-format + - "true" - --max-daemon-unhealthy-seconds - "4294967295" # Effectively disable the daemon monitor because bridge daemon is flaky in localnet. environment: @@ -45,6 +47,8 @@ services: - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,b69182310be02559483e42c77b7b104352713166@dydxprotocold1:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656" - --bridge-daemon-eth-rpc-endpoint - "${ETH_RPC_ENDPOINT}" + - --dd-error-tracking-format + - "true" - --max-daemon-unhealthy-seconds - "4294967295" environment: @@ -71,6 +75,8 @@ services: - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,b69182310be02559483e42c77b7b104352713166@dydxprotocold1:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656" - --bridge-daemon-eth-rpc-endpoint - "${ETH_RPC_ENDPOINT}" + - --dd-error-tracking-format + - "true" - --max-daemon-unhealthy-seconds - "4294967295" environment: @@ -95,6 +101,8 @@ services: - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,b69182310be02559483e42c77b7b104352713166@dydxprotocold1:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656" - --bridge-daemon-eth-rpc-endpoint - "${ETH_RPC_ENDPOINT}" + - --dd-error-tracking-format + - "true" - --max-daemon-unhealthy-seconds - "4294967295" environment: diff --git a/protocol/go.mod b/protocol/go.mod index 6054578bf1..df822eff98 100644 --- a/protocol/go.mod +++ b/protocol/go.mod @@ -49,6 +49,7 @@ require ( github.com/deckarep/golang-set/v2 v2.3.0 github.com/ethereum/go-ethereum v1.12.0 github.com/ory/dockertest/v3 v3.10.0 + github.com/rs/zerolog v1.29.1 github.com/shopspring/decimal v1.3.1 google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 ) @@ -285,7 +286,6 @@ require ( github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rs/cors v1.8.3 // indirect - github.com/rs/zerolog v1.29.1 // indirect github.com/ryancurrah/gomodguard v1.3.0 // indirect github.com/ryanrolds/sqlclosecheck v0.4.0 // indirect github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect