feat(config): configuration of deadlock detection (#880)

* feat(config): support configuration of deadlock detection

* build: remove deadlock build tag

* chore(config): lint

.github/workflows/docker.yml

@@ -62,7 +62,7 @@ jobs:
             if ('${{ github.ref_type }}' == 'tag' && ! '${{ github.ref_name }}'.includes('-dev')) {
               return 'tenderdash,stable'
             }
-            return 'tenderdash,dev,deadlock'
+            return 'tenderdash,dev'
 
       - name: Set Docker tags and labels
         id: docker_meta

Makefile

@@ -87,11 +87,6 @@ ifeq (boltdb,$(findstring boltdb,$(TENDERMINT_BUILD_OPTIONS)))
   BUILD_TAGS += boltdb
 endif
 
-# handle deadlock
-ifeq (deadlock,$(findstring deadlock,$(TENDERMINT_BUILD_OPTIONS)))
-  BUILD_TAGS += deadlock
-endif
-
 # allow users to pass additional flags via the conventional LDFLAGS variable
 LD_FLAGS += $(LDFLAGS)
 

cmd/tenderdash/commands/root.go

@@ -12,6 +12,7 @@ import (
 	"github.com/dashpay/tenderdash/config"
 	"github.com/dashpay/tenderdash/libs/cli"
 	"github.com/dashpay/tenderdash/libs/log"
+	"github.com/dashpay/tenderdash/node"
 )
 
 const ctxTimeout = 4 * time.Second
@@ -58,6 +59,8 @@ func RootCommand(conf *config.Config, logger log.Logger) *cobra.Command {
 				logger.Info("WARNING", "deprecated field warning", warning)
 			}
 
+			node.SetupDeadlockDetection(&conf.BaseConfig)
+
 			return nil
 		},
 	}

cmd/tenderdash/commands/version.go

@@ -3,7 +3,6 @@ package commands
 import (
 	"runtime"
 
-	"github.com/sasha-s/go-deadlock"
 	"github.com/spf13/cobra"
 
 	"github.com/dashpay/tenderdash/version"
@@ -20,11 +19,6 @@ var VersionCmd *cobra.Command = func() *cobra.Command {
 			cmd.Println(version.TMCoreSemVer)
 			if verbose {
 				cmd.Println("Go version: " + runtime.Version())
-				if deadlock.Opts.Disable {
-					cmd.Println("Deadlock detection: disabled")
-				} else {
-					cmd.Println("Deadlock detection: enabled, timeout: ", deadlock.Opts.DeadlockTimeout.String())
-				}
 			}
 		},
 	}

config/config.go

@@ -240,21 +240,28 @@ type BaseConfig struct { //nolint: maligned
 	// so the app can decide if we should keep the connection or not
 	FilterPeers bool `mapstructure:"filter-peers"` // false
 
+	// If set to positive duration, deadlock detection is enabled and set to the given time.
+	// Use 0 to disable.
+	//
+	// Default: 0
+	DeadlockDetection time.Duration `mapstructure:"deadlock-detection"`
+
 	Other map[string]interface{} `mapstructure:",remain"`
 }
 
 // DefaultBaseConfig returns a default base configuration for a Tendermint node
 func DefaultBaseConfig() BaseConfig {
 	return BaseConfig{
-		Genesis:     defaultGenesisJSONPath,
-		NodeKey:     defaultNodeKeyPath,
-		Mode:        defaultMode,
-		Moniker:     defaultMoniker,
-		LogLevel:    DefaultLogLevel,
-		LogFormat:   log.LogFormatPlain,
-		FilterPeers: false,
-		DBBackend:   "goleveldb",
-		DBPath:      "data",
+		Genesis:           defaultGenesisJSONPath,
+		NodeKey:           defaultNodeKeyPath,
+		Mode:              defaultMode,
+		Moniker:           defaultMoniker,
+		LogLevel:          DefaultLogLevel,
+		LogFormat:         log.LogFormatPlain,
+		FilterPeers:       false,
+		DBBackend:         "goleveldb",
+		DBPath:            "data",
+		DeadlockDetection: 0,
 	}
 }
 
@@ -264,6 +271,7 @@ func TestBaseConfig() BaseConfig {
 	cfg.chainID = factory.DefaultTestChainID
 	cfg.Mode = ModeValidator
 	cfg.DBBackend = "memdb"
+	cfg.DeadlockDetection = 10 * time.Second
 	return cfg
 }
 
@@ -339,6 +347,10 @@ func (cfg BaseConfig) ValidateBasic() error {
 		return fmt.Errorf("unknown mode: %v", cfg.Mode)
 	}
 
+	if cfg.DeadlockDetection < 0 {
+		return errors.New("deadlock-detection can't be negative")
+	}
+
 	return nil
 }
 

config/toml.go

@@ -146,6 +146,12 @@ node-key-file = "{{ js .BaseConfig.NodeKey }}"
 # so the app can decide if we should keep the connection or not
 filter-peers = {{ .BaseConfig.FilterPeers }}
 
+
+# If set to positive duration, deadlock detection is enabled and set to the given time.
+# Use 0 to disable.
+# Default: 0
+deadlock-detection = "{{ .BaseConfig.DeadlockDetection }}"
+
 #######################################################
 ###       ABCI App Connection Options               ###
 #######################################################

node/deadlock.go

@@ -0,0 +1,24 @@
+package node
+
+import (
+	"fmt"
+
+	"github.com/dashpay/tenderdash/config"
+	sync "github.com/sasha-s/go-deadlock"
+)
+
+func SetupDeadlockDetection(cfg *config.BaseConfig) {
+	if cfg.DeadlockDetection == 0 {
+		sync.Opts.Disable = true
+	} else {
+		sync.Opts.Disable = false
+		sync.Opts.OnPotentialDeadlock = func() {
+			fmt.Println("===========================")
+			fmt.Println("POTENTIAL DEADLOCK DETECTED")
+			fmt.Println("===========================")
+		}
+
+		// Set deadlock timeout to 5 minutes, to give enough time to state sync to execute and retry.
+		sync.Opts.DeadlockTimeout = cfg.DeadlockDetection
+	}
+}