[TT-1218] add configurable startup timeout (#1008)

* add configurable startup timeout
* wait 2x longer for non-geth eth2 clients to be ready (they are noticeably slower)

config/private_ethereum_network.go

@@ -354,5 +354,5 @@ func (c *EthereumChainConfig) GetDefaultWaitDuration() time.Duration {
 }
 
 func (c *EthereumChainConfig) GetDefaultFinalizationWaitDuration() time.Duration {
-	return time.Duration(5 * time.Minute)
+	return 5 * time.Minute
 }

docker/test_env/besu_eth1.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
@@ -26,6 +27,7 @@ func NewBesuEth1(networks []string, chainConfig *config.EthereumChainConfig, opt
 			Networks:         networks,
 			ContainerImage:   parts[0],
 			ContainerVersion: parts[1],
+			StartupTimeout:   2 * time.Minute,
 		},
 		chainConfig:     chainConfig,
 		l:               logging.GetTestLogger(nil),
@@ -63,8 +65,8 @@ func (g *Besu) getEth1ContainerRequest() (*tc.ContainerRequest, error) {
 		WaitingFor: tcwait.ForAll(
 			tcwait.ForLog("WebSocketService | Websocket service started"),
 			NewWebSocketStrategy(NatPort(DEFAULT_EVM_NODE_WS_PORT), g.l),
-			NewHTTPStrategy("/", NatPort(DEFAULT_EVM_NODE_HTTP_PORT)).WithStatusCode(201),
-		),
+			NewHTTPStrategy("/", NatPort(DEFAULT_EVM_NODE_HTTP_PORT)).WithStatusCode(201)).
+			WithStartupTimeoutDefault(g.StartupTimeout),
 		Entrypoint: []string{
 			"besu",
 			"--genesis-file", "/opt/besu/genesis/genesis.json",

docker/test_env/besu_eth2.go

@@ -25,6 +25,7 @@ func NewBesuEth2(networks []string, chainConfig *config.EthereumChainConfig, gen
 			Networks:         networks,
 			ContainerImage:   parts[0],
 			ContainerVersion: parts[1],
+			StartupTimeout:   2 * time.Minute,
 		},
 		chainConfig:     chainConfig,
 		posSettings:     posSettings{generatedDataHostDir: generatedDataHostDir},
@@ -90,9 +91,8 @@ func (g *Besu) getEth2ContainerRequest() (*tc.ContainerRequest, error) {
 		ExposedPorts: []string{NatPortFormat(DEFAULT_EVM_NODE_HTTP_PORT), NatPortFormat(DEFAULT_EVM_NODE_WS_PORT), NatPortFormat(ETH2_EXECUTION_PORT)},
 		WaitingFor: tcwait.ForAll(
 			tcwait.ForLog("Ethereum main loop is up").
-				WithStartupTimeout(120 * time.Second).
-				WithPollInterval(1 * time.Second),
-		),
+				WithPollInterval(1 * time.Second)).
+			WithStartupTimeoutDefault(g.StartupTimeout),
 		User: "0:0", //otherwise in CI we get "permission denied" error, when trying to access data from mounted volume
 		Cmd:  cmd,
 		Env: map[string]string{

docker/test_env/env_component.go

@@ -31,6 +31,7 @@ type EnvComponent struct {
 	PostStopsHooks     []tc.ContainerHook   `json:"-"`
 	PreTerminatesHooks []tc.ContainerHook   `json:"-"`
 	LogLevel           string               `json:"-"`
+	StartupTimeout     time.Duration        `json:"-"`
 }
 
 type EnvComponentOption = func(c *EnvComponent)
@@ -43,6 +44,14 @@ func WithContainerName(name string) EnvComponentOption {
 	}
 }
 
+func WithStartupTimeout(timeout time.Duration) EnvComponentOption {
+	return func(c *EnvComponent) {
+		if timeout != 0 {
+			c.StartupTimeout = timeout
+		}
+	}
+}
+
 func WithContainerImageWithVersion(imageWithVersion string) EnvComponentOption {
 	return func(c *EnvComponent) {
 		split := strings.Split(imageWithVersion, ":")

docker/test_env/erigon_eth1.go

@@ -29,6 +29,7 @@ func NewErigonEth1(networks []string, chainConfig *config.EthereumChainConfig, o
 			Networks:         networks,
 			ContainerImage:   parts[0],
 			ContainerVersion: parts[1],
+			StartupTimeout:   2 * time.Minute,
 		},
 		chainConfig:     chainConfig,
 		l:               logging.GetTestLogger(nil),
@@ -110,10 +111,9 @@ func (g *Erigon) getEth1ContainerRequest() (*tc.ContainerRequest, error) {
 		ExposedPorts:  []string{NatPortFormat(DEFAULT_EVM_NODE_HTTP_PORT)},
 		WaitingFor: tcwait.ForAll(
 			tcwait.ForLog("Started P2P networking").
-				WithStartupTimeout(120*time.Second).
 				WithPollInterval(1*time.Second),
-			NewWebSocketStrategy(NatPort(DEFAULT_EVM_NODE_HTTP_PORT), g.l),
-		),
+			NewWebSocketStrategy(NatPort(DEFAULT_EVM_NODE_HTTP_PORT), g.l)).
+			WithStartupTimeoutDefault(g.StartupTimeout),
 		User: "0:0",
 		Entrypoint: []string{
 			"sh",

docker/test_env/erigon_eth2.go

@@ -28,6 +28,7 @@ func NewErigonEth2(networks []string, chainConfig *config.EthereumChainConfig, g
 			Networks:         networks,
 			ContainerImage:   parts[0],
 			ContainerVersion: parts[1],
+			StartupTimeout:   2 * time.Minute,
 		},
 		chainConfig:          chainConfig,
 		generatedDataHostDir: generatedDataHostDir,
@@ -73,9 +74,8 @@ func (g *Erigon) getEth2ContainerRequest() (*tc.ContainerRequest, error) {
 		ExposedPorts:  []string{NatPortFormat(DEFAULT_EVM_NODE_HTTP_PORT), NatPortFormat(ETH2_EXECUTION_PORT)},
 		WaitingFor: tcwait.ForAll(
 			tcwait.ForLog("Started P2P networking").
-				WithStartupTimeout(120 * time.Second).
-				WithPollInterval(1 * time.Second),
-		),
+				WithPollInterval(1 * time.Second)).
+			WithStartupTimeoutDefault(120 * time.Second),
 		User: "0:0",
 		Entrypoint: []string{
 			"sh",

docker/test_env/eth2_init_helpers.go

@@ -32,7 +32,8 @@ type AfterGenesisHelper struct {
 func NewInitHelper(chainConfig config.EthereumChainConfig, customConfigDataDir string, opts ...EnvComponentOption) *AfterGenesisHelper {
 	g := &AfterGenesisHelper{
 		EnvComponent: EnvComponent{
-			ContainerName: fmt.Sprintf("%s-%s", "after-genesis-helper", uuid.NewString()[0:8]),
+			ContainerName:  fmt.Sprintf("%s-%s", "after-genesis-helper", uuid.NewString()[0:8]),
+			StartupTimeout: 20 * time.Second,
 		},
 		chainConfig:         chainConfig,
 		customConfigDataDir: customConfigDataDir,
@@ -97,7 +98,7 @@ func (g *AfterGenesisHelper) getContainerRequest(networks []string) (*tc.Contain
 		ImagePlatform: "linux/x86_64",
 		Networks:      networks,
 		WaitingFor: NewExitCodeStrategy().WithExitCode(0).
-			WithPollInterval(1 * time.Second).WithTimeout(10 * time.Second),
+			WithPollInterval(1 * time.Second).WithTimeout(g.StartupTimeout),
 		Entrypoint: []string{"sh", "/init.sh"},
 		Files: []tc.ContainerFile{
 			{

docker/test_env/ethereum_env.go

@@ -432,17 +432,21 @@ func (en *EthereumNetwork) startEth2() (blockchain.EVMNetwork, RpcProvider, erro
 
 	opts := en.getExecutionLayerEnvComponentOpts()
 
+	chainReadyWaitTime := en.EthereumChainConfig.GetDefaultWaitDuration()
 	var client ExecutionClient
 	var clientErr error
 	switch *en.ExecutionLayer {
 	case config.ExecutionLayer_Geth:
 		client, clientErr = NewGethEth2(dockerNetworks, en.EthereumChainConfig, generatedDataHostDir, config.ConsensusLayer_Prysm, opts...)
 	case config.ExecutionLayer_Nethermind:
 		client, clientErr = NewNethermindEth2(dockerNetworks, en.EthereumChainConfig, generatedDataHostDir, config.ConsensusLayer_Prysm, opts...)
+		chainReadyWaitTime = chainReadyWaitTime * 2
 	case config.ExecutionLayer_Erigon:
 		client, clientErr = NewErigonEth2(dockerNetworks, en.EthereumChainConfig, generatedDataHostDir, config.ConsensusLayer_Prysm, opts...)
+		chainReadyWaitTime = chainReadyWaitTime * 2
 	case config.ExecutionLayer_Besu:
 		client, clientErr = NewBesuEth2(dockerNetworks, en.EthereumChainConfig, generatedDataHostDir, config.ConsensusLayer_Prysm, opts...)
+		chainReadyWaitTime = chainReadyWaitTime * 2
 	default:
 		return blockchain.EVMNetwork{}, RpcProvider{}, fmt.Errorf(MsgUnsupportedExecutionLayer, *en.ExecutionLayer)
 	}
@@ -481,7 +485,7 @@ func (en *EthereumNetwork) startEth2() (blockchain.EVMNetwork, RpcProvider, erro
 		return blockchain.EVMNetwork{}, RpcProvider{}, errors.Wrapf(err, "failed to start validator")
 	}
 
-	err = client.WaitUntilChainIsReady(testcontext.Get(en.t), en.EthereumChainConfig.GetDefaultWaitDuration())
+	err = client.WaitUntilChainIsReady(testcontext.Get(en.t), chainReadyWaitTime)
 	if err != nil {
 		return blockchain.EVMNetwork{}, RpcProvider{}, errors.Wrapf(err, "failed to wait for chain to be ready")
 	}

docker/test_env/genesis_generator.go

@@ -38,6 +38,7 @@ func NewEthGenesisGenerator(chainConfig config.EthereumChainConfig, generatedDat
 			ContainerName:    fmt.Sprintf("%s-%s", "eth-genesis-generator", uuid.NewString()[0:8]),
 			ContainerImage:   parts[0],
 			ContainerVersion: parts[1],
+			StartupTimeout:   30 * time.Second,
 		},
 		chainConfig:          chainConfig,
 		generatedDataHostDir: generatedDataHostDir,
@@ -131,9 +132,8 @@ func (g *EthGenesisGeneretor) getContainerRequest(networks []string) (*tc.Contai
 		WaitingFor: tcwait.ForAll(
 			tcwait.ForLog("+ terminalTotalDifficulty=0"),
 			tcwait.ForLog("+ sed -i 's/TERMINAL_TOTAL_DIFFICULTY:.*/TERMINAL_TOTAL_DIFFICULTY: 0/' /data/metadata/config.yaml").
-				WithStartupTimeout(20*time.Second).
 				WithPollInterval(1*time.Second),
-		),
+		).WithStartupTimeoutDefault(g.StartupTimeout),
 		Cmd: []string{"all"},
 		Files: []tc.ContainerFile{
 			{

docker/test_env/geth_eth1.go

@@ -28,6 +28,7 @@ func NewGethEth1(networks []string, chainConfig *config.EthereumChainConfig, opt
 			Networks:         networks,
 			ContainerImage:   parts[0],
 			ContainerVersion: parts[1],
+			StartupTimeout:   2 * time.Minute,
 		},
 		chainConfig:     chainConfig,
 		l:               logging.GetTestLogger(nil),
@@ -122,10 +123,9 @@ func (g *Geth) getEth1ContainerRequest() (*tc.ContainerRequest, error) {
 			NewHTTPStrategy("/", NatPort(DEFAULT_EVM_NODE_HTTP_PORT)),
 			tcwait.ForLog(websocketMsg),
 			tcwait.ForLog("Started P2P networking").
-				WithStartupTimeout(120*time.Second).
 				WithPollInterval(1*time.Second),
-			NewWebSocketStrategy(NatPort(DEFAULT_EVM_NODE_WS_PORT), g.l),
-		),
+			NewWebSocketStrategy(NatPort(DEFAULT_EVM_NODE_WS_PORT), g.l)).
+			WithStartupTimeoutDefault(g.StartupTimeout),
 		Entrypoint: entrypoint,
 		Files: []tc.ContainerFile{
 			{

docker/test_env/geth_eth2.go

@@ -28,6 +28,7 @@ func NewGethEth2(networks []string, chainConfig *config.EthereumChainConfig, gen
 			Networks:         networks,
 			ContainerImage:   parts[0],
 			ContainerVersion: parts[1],
+			StartupTimeout:   2 * time.Minute,
 		},
 		chainConfig:          chainConfig,
 		generatedDataHostDir: generatedDataHostDir,
@@ -73,9 +74,8 @@ func (g *Geth) getEth2ContainerRequest() (*tc.ContainerRequest, error) {
 		ExposedPorts:  []string{NatPortFormat(DEFAULT_EVM_NODE_HTTP_PORT), NatPortFormat(DEFAULT_EVM_NODE_WS_PORT), NatPortFormat(ETH2_EXECUTION_PORT)},
 		WaitingFor: tcwait.ForAll(
 			tcwait.ForLog("WebSocket enabled").
-				WithStartupTimeout(120 * time.Second).
-				WithPollInterval(1 * time.Second),
-		),
+				WithPollInterval(1 * time.Second)).
+			WithStartupTimeoutDefault(g.StartupTimeout),
 		Entrypoint: []string{
 			"sh",
 			"/init.sh",

docker/test_env/kafka.go

@@ -56,8 +56,9 @@ func NewKafka(networks []string) *Kafka {
 	}
 	return &Kafka{
 		EnvComponent: EnvComponent{
-			ContainerName: containerName,
-			Networks:      networks,
+			ContainerName:  containerName,
+			Networks:       networks,
+			StartupTimeout: 1 * time.Minute,
 		},
 		EnvVars: defaultEnvVars,
 		l:       log.Logger,
@@ -169,7 +170,7 @@ func (k *Kafka) getContainerRequest() (tc.ContainerRequest, error) {
 		Env:          k.EnvVars,
 		Networks:     k.Networks,
 		WaitingFor: tcwait.ForLog("[KafkaServer id=1] started (kafka.server.KafkaServer)").
-			WithStartupTimeout(30 * time.Second).
+			WithStartupTimeout(k.StartupTimeout).
 			WithPollInterval(100 * time.Millisecond),
 	}, nil
 }

docker/test_env/killgrave.go

@@ -82,8 +82,9 @@ type KillgraveAdapterResult struct {
 func NewKillgrave(networks []string, impostersDirectoryPath string, opts ...EnvComponentOption) *Killgrave {
 	k := &Killgrave{
 		EnvComponent: EnvComponent{
-			ContainerName: fmt.Sprintf("%s-%s", "killgrave", uuid.NewString()[0:3]),
-			Networks:      networks,
+			ContainerName:  fmt.Sprintf("%s-%s", "killgrave", uuid.NewString()[0:3]),
+			Networks:       networks,
+			StartupTimeout: 2 * time.Minute,
 		},
 		InternalPort:  "3000",
 		impostersPath: impostersDirectoryPath,
@@ -175,7 +176,7 @@ func (k *Killgrave) getContainerRequest() (tc.ContainerRequest, error) {
 				ReadOnly: false,
 			})
 		},
-		WaitingFor: wait.ForLog("The fake server is on tap now"),
+		WaitingFor: wait.ForLog("The fake server is on tap now").WithStartupTimeout(k.StartupTimeout),
 		LifecycleHooks: []tc.ContainerLifecycleHooks{
 			{
 				PostStarts: k.PostStartsHooks,

docker/test_env/mockserver.go

@@ -35,8 +35,9 @@ type MockServer struct {
 func NewMockServer(networks []string, opts ...EnvComponentOption) *MockServer {
 	ms := &MockServer{
 		EnvComponent: EnvComponent{
-			ContainerName: fmt.Sprintf("%s-%s", "mockserver", uuid.NewString()[0:8]),
-			Networks:      networks,
+			ContainerName:  fmt.Sprintf("%s-%s", "mockserver", uuid.NewString()[0:8]),
+			Networks:       networks,
+			StartupTimeout: 1 * time.Minute,
 		},
 		l: log.Logger,
 	}
@@ -120,8 +121,7 @@ func (ms *MockServer) getContainerRequest() (tc.ContainerRequest, error) {
 		},
 		Networks: ms.Networks,
 		WaitingFor: tcwait.ForLog("INFO 1080 started on port: 1080").
-			WithStartupTimeout(30 * time.Second).
-			WithPollInterval(100 * time.Millisecond),
+			WithPollInterval(100 * time.Millisecond).WithStartupTimeout(ms.StartupTimeout),
 		LifecycleHooks: []tc.ContainerLifecycleHooks{
 			{
 				PostStarts: ms.PostStartsHooks,

docker/test_env/nethermind_eth1.go

@@ -28,6 +28,7 @@ func NewNethermindEth1(networks []string, chainConfig *config.EthereumChainConfi
 			Networks:         networks,
 			ContainerImage:   parts[0],
 			ContainerVersion: parts[1],
+			StartupTimeout:   2 * time.Minute,
 		},
 		chainConfig:     chainConfig,
 		l:               logging.GetTestLogger(nil),
@@ -117,9 +118,8 @@ func (g *Nethermind) getEth1ContainerRequest() (*tc.ContainerRequest, error) {
 		ExposedPorts: []string{NatPortFormat(DEFAULT_EVM_NODE_HTTP_PORT), NatPortFormat(DEFAULT_EVM_NODE_WS_PORT)},
 		WaitingFor: tcwait.ForAll(
 			tcwait.ForLog("Nethermind initialization completed").
-				WithStartupTimeout(120 * time.Second).
 				WithPollInterval(1 * time.Second),
-		),
+		).WithStartupTimeoutDefault(g.StartupTimeout),
 		Cmd: []string{
 			"--config=/none.cfg",
 			"--Init.ChainSpecPath=/chainspec.json",

docker/test_env/nethermind_eth2.go

@@ -26,6 +26,7 @@ func NewNethermindEth2(networks []string, chainConfig *config.EthereumChainConfi
 			Networks:         networks,
 			ContainerImage:   parts[0],
 			ContainerVersion: parts[1],
+			StartupTimeout:   2 * time.Minute,
 		},
 		generatedDataHostDir: generatedDataHostDir,
 		chainConfig:          chainConfig,
@@ -100,9 +101,8 @@ func (g *Nethermind) getEth2ContainerRequest() (*tc.ContainerRequest, error) {
 		ExposedPorts: []string{NatPortFormat(DEFAULT_EVM_NODE_HTTP_PORT), NatPortFormat(DEFAULT_EVM_NODE_WS_PORT), NatPortFormat(ETH2_EXECUTION_PORT)},
 		WaitingFor: tcwait.ForAll(
 			tcwait.ForLog("Nethermind initialization completed").
-				WithStartupTimeout(120 * time.Second).
 				WithPollInterval(1 * time.Second),
-		),
+		).WithStartupTimeoutDefault(g.StartupTimeout),
 		Cmd: command,
 		Files: []tc.ContainerFile{
 			{

docker/test_env/postgres.go

@@ -94,6 +94,7 @@ func NewPostgresDb(networks []string, opts ...PostgresDbOption) (*PostgresDb, er
 			ContainerVersion: imageParts[1],
 			ContainerEnvs:    map[string]string{},
 			Networks:         networks,
+			StartupTimeout:   2 * time.Minute,
 		},
 		User:         "postgres",
 		Password:     "mysecretpassword",
@@ -194,7 +195,7 @@ func (pg *PostgresDb) getContainerRequest() *tc.ContainerRequest {
 		Networks:     pg.Networks,
 		WaitingFor: tcwait.ForExec([]string{"psql", "-h", "127.0.0.1",
 			"-U", pg.User, "-c", "select", "1", "-d", pg.DbName}).
-			WithStartupTimeout(20 * time.Second),
+			WithStartupTimeout(pg.StartupTimeout),
 		LifecycleHooks: []tc.ContainerLifecycleHooks{
 			{
 				PostStarts: pg.PostStartsHooks,

docker/test_env/prysm.go

@@ -49,6 +49,7 @@ func NewPrysmBeaconChain(networks []string, chainConfig *config.EthereumChainCon
 			Networks:         networks,
 			ContainerImage:   parts[0],
 			ContainerVersion: parts[1],
+			StartupTimeout:   2 * time.Minute,
 		},
 		chainConfig:              chainConfig,
 		generatedDataHostDir:     customConfigDataDir,
@@ -114,14 +115,19 @@ func (g *PrysmBeaconChain) StartContainer() error {
 }
 
 func (g *PrysmBeaconChain) getContainerRequest(networks []string) (*tc.ContainerRequest, error) {
+	timeout := g.chainConfig.GetDefaultWaitDuration()
+	if g.StartupTimeout < timeout {
+		timeout = g.StartupTimeout
+	}
+
 	return &tc.ContainerRequest{
 		Name:          g.ContainerName,
 		Image:         g.GetImageWithVersion(),
 		ImagePlatform: "linux/amd64",
 		Networks:      networks,
 		WaitingFor: tcwait.ForAll(
 			tcwait.ForLog("Starting beacon node").
-				WithStartupTimeout(g.chainConfig.GetDefaultWaitDuration()).
+				WithStartupTimeout(timeout).
 				WithPollInterval(2 * time.Second),
 		),
 		Cmd: []string{

docker/test_env/schema_registry.go

@@ -34,8 +34,9 @@ func NewSchemaRegistry(networks []string) *SchemaRegistry {
 	}
 	return &SchemaRegistry{
 		EnvComponent: EnvComponent{
-			ContainerName: fmt.Sprintf("schema-registry-%s", id.String()),
-			Networks:      networks,
+			ContainerName:  fmt.Sprintf("schema-registry-%s", id.String()),
+			Networks:       networks,
+			StartupTimeout: 1 * time.Minute,
 		},
 		EnvVars: defaultEnvVars,
 	}
@@ -117,7 +118,7 @@ func (r *SchemaRegistry) getContainerRequest() (tc.ContainerRequest, error) {
 		Env:          r.EnvVars,
 		Networks:     r.Networks,
 		WaitingFor: tcwait.ForLog("INFO Server started, listening for requests").
-			WithStartupTimeout(30 * time.Second).
+			WithStartupTimeout(r.StartupTimeout).
 			WithPollInterval(100 * time.Millisecond),
 	}, nil
 }