Remove common secret config keys and BASE64_NETWORK_CONFIG env (#1086)

config/README.md

@@ -250,8 +250,6 @@ arbitrum_goerli = ["1810868fc221b9f50b5b3e0186d8a5f343f892e51ce12a9e818f936ec0b6
 new_network = ["ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"]
 ```
 
-It not only stores the configuration of selected networks and RPC endpoints and wallet keys, but via `Default()` method provides a way to read from env var `BASE64_NETWORK_CONFIG` a base64-ed configuration of RPC endpoints and wallet keys. This could prove useful in the CI, where we could store as a secret a default configuration of stable endpoints, so that when we run a test job all that we have to provide is the network name and nothing more as it's pretty tedious, especially for on-demand jobs, to have to pass the whole RPC/wallet configuration every time you run it.
-
 If in your product config you want to support case-insensitive network names and map keys remember to run `NetworkConfig.UpperCaseNetworkNames()` on your config before using it.
 
 ## Providing custom values in the CI
@@ -350,7 +348,7 @@ It's easy. All you need to do is:
 - Base64 it: `cat your.toml | base64`
 - Set the base64 result as `BASE64_CONFIG_OVERRIDE` environment variable.
 
-Both `BASE64_CONFIG_OVERRIDE` and `BASE64_NETWORK_CONFIG` will be automatically forwarded to k8s (as long as they are set and available to the test process), when creating the environment programmatically via `environment.New()`.
+`BASE64_CONFIG_OVERRIDE` will be automatically forwarded to k8s (as long as it is set and available to the test process), when creating the environment programmatically via `environment.New()`.
 
 Quick example:
 

config/chainlink_image.go

@@ -5,7 +5,7 @@ import (
 )
 
 type ChainlinkImageConfig struct {
-	Image           *string `toml:"image"`
+	Image           *string `toml:"-"`
 	Version         *string `toml:"version"`
 	PostgresVersion *string `toml:"postgres_version,omitempty"`
 }

config/examples/testconfig.go

@@ -179,10 +179,9 @@ func (c *TestConfig) readNetworkConfiguration() error {
 	if c.Network == nil {
 		c.Network = &ctf_config.NetworkConfig{}
 	}
-	err := c.Network.Default()
-	if err != nil {
-		return errors.Wrapf(err, "error reading default network config")
-	}
+
+	c.Network.UpperCaseNetworkNames()
+	c.Network.OverrideURLsAndKeysFromEVMNetwork()
 
 	// this is the only value we need to generate dynamically before starting a new simulated chain
 	if c.PrivateEthereumNetwork != nil && c.PrivateEthereumNetwork.EthereumChainConfig != nil {

config/logging.go

@@ -13,7 +13,7 @@ type LoggingConfig struct {
 	TestLogCollect         *bool            `toml:"test_log_collect,omitempty"`
 	ShowHTMLCoverageReport *bool            `toml:"show_html_coverage_report,omitempty"` // Show reports with go coverage data
 	RunId                  *string          `toml:"run_id,omitempty"`
-	Loki                   *LokiConfig      `toml:"Loki,omitempty"`
+	Loki                   *LokiConfig      `toml:"-"`
 	Grafana                *GrafanaConfig   `toml:"Grafana,omitempty"`
 	LogStream              *LogStreamConfig `toml:"LogStream,omitempty"`
 }
@@ -68,10 +68,10 @@ func (l *LogStreamConfig) Validate() error {
 }
 
 type LokiConfig struct {
-	TenantId    *string `toml:"tenant_id"`
-	Endpoint    *string `toml:"endpoint"`
-	BasicAuth   *string `toml:"basic_auth_secret"`
-	BearerToken *string `toml:"bearer_token_secret"`
+	TenantId    *string `toml:"-"`
+	Endpoint    *string `toml:"-"`
+	BasicAuth   *string `toml:"-"`
+	BearerToken *string `toml:"-"`
 }
 
 // Validate checks that the loki config is valid, which means that
@@ -93,7 +93,7 @@ type GrafanaConfig struct {
 	BaseUrl      *string `toml:"base_url"`
 	DashboardUrl *string `toml:"dashboard_url"`
 	DashboardUID *string `toml:"dashboard_uid"` // UID of the dashboard to put annotations on
-	BearerToken  *string `toml:"bearer_token_secret"`
+	BearerToken  *string `toml:"-"`
 }
 
 // Validate checks that the grafana config is valid, which means that

config/network.go

@@ -1,22 +1,14 @@
 package config
 
 import (
-	"encoding/base64"
 	"errors"
 	"fmt"
-	"os"
 	"strings"
 
-	"github.com/pelletier/go-toml/v2"
-
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
 	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 )
 
-const (
-	Base64NetworkConfigEnvVarName = "BASE64_NETWORK_CONFIG"
-)
-
 type AnvilConfig struct {
 	URL                *string                  `toml:"url,omitempty"`                   // Needed if you want to fork a network. URL is the URL of the node to fork from. Refer to https://book.getfoundry.sh/reference/anvil/#options
 	BlockNumber        *int64                   `toml:"block_number,omitempty"`          // Needed if fork URL is provided for forking. BlockNumber is the block number to fork from. Refer to https://book.getfoundry.sh/reference/anvil/#options
@@ -90,42 +82,6 @@ func (n NetworkConfig) IsSimulatedGethSelected() bool {
 	return false
 }
 
-func (n *NetworkConfig) applySecrets() error {
-	encodedEndpoints, isSet := os.LookupEnv(Base64NetworkConfigEnvVarName)
-	if !isSet {
-		return nil
-	}
-
-	err := n.applyBase64Encoded(encodedEndpoints)
-	if err != nil {
-		return fmt.Errorf("error reading network encoded endpoints: %w", err)
-	}
-
-	return nil
-}
-
-func (n *NetworkConfig) applyDecoded(configDecoded string) error {
-	if configDecoded == "" {
-		return nil
-	}
-
-	var cfg NetworkConfig
-	err := toml.Unmarshal([]byte(configDecoded), &cfg)
-	if err != nil {
-		return fmt.Errorf("error unmarshalling network config: %w", err)
-	}
-
-	cfg.UpperCaseNetworkNames()
-
-	err = n.applyDefaults(&cfg)
-	if err != nil {
-		return fmt.Errorf("error applying overrides from decoded network config file to config: %w", err)
-	}
-	n.OverrideURLsAndKeysFromEVMNetwork()
-
-	return nil
-}
-
 // OverrideURLsAndKeysFromEVMNetwork applies the URLs and keys from the EVMNetworks to the NetworkConfig
 // it overrides the URLs and Keys present in RpcHttpUrls, RpcWsUrls and WalletKeys in the NetworkConfig
 // with the URLs and Keys provided in the EVMNetworks
@@ -158,19 +114,6 @@ func (n *NetworkConfig) OverrideURLsAndKeysFromEVMNetwork() {
 	}
 }
 
-func (n *NetworkConfig) applyBase64Encoded(configEncoded string) error {
-	if configEncoded == "" {
-		return nil
-	}
-
-	decoded, err := base64.StdEncoding.DecodeString(configEncoded)
-	if err != nil {
-		return err
-	}
-
-	return n.applyDecoded(string(decoded))
-}
-
 // Validate checks if all required fields are set, meaning that there must be at least
 // 1 selected network and unless it's a simulated network, there must be at least 1
 // rpc endpoint for HTTP and WS and 1 private key for funding wallet
@@ -251,79 +194,3 @@ func (n *NetworkConfig) UpperCaseNetworkNames() {
 		n.SelectedNetworks[i] = strings.ToUpper(network)
 	}
 }
-
-func (n *NetworkConfig) applyDefaults(defaults *NetworkConfig) error {
-	if defaults == nil {
-		return nil
-	}
-
-	if defaults.SelectedNetworks != nil {
-		n.SelectedNetworks = defaults.SelectedNetworks
-	}
-	if defaults.EVMNetworks != nil {
-		if n.EVMNetworks == nil || len(n.EVMNetworks) == 0 {
-			n.EVMNetworks = defaults.EVMNetworks
-		} else {
-			for network, cfg := range defaults.EVMNetworks {
-				if _, ok := n.EVMNetworks[network]; !ok {
-					n.EVMNetworks[network] = cfg
-				}
-			}
-		}
-	}
-	if defaults.AnvilConfigs != nil {
-		if n.AnvilConfigs == nil || len(n.AnvilConfigs) == 0 {
-			n.AnvilConfigs = defaults.AnvilConfigs
-		} else {
-			for network, cfg := range defaults.AnvilConfigs {
-				if _, ok := n.AnvilConfigs[network]; !ok {
-					n.AnvilConfigs[network] = cfg
-				}
-			}
-		}
-	}
-	if defaults.RpcHttpUrls != nil {
-		if n.RpcHttpUrls == nil || len(n.RpcHttpUrls) == 0 {
-			n.RpcHttpUrls = defaults.RpcHttpUrls
-		} else {
-			for network, urls := range defaults.RpcHttpUrls {
-				if _, ok := n.RpcHttpUrls[network]; !ok {
-					n.RpcHttpUrls[network] = urls
-				}
-			}
-		}
-	}
-	if defaults.RpcWsUrls != nil {
-		if n.RpcWsUrls == nil || len(n.RpcWsUrls) == 0 {
-			n.RpcWsUrls = defaults.RpcWsUrls
-		} else {
-			for network, urls := range defaults.RpcWsUrls {
-				if _, ok := n.RpcWsUrls[network]; !ok {
-					n.RpcWsUrls[network] = urls
-				}
-			}
-		}
-	}
-	if defaults.WalletKeys != nil {
-		if n.WalletKeys == nil || len(n.WalletKeys) == 0 {
-			n.WalletKeys = defaults.WalletKeys
-		} else {
-
-			for network, keys := range defaults.WalletKeys {
-				if _, ok := n.WalletKeys[network]; !ok {
-					n.WalletKeys[network] = keys
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-// Default applies default values to the network config after reading it
-// from BASE64_NETWORK_CONFIG env var. It will only fill in the gaps, not override
-// meaning that if you provided WS RPC endpoint in your network config, but not the
-// HTTP one, then only HTTP will be taken from default config (provided it's there)
-func (n *NetworkConfig) Default() error {
-	return n.applySecrets()
-}

config/pyroscope.go

@@ -9,9 +9,9 @@ import (
 
 type PyroscopeConfig struct {
 	Enabled     *bool   `toml:"enabled"`
-	ServerUrl   *string `toml:"server_url"`
-	Key         *string `toml:"key_secret"`
-	Environment *string `toml:"environment"`
+	ServerUrl   *string `toml:"-"`
+	Key         *string `toml:"-"`
+	Environment *string `toml:"-"`
 }
 
 // Validate checks that the pyroscope config is valid, which means that

config/testconfig.go

@@ -314,7 +314,7 @@ func LoadSecretEnvsFromFiles() error {
 				logger.Debug().Msgf("No test secrets file found at %s", path)
 				continue
 			}
-			return errors.Wrapf(err, "error reading test secrets file at %s", path)
+			return fmt.Errorf("error reading test secrets file at %s", path)
 		}
 
 		// Set env vars from file only if they are not already set

config/testconfig_test.go

@@ -25,6 +25,8 @@ func TestReadConfigValuesFromEnvVars(t *testing.T) {
 				os.Setenv("E2E_TEST_GROUP3_RPC_WS_URL_1", "wsUrl1")
 				os.Setenv("E2E_TEST_CHAINLINK_IMAGE", "imageValue")
 				os.Setenv("E2E_TEST_PYROSCOPE_ENABLED", "true")
+				os.Setenv("E2E_TEST_PYROSCOPE_SERVER_URL", "serverUrl")
+				os.Setenv("E2E_TEST_PYROSCOPE_KEY", "serverKey")
 				os.Setenv("E2E_TEST_SELECTED_NETWORK", "networkA,networkB")
 			},
 			cleanupFunc: func() {
@@ -33,6 +35,8 @@ func TestReadConfigValuesFromEnvVars(t *testing.T) {
 				os.Unsetenv("E2E_TEST_GROUP3_RPC_WS_URL_1")
 				os.Unsetenv("E2E_TEST_CHAINLINK_IMAGE")
 				os.Unsetenv("E2E_TEST_PYROSCOPE_ENABLED")
+				os.Unsetenv("E2E_TEST_PYROSCOPE_SERVER_URL")
+				os.Unsetenv("E2E_TEST_PYROSCOPE_KEY")
 				os.Unsetenv("E2E_TEST_SELECTED_NETWORK")
 			},
 			expectedConfig: TestConfig{
@@ -42,7 +46,7 @@ func TestReadConfigValuesFromEnvVars(t *testing.T) {
 					RpcHttpUrls:      map[string][]string{"GROUP2": {"httpUrl1"}},
 					RpcWsUrls:        map[string][]string{"GROUP3": {"wsUrl1"}},
 				},
-				Pyroscope:      &PyroscopeConfig{Enabled: ptr.Ptr[bool](true)},
+				Pyroscope:      &PyroscopeConfig{Enabled: ptr.Ptr[bool](true), ServerUrl: ptr.Ptr[string]("serverUrl"), Key: ptr.Ptr[string]("serverKey")},
 				ChainlinkImage: &ChainlinkImageConfig{Image: newString("imageValue")},
 			},
 		},

k8s/config/overrides.go

@@ -105,10 +105,6 @@ const (
 	EnvBase64ConfigOverride             = "BASE64_CONFIG_OVERRIDE"
 	EnvBase64ConfigOverriderDescription = "Base64-encoded TOML config (should contain at least chainlink image and version)"
 	EnvBase64ConfigOverrideExample      = "W0NoYWlubGlua0ltYWdlXQppbWFnZT0icHVibGljLmVjci5hd3MvY2hhaW5saW5rL2NoYWlubGluayIKdmVyc2lvbj0iMi43LjEtYXV0b21hdGlvbi0yMDIzMTEyNyIKCltBdXRvbWF0aW9uXQpbQXV0b21hdGlvbi5HZW5lcmFsXQpkdXJhdGlvbj0yMDAK"
-
-	EnvBase64NetworkConfig            = "BASE64_NETWORK_CONFIG"
-	EnvBase64NetworkConfigDescription = "Base64-encoded TOML network config with default RPC/WS endpoints and private keys"
-	EnvBase64NetworkConfigExample     = "Nope :-)"
 )
 
 var (

k8s/environment/runner.go

@@ -399,7 +399,6 @@ func jobEnvVars(props *Props) *[]*k8s.EnvVar {
 		config.EnvVarInternalDockerRepo,
 		config.EnvVarLocalCharts,
 		config.EnvBase64ConfigOverride,
-		config.EnvBase64NetworkConfig,
 	}
 	for _, k := range lookups {
 		v, success := os.LookupEnv(k)