diff --git a/README.md b/README.md index 1e51ef3c4..2670d2aab 100644 --- a/README.md +++ b/README.md @@ -416,3 +416,10 @@ We have two ways to add new images to the ecr. The first two requirements are th 1. If it does not have version numbers or is gcr then you can add it [here](./scripts/mirror.json) 2. You can add to the [mirror matrix](./.github/workflows/update-internal-mirrors.yaml) the new image name and an expression to get the latest versions added when the workflow runs. You can check the postgres one used in there for an example but basically the expression should filter out only the latest image or 2 for that particular version when calling the dockerhub endpoint, example curl call `curl -s "https://hub.docker.com/v2/repositories/${image_name}/tags/?page_size=100" | jq -r '.results[].name' | grep -E ${image_expression}` where image_name could be `library/postgres` and image_expression could be `'^[0-9]+\.[0-9]+$'`. Adding your ecr to this matrix should make sure we always have the latest versions for that expression. + +## Debugging HTTP and RPC calls + +```bash +export SETH_LOG_LEVEL=info +export RESTY_DEBUG=true +``` diff --git a/blockchain/config.go b/blockchain/config.go index 5b690e8f2..591f0e402 100644 --- a/blockchain/config.go +++ b/blockchain/config.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "strings" "time" @@ -82,6 +83,9 @@ type EVMNetwork struct { // Only used internally, do not set URL string `ignored:"true"` + + // Only used internally, do not set + Headers http.Header `toml:"evm_headers" json:"evm_headers"` } // LoadNetworkFromEnvironment loads an EVM network from default environment variables. Helpful in soak tests diff --git a/blockchain/ethereum.go b/blockchain/ethereum.go index 4867a2645..3565f2400 100644 --- a/blockchain/ethereum.go +++ b/blockchain/ethereum.go @@ -1,3 +1,6 @@ +/* +This should be removed when we migrate all Ethereum client code to Seth +*/ package blockchain // Contains implementations for multi and single node ethereum clients @@ -32,7 +35,10 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/utils/conversions" ) -const MaxTimeoutForFinality = 15 * time.Minute +const ( + MaxTimeoutForFinality = 15 * time.Minute + DefaultDialTimeout = 1 * time.Minute +) // EthereumClient wraps the client and the BlockChain network to interact with an EVM based Blockchain type EthereumClient struct { @@ -65,14 +71,13 @@ func newEVMClient(networkSettings EVMNetwork, logger zerolog.Logger) (EVMClient, Bool("Supports EIP-1559", networkSettings.SupportsEIP1559). Bool("Finality Tag", networkSettings.FinalityTag). Msg("Connecting client") - cl, err := ethclient.Dial(networkSettings.URL) - if err != nil { - return nil, err - } - raw, err := rpc.Dial(networkSettings.URL) + ctx, cancel := context.WithTimeout(context.Background(), DefaultDialTimeout) + defer cancel() + raw, err := rpc.DialOptions(ctx, networkSettings.URL) if err != nil { return nil, err } + cl := ethclient.NewClient(raw) ec := &EthereumClient{ NetworkConfig: networkSettings, diff --git a/client/explorer.go b/client/explorer.go index 8db60737a..0ebc9a932 100644 --- a/client/explorer.go +++ b/client/explorer.go @@ -15,11 +15,11 @@ type ExplorerClient struct { } // NewExplorerClient creates a new explorer mock client -func NewExplorerClient(cfg *ExplorerConfig) *ExplorerClient { +func NewExplorerClient(cfg *ExplorerConfig) (*ExplorerClient, error) { return &ExplorerClient{ Config: cfg, APIClient: resty.New().SetBaseURL(cfg.URL), - } + }, nil } // PostAdminNodes is used to exercise the POST /api/v1/admin/nodes endpoint diff --git a/client/kafka_rest.go b/client/kafka_rest.go index e698fef42..9ec4e1eb4 100644 --- a/client/kafka_rest.go +++ b/client/kafka_rest.go @@ -19,11 +19,11 @@ type KafkaRestConfig struct { } // NewKafkaRestClient creates a new KafkaRestClient -func NewKafkaRestClient(cfg *KafkaRestConfig) *KafkaRestClient { +func NewKafkaRestClient(cfg *KafkaRestConfig) (*KafkaRestClient, error) { return &KafkaRestClient{ Config: cfg, APIClient: resty.New().SetBaseURL(cfg.URL), - } + }, nil } // GetTopics Get a list of Kafka topics. diff --git a/client/mockserver.go b/client/mockserver.go index 61e49aa3b..0c2a74d47 100644 --- a/client/mockserver.go +++ b/client/mockserver.go @@ -1,8 +1,10 @@ package client import ( + "crypto/tls" "fmt" "net/http" + "os" "strings" "github.com/go-resty/resty/v2" @@ -22,32 +24,39 @@ type MockserverClient struct { type MockserverConfig struct { LocalURL string ClusterURL string + Headers map[string]string } // ConnectMockServer creates a connection to a deployed mockserver in the environment -func ConnectMockServer(e *environment.Environment) (*MockserverClient, error) { +func ConnectMockServer(e *environment.Environment) *MockserverClient { c := NewMockserverClient(&MockserverConfig{ LocalURL: e.URLs[mockserver.LocalURLsKey][0], ClusterURL: e.URLs[mockserver.InternalURLsKey][0], }) - return c, nil + return c } // ConnectMockServerURL creates a connection to a mockserver at a given url, should only be used for inside K8s tests -func ConnectMockServerURL(url string) (*MockserverClient, error) { +func ConnectMockServerURL(url string) *MockserverClient { c := NewMockserverClient(&MockserverConfig{ LocalURL: url, ClusterURL: url, }) - return c, nil + return c } // NewMockserverClient returns a mockserver client func NewMockserverClient(cfg *MockserverConfig) *MockserverClient { log.Debug().Str("Local URL", cfg.LocalURL).Str("Remote URL", cfg.ClusterURL).Msg("Connected to MockServer") + isDebug := os.Getenv("RESTY_DEBUG") == "true" return &MockserverClient{ - Config: cfg, - APIClient: resty.New().SetBaseURL(cfg.LocalURL), + Config: cfg, + APIClient: resty.New(). + SetBaseURL(cfg.LocalURL). + SetHeaders(cfg.Headers). + SetDebug(isDebug). + //nolint + SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}), } } @@ -114,7 +123,7 @@ func (em *MockserverClient) SetValuePath(path string, v int) error { initializers := []HttpInitializer{initializer} resp, err := em.APIClient.R().SetBody(&initializers).Put("/expectation") if resp.StatusCode() != http.StatusCreated { - err = fmt.Errorf("status code expected %d got %d", http.StatusCreated, resp.StatusCode()) + err = fmt.Errorf("status code expected %d got %d, err: %s", http.StatusCreated, resp.StatusCode(), err) } return err } diff --git a/crib/README.md b/crib/README.md new file mode 100644 index 000000000..34f9a5f7c --- /dev/null +++ b/crib/README.md @@ -0,0 +1,25 @@ +### CRIB Connector + +This is a simple CRIB connector for OCRv1 CRIB +This code is temporary and may be removed in the future if connection logic will be simplified with [ARC](https://github.com/actions/actions-runner-controller) + +## Example + +Go to the [CRIB](https://github.com/smartcontractkit/crib) repository and spin up a cluster. + +```shell +./scripts/cribbit.sh crib-oh-my-crib +devspace deploy --debug --profile local-dev-simulated-core-ocr1 +``` + +## Run an example test + +```shell +export CRIB_NAMESPACE=crib-oh-my-crib +export CRIB_NETWORK=geth # only "geth" is supported for now +export CRIB_NODES=5 # min 5 nodes +#export SETH_LOG_LEVEL=debug # these two can be enabled to debug connection issues +#export RESTY_DEBUG=true +export GAP_URL=https://localhost:8080/primary # only applicable in CI, unset the var to connect locally +go test -v -run TestCRIB +``` diff --git a/crib/config.go b/crib/config.go new file mode 100644 index 000000000..af3485ed5 --- /dev/null +++ b/crib/config.go @@ -0,0 +1,137 @@ +package crib + +import ( + "fmt" + "net/http" + "os" + "strconv" + + "github.com/pkg/errors" + + "github.com/smartcontractkit/chainlink-testing-framework/logging" +) + +const ( + /* + These are constants for simulated CRIB that should never change + Ideally, they should be placed into CRIB repository, however, for simplicity we keep all environment connectors in CTF + CRIB: https://github.com/smartcontractkit/crib/tree/main/core + Core Chart: https://github.com/smartcontractkit/infra-charts/tree/main/chainlink-cluster + */ + MockserverCRIBTemplate = "https://%s-mockserver%s" + InternalNodeDNSTemplate = "app-node%d" + IngressNetworkWSURLTemplate = "wss://%s-geth-%d-ws%s" + IngressNetworkHTTPURLTemplate = "https://%s-geth-%d-http%s" + // DefaultSimulatedPrivateKey is a first key used for Geth/Hardhat/Anvil + DefaultSimulatedPrivateKey = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + // DefaultSimulatedNetworkChainID is a default chainID we use for Geth/Hardhat/Anvil + DefaultSimulatedNetworkChainID = 1337 + HostHeader = "X-Original-Host" +) + +// ConnectionVars common K8s connection vars +type ConnectionVars struct { + IngressSuffix string + Namespace string + Network string + Nodes int +} + +// CoreDONConnectionConfig Chainlink DON connection config +type CoreDONConnectionConfig struct { + *ConnectionVars + PrivateKeys []string + NodeURLs []string + NodeInternalDNS []string + NodeHeaders []map[string]string + BlockchainNodeHeaders http.Header + MockserverHeaders map[string]string + ChainID int64 + NetworkWSURL string + NetworkHTTPURL string + MockserverURL string +} + +// CoreDONSimulatedConnection returns all vars required to connect to core DON Simulated CRIB +// connects in CI via GAP if GAP_URL is provided +func CoreDONSimulatedConnection() (*CoreDONConnectionConfig, error) { + vars, err := ReadCRIBVars() + if err != nil { + return nil, err + } + var conn *CoreDONConnectionConfig + clNodeURLs := make([]string, 0) + clNodesInternalDNS := make([]string, 0) + clNodesHeaders := make([]map[string]string, 0) + for i := 1; i <= vars.Nodes; i++ { + clNodesInternalDNS = append(clNodesInternalDNS, fmt.Sprintf(InternalNodeDNSTemplate, i)) + clNodesHeaders = append(clNodesHeaders, map[string]string{ + HostHeader: fmt.Sprintf("%s-node%d%s", vars.Namespace, i, vars.IngressSuffix), + }) + } + conn = &CoreDONConnectionConfig{ + ConnectionVars: vars, + PrivateKeys: []string{DefaultSimulatedPrivateKey}, + NodeURLs: clNodeURLs, + NodeInternalDNS: clNodesInternalDNS, + NodeHeaders: clNodesHeaders, + BlockchainNodeHeaders: http.Header{ + HostHeader: []string{fmt.Sprintf("%s-geth-%d-http%s", vars.Namespace, DefaultSimulatedNetworkChainID, vars.IngressSuffix)}, + }, + MockserverHeaders: map[string]string{ + HostHeader: fmt.Sprintf("%s-mockserver%s", vars.Namespace, vars.IngressSuffix), + }, + ChainID: DefaultSimulatedNetworkChainID, + } + // GAP connection + gapURL := os.Getenv("GAP_URL") + if gapURL == "" { + logging.L.Info().Msg("Connecting to CRIB locally") + for i := 1; i <= vars.Nodes; i++ { + conn.NodeURLs = append(conn.NodeURLs, fmt.Sprintf("https://%s-node%d%s", vars.Namespace, i, vars.IngressSuffix)) + } + conn.NetworkWSURL = fmt.Sprintf(IngressNetworkWSURLTemplate, vars.Namespace, DefaultSimulatedNetworkChainID, vars.IngressSuffix) + conn.NetworkHTTPURL = fmt.Sprintf(IngressNetworkHTTPURLTemplate, vars.Namespace, DefaultSimulatedNetworkChainID, vars.IngressSuffix) + conn.MockserverURL = fmt.Sprintf(MockserverCRIBTemplate, vars.Namespace, vars.IngressSuffix) + } else { + logging.L.Info().Msg("Connecting to CRIB using GAP") + for i := 1; i <= vars.Nodes; i++ { + conn.NodeURLs = append(conn.NodeURLs, gapURL) + } + conn.NetworkWSURL = gapURL + conn.NetworkHTTPURL = gapURL + conn.MockserverURL = gapURL + } + logging.L.Debug().Any("ConnectionInfo", conn).Msg("CRIB connection info") + return conn, nil +} + +// ReadCRIBVars read CRIB environment variables +func ReadCRIBVars() (*ConnectionVars, error) { + ingressSuffix := os.Getenv("K8S_STAGING_INGRESS_SUFFIX") + if ingressSuffix == "" { + return nil, errors.New("K8S_STAGING_INGRESS_SUFFIX must be set to connect to k8s ingresses") + } + cribNamespace := os.Getenv("CRIB_NAMESPACE") + if cribNamespace == "" { + return nil, errors.New("CRIB_NAMESPACE must be set to connect") + } + cribNetwork := os.Getenv("CRIB_NETWORK") + if cribNetwork == "" { + return nil, errors.New("CRIB_NETWORK must be set to connect, only 'geth' is supported for now") + } + cribNodes := os.Getenv("CRIB_NODES") + nodes, err := strconv.Atoi(cribNodes) + if err != nil { + return nil, errors.New("CRIB_NODES must be a number, 5-19 nodes") + } + if nodes < 2 { + return nil, fmt.Errorf("not enough chainlink nodes, need at least 2") + } + return &ConnectionVars{ + IngressSuffix: ingressSuffix, + Namespace: cribNamespace, + Network: cribNetwork, + Nodes: nodes, + }, nil +} diff --git a/go.mod b/go.mod index a3783c808..645c1dbe1 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/prometheus/common v0.45.0 github.com/rs/zerolog v1.30.0 github.com/slack-go/slack v0.12.2 - github.com/smartcontractkit/seth v1.0.11 + github.com/smartcontractkit/seth v1.1.1 github.com/smartcontractkit/wasp v0.4.1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 @@ -68,6 +68,7 @@ require ( github.com/allegro/bigcache v1.2.1 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect github.com/aws/aws-sdk-go v1.45.25 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 57a7dde24..bc225bbf1 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= +github.com/awalterschulze/gographviz v2.0.3+incompatible h1:9sVEXJBJLwGX7EQVhLm2elIKCm7P2YHFC8v6096G09E= +github.com/awalterschulze/gographviz v2.0.3+incompatible/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4= github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= @@ -1017,8 +1019,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ= github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= -github.com/smartcontractkit/seth v1.0.11 h1:Ct8wSifW2ZXnyHtYRKaawBnpW7Ef0m8zItUcqYPallM= -github.com/smartcontractkit/seth v1.0.11/go.mod h1:fVCE+8LD6AbU7d8BHZ1uS/ceJ+8JO0iVTUcDdQmGPAc= +github.com/smartcontractkit/seth v1.1.1 h1:6hvexjJD7ek8ht/CLlEwQcH21K2E/WEYwbSRdKInZmM= +github.com/smartcontractkit/seth v1.1.1/go.mod h1:cDfKHi/hJLpO9sRpVbrflrHCOV+MJPAMJHloExJnIXk= github.com/smartcontractkit/wasp v0.4.1 h1:qgIx2s+eCwH0OaBKaHEAHUQ1Z47bAgDu+ICS9IOqvGQ= github.com/smartcontractkit/wasp v0.4.1/go.mod h1:3qiofyI3pkbrc48a3CVshbMfgl74SiuPL/tm30d9Wb4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= diff --git a/grafana/client.go b/grafana/client.go index 4e0effd3f..b5633284b 100644 --- a/grafana/client.go +++ b/grafana/client.go @@ -64,9 +64,11 @@ func (c *Client) GetDatasources() (map[string]string, *resty.Response, error) { } func NewGrafanaClient(url, apiKey string) *Client { - isDebug := os.Getenv("DEBUG_RESTY") == "true" - resty := resty.New().SetDebug(isDebug).SetBaseURL(url).SetHeader("Authorization", "Bearer "+apiKey) - + isDebug := os.Getenv("RESTY_DEBUG") == "true" + resty := resty.New(). + SetDebug(isDebug). + SetBaseURL(url). + SetHeader("Authorization", "Bearer "+apiKey) return &Client{ resty: resty, AlertManager: &AlertManagerClient{resty: resty},