diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ff1a6dd3..d3f57b15c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,6 +131,7 @@ * [ENHANCEMENT] Lifecycler: allow instances to register in ascending order of ids in case of spread minimizing token generation strategy. #326 * [ENHANCEMENT] Remove dependency on `github.com/weaveworks/common` package by migrating code to a corresponding package in `github.com/grafana/dskit`. #342 * [ENHANCEMENT] Add ability to pass TLS certificates and keys inline when configuring server-side TLS. #349 +* [ENHANCEMENT] Migrate `github.com/weaveworks/common/aws` and `github.com/weaveworks/common/test` packages to corresponding packages in `github.com/grafana/dskit`. #356 * [BUGFIX] spanlogger: Support multiple tenant IDs. #59 * [BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. #85 * [BUGFIX] Ring: `ring_member_ownership_percent` and `ring_tokens_owned` metrics are not updated on scale down. #109 diff --git a/aws/config.go b/aws/config.go new file mode 100644 index 000000000..ad4ecefd3 --- /dev/null +++ b/aws/config.go @@ -0,0 +1,69 @@ +// Provenance-includes-location: https://github.com/weaveworks/common/blob/main/aws/config.go +// Provenance-includes-license: Apache-2.0 +// Provenance-includes-copyright: Weaveworks Ltd. + +package aws + +import ( + "fmt" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" +) + +// ConfigFromURL returns AWS config from given URL. It expects escaped +// AWS Access key ID & Secret Access Key to be encoded in the URL. It +// also expects region specified as a host (letting AWS generate full +// endpoint) or fully valid endpoint with dummy region assumed (e.g +// for URLs to emulated services). +func ConfigFromURL(awsURL *url.URL) (*aws.Config, error) { + config := aws.NewConfig(). + // Use a custom http.Client with the golang defaults but also specifying + // MaxIdleConnsPerHost because of a bug in golang https://github.com/golang/go/issues/13801 + // where MaxIdleConnsPerHost does not work as expected. + WithHTTPClient(&http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + MaxIdleConnsPerHost: 100, + TLSHandshakeTimeout: 3 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + }) + + if awsURL.User != nil { + username := awsURL.User.Username() + password, _ := awsURL.User.Password() + + // We request at least the username or password being set to enable the static credentials. + if username != "" || password != "" { + config = config.WithCredentials(credentials.NewStaticCredentials(username, password, "")) + } + } + + if strings.Contains(awsURL.Host, ".") { + region := os.Getenv("AWS_REGION") + if region == "" { + region = "dummy" + } + if awsURL.Scheme == "https" { + return config.WithEndpoint(fmt.Sprintf("https://%s", awsURL.Host)).WithRegion(region), nil + } + return config.WithEndpoint(fmt.Sprintf("http://%s", awsURL.Host)).WithRegion(region), nil + } + + // Let AWS generate default endpoint based on region passed as a host in URL. + return config.WithRegion(awsURL.Host), nil +} diff --git a/aws/config_test.go b/aws/config_test.go new file mode 100644 index 000000000..4e77b5cbd --- /dev/null +++ b/aws/config_test.go @@ -0,0 +1,89 @@ +// Provenance-includes-location: https://github.com/weaveworks/common/blob/main/aws/config_test.go +// Provenance-includes-license: Apache-2.0 +// Provenance-includes-copyright: Weaveworks Ltd. + +package aws + +import ( + "net/url" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAWSConfigFromURL(t *testing.T) { + for i, tc := range []struct { + url string + expectedKey string + expectedSecret string + expectedRegion string + expectedEp string + }{ + { + "s3://abc:123@s3.default.svc.cluster.local:4569", + "abc", + "123", + "dummy", + "http://s3.default.svc.cluster.local:4569", + }, + { + "s3://@us-east-1/test-bucket", + "", + "", + "us-east-1", + "", + }, + { + "dynamodb://user:pass@dynamodb.default.svc.cluster.local:8000/cortex", + "user", + "pass", + "dummy", + "http://dynamodb.default.svc.cluster.local:8000", + }, + { + // No credentials. + "s3://s3.default.svc.cluster.local:4569", + "", + "", + "dummy", + "http://s3.default.svc.cluster.local:4569", + }, + { + "s3://keyWithEscapedSlashAtTheEnd%2F:%24%2C%26%2C%2B%2C%27%2C%2F%2C%3A%2C%3B%2C%3D%2C%3F%2C%40@eu-west-2/bucket1", + "keyWithEscapedSlashAtTheEnd/", + "$,&,+,',/,:,;,=,?,@", + "eu-west-2", + "", + }, + } { + t.Run(strconv.Itoa(i), func(t *testing.T) { + parsedURL, err := url.Parse(tc.url) + require.NoError(t, err) + + cfg, err := ConfigFromURL(parsedURL) + require.NoError(t, err) + + if tc.expectedKey == "" && tc.expectedSecret == "" { + assert.Nil(t, cfg.Credentials) + } else { + require.NotNil(t, cfg.Credentials) + val, err := cfg.Credentials.Get() + require.NoError(t, err) + assert.Equal(t, tc.expectedKey, val.AccessKeyID) + assert.Equal(t, tc.expectedSecret, val.SecretAccessKey) + } + + require.NotNil(t, cfg.Region) + assert.Equal(t, tc.expectedRegion, *cfg.Region) + + if tc.expectedEp != "" { + require.NotNil(t, cfg.Endpoint) + assert.Equal(t, tc.expectedEp, *cfg.Endpoint) + } else { + assert.Nil(t, cfg.Endpoint) + } + }) + } +} diff --git a/go.mod b/go.mod index 08bc91aa2..bbc94ff4f 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,10 @@ require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 github.com/alicebob/miniredis v2.5.0+incompatible github.com/armon/go-metrics v0.3.10 + github.com/aws/aws-sdk-go v1.44.321 github.com/cespare/xxhash v1.1.0 github.com/cristalhq/hedgedhttp v0.7.0 + github.com/davecgh/go-spew v1.1.1 github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb github.com/felixge/httpsnoop v1.0.3 github.com/go-kit/log v0.2.1 @@ -29,6 +31,7 @@ require ( github.com/opentracing-contrib/go-stdlib v1.0.0 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/prometheus/client_golang v1.15.1 github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.44.0 @@ -62,7 +65,6 @@ require ( github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fatih/color v1.13.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -84,7 +86,6 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/onsi/gomega v1.24.0 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 2d975a707..bc6d33e6c 100644 --- a/go.sum +++ b/go.sum @@ -409,6 +409,8 @@ github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8 github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go v1.44.321 h1:iXwFLxWjZPjYqjPq0EcCs46xX7oDLEELte1+BzgpKk8= +github.com/aws/aws-sdk-go v1.44.321/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -659,6 +661,9 @@ github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -964,6 +969,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= @@ -1098,6 +1104,7 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1105,6 +1112,7 @@ golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/test/diff.go b/test/diff.go new file mode 100644 index 000000000..3cc7ea638 --- /dev/null +++ b/test/diff.go @@ -0,0 +1,28 @@ +// Provenance-includes-location: https://github.com/weaveworks/common/blob/main/test/diff.go +// Provenance-includes-license: Apache-2.0 +// Provenance-includes-copyright: Weaveworks Ltd. + +package test + +import ( + "github.com/davecgh/go-spew/spew" + "github.com/pmezard/go-difflib/difflib" +) + +// Diff diffs two arbitrary data structures, giving human-readable output. +func Diff(want, have interface{}) string { + config := spew.NewDefaultConfig() + // Set ContinueOnMethod to true if you cannot see a difference and + // want to look beyond the String() method + config.ContinueOnMethod = false + config.SortKeys = true + config.SpewKeys = true + text, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ + A: difflib.SplitLines(config.Sdump(want)), + B: difflib.SplitLines(config.Sdump(have)), + FromFile: "want", + ToFile: "have", + Context: 3, + }) + return "\n" + text +}