diff --git a/cli/linter/schema.json b/cli/linter/schema.json index ce17033a275..67b41e4f300 100644 --- a/cli/linter/schema.json +++ b/cli/linter/schema.json @@ -1135,6 +1135,20 @@ } } }, + "secrets_manager": { + "type": ["object", "null"], + "properties": { + "access_key_id": { + "type": "string" + }, + "secret_access_key": { + "type": "string" + }, + "region": { + "type": "string" + } + } + }, "vault": { "type": ["object", "null"], "properties": { diff --git a/config/config.go b/config/config.go index cf8bd417652..55e34d6bdc5 100644 --- a/config/config.go +++ b/config/config.go @@ -1069,8 +1069,9 @@ type Config struct { // This section enables the use of the KV capabilities to substitute configuration values. // See more details https://tyk.io/docs/tyk-configuration-reference/kv-store/ KV struct { - Consul ConsulConfig `json:"consul"` - Vault VaultConfig `json:"vault"` + Consul ConsulConfig `json:"consul"` + SecretsManager SecretsManagerConfig `json:"secrets_manager"` + Vault VaultConfig `json:"vault"` } `json:"kv"` // Secrets are key-value pairs that can be accessed in the dashboard via "secrets://" @@ -1213,6 +1214,18 @@ type ConsulConfig struct { } `json:"tls_config"` } +// SecretsManagerConfig is used to configure the AWS Secrets Manager client. +type SecretsManagerConfig struct { + // AccessKeyID is the AWS access key ID. + AccessKeyID string `json:"access_key_id"` + + // SecretAccessKey is the AWS secret access key. + SecretAccessKey string `json:"secret_access_key"` + + // Region is the AWS region. + Region string `json:"region"` +} + // GetEventTriggers returns event triggers. There was a typo in the json tag. // To maintain backward compatibility, this solution is chosen. func (c Config) GetEventTriggers() map[apidef.TykEvent][]TykEventHandler { diff --git a/config/config_test.go b/config/config_test.go index 501456d7747..78646aed8f7 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -353,3 +353,23 @@ func TestPortsWhiteListDecoder(t *testing.T) { assert.Contains(t, tlsWhiteList.Ports, 6000, "tls should have 6000 port") assert.Contains(t, tlsWhiteList.Ports, 6015, "tls should have 6015 port") } + +func TestConfigKVSecretsManager(t *testing.T) { + var c Config + + err := os.Setenv("TYK_GW_KV_SECRETSMANAGER_ACCESSKEYID", "AKIAIOSFODNN7EXAMPLE") + assert.NoError(t, err) + + err = os.Setenv("TYK_GW_KV_SECRETSMANAGER_SECRETACCESSKEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY") + assert.NoError(t, err) + + err = os.Setenv("TYK_GW_KV_SECRETSMANAGER_REGION", "us-east-1") + assert.NoError(t, err) + + err = envconfig.Process("TYK_GW", &c) + assert.NoError(t, err) + + assert.Equal(t, "AKIAIOSFODNN7EXAMPLE", c.KV.SecretsManager.AccessKeyID) + assert.Equal(t, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", c.KV.SecretsManager.SecretAccessKey) + assert.Equal(t, "us-east-1", c.KV.SecretsManager.Region) +} diff --git a/docs/kv.md b/docs/kv.md new file mode 100644 index 00000000000..995f2bf4234 --- /dev/null +++ b/docs/kv.md @@ -0,0 +1,91 @@ +# External Key-Value Storage + +Tyk Gateway supports storing configuration data in key-value (KV) stores such as AWS Secrets Manager, Consul, and Vault +and then referencing these values the gateway configuration or API definitions deployed on the gateway. + +## Supported KV Stores + +Tyk Gateway supports the following KV stores: + +- [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/) +- [Consul](https://www.consul.io/) +- [Vault](https://www.vaultproject.io/) +- Tyk Config Secrets +- Environment Variables + +## Accessing KV Store Data + +You can configure Tyk Gateway to retrieve values from KV stores in the following places: + +- Tyk Gateway configuration file (tyk.conf) +- API definitions +- Transform Middleware + +### Gateway Configuration + +You can retrieve values from KV stores for the following fields Tyk Gateway configuration fields: + +- `secret` +- `node_secret` +- `storage.password` +- `cache_storage.password` +- `security.private_certificate_encoding_secret` +- `db_app_conf_options.connection_string` +- `policies.policy_connection_string` + +Use the following notation to reference KV store values in the gateway configuration file: + +| Store | Notation | +|-----------------------------|-------------------------------------------------------------| +| AWS Secrets Manager \[1\] | `secretsmanager://{key}` or `secretsmanager://{path}.{key}` | +| Consul | `consul://{key}` | +| Vault \[1\] | `vault://{path}.{key}` | +| Tyk config secrets | `secrets://{key}` | +| Environment variables \[2\] | `env://{key}` | + +\[1\] Path key value must be a JSON document to support `{path}.{key}` references (e.g., `{"key": "value"}`).\ +\[2\] Gateway configuration environment variable names must be prefixed with `TYK_SECRET_{KEY}` (e.g., +`TYK_SECRET_MY_API_SECRET` is referenced as `env://MY_API_SECRET`). Environment variables names must be uppercase. + +### API Definition + +Use the following notation to reference KV store values in API definition fields: + +| Store | Notation | +|-----------------------------|--------------------------| +| AWS Secrets Manager \[1\] | `secretsmanager://{key}` | +| Consul \[1\] | `consul://{key}` | +| Vault \[1\] | `vault://{key}` | +| Tyk config secrets | `secrets://{key}` | +| Environment variables \[2\] | `env://{key}` | + +\[1\] External API definition secrets must be stored in a JSON document in the KV store with the key `tyk-apis`. +In this case, `{key}` refers to the JSON document key (e.g., `{"key": "value"}`).\ +\[2\] API definition environment variables can be any name (e.g. `MY_API_SECRET` is referenced as +`env://MY_API_SECRET`). They **do not** require the prefix `TYK_SECRET_{KEY}`. + +### Transform Middleware + +You can retrieve values from KV stores for the following transform middleware: + +- Authentication Token (signature secret) +- Persist GraphQL Operation (request path) +- Rate Limiting (rate limit pattern) +- Request Body +- Request Headers +- Response Headers +- URL Rewrite + +Use the following notation to reference KV store values in transform middleware: + +| Store | Notation | +|-----------------------------|-------------------------------------------------------------------------| +| AWS Secrets Manager \[1\] | `$secret_secretsmanager.{key}` or `$secret_secretsmanager.{path}.{key}` | +| Consul | `$secret_consul.{key}` | +| Vault \[1\] | `$secret_vault.{path}.{key}` | +| Tyk config secrets | `$secret_conf.{key}` | +| Environment variables \[2\] | `$secret_env.{key}` | + +\[1\] Path key value must be a JSON document to support `{path}.{key}` references (e.g., `{"key": "value"}`).\ +\[2\] Transform middleware environment variable names must be prefixed with `TYK_SECRET_{KEY}` (e.g., +`TYK_SECRET_MY_API_SECRET` is referenced as `$secret_env.my_api_secret`). diff --git a/gateway/api_definition.go b/gateway/api_definition.go index f70e25f4701..86a9720a2c5 100644 --- a/gateway/api_definition.go +++ b/gateway/api_definition.go @@ -34,7 +34,7 @@ import ( "github.com/cenk/backoff" - sprig "github.com/Masterminds/sprig/v3" + "github.com/Masterminds/sprig/v3" "github.com/sirupsen/logrus" @@ -558,12 +558,13 @@ func (a APIDefinitionLoader) FromDashboardService(endpoint string) ([]*APISpec, var envRegex = regexp.MustCompile(`env://([^"]+)`) const ( - prefixEnv = "env://" - prefixSecrets = "secrets://" - prefixConsul = "consul://" - prefixVault = "vault://" - prefixKeys = "tyk-apis" - vaultSecretPath = "secret/data/" + prefixEnv = "env://" + prefixSecrets = "secrets://" + prefixConsul = "consul://" + prefixSecretsManager = "secretsmanager://" + prefixVault = "vault://" + prefixKeys = "tyk-apis" + vaultSecretPath = "secret/data/" ) func (a APIDefinitionLoader) replaceSecrets(in []byte) []byte { @@ -597,6 +598,12 @@ func (a APIDefinitionLoader) replaceSecrets(in []byte) []byte { } } + if strings.Contains(input, prefixSecretsManager) { + if err := a.replaceSecretsManagerSecrets(&input); err != nil { + log.WithError(err).Error("Couldn't replace Secrets Manager secrets") + } + } + if strings.Contains(input, prefixVault) { if err := a.replaceVaultSecrets(&input); err != nil { log.WithError(err).Error("Couldn't replace vault secrets") @@ -624,6 +631,29 @@ func (a APIDefinitionLoader) replaceConsulSecrets(input *string) error { return nil } +func (a APIDefinitionLoader) replaceSecretsManagerSecrets(input *string) error { + if err := a.Gw.setUpSecretsManager(); err != nil { + return err + } + + value, err := a.Gw.secretsManagerKVStore.Get(prefixKeys) + if err != nil { + return err + } + + jsonValue := make(map[string]interface{}) + err = json.Unmarshal([]byte(value), &jsonValue) + if err != nil { + return fmt.Errorf("error unmarshalling secret string: %w", err) + } + + for k, v := range jsonValue { + *input = strings.Replace(*input, prefixSecretsManager+k, fmt.Sprintf("%v", v), -1) + } + + return nil +} + func (a APIDefinitionLoader) replaceVaultSecrets(input *string) error { if err := a.Gw.setUpVault(); err != nil { return err @@ -651,7 +681,7 @@ func (a APIDefinitionLoader) replaceVaultSecrets(input *string) error { return nil } -// FromCloud will connect and download ApiDefintions from a Mongo DB instance. +// FromRPC will connect and download API definitions. func (a APIDefinitionLoader) FromRPC(store RPCDataLoader, orgId string, gw *Gateway) ([]*APISpec, error) { if rpc.IsEmergencyMode() { return gw.LoadDefinitionsFromRPCBackup() @@ -759,6 +789,7 @@ func (a APIDefinitionLoader) FromDir(dir string) []*APISpec { // Grab json files from directory paths, _ := filepath.Glob(filepath.Join(dir, "*.json")) for _, path := range paths { + // TODO(#6563): Why aren't OAS definitions loaded here? This prevents secrets from being replaced in OAS definitions. if strings.HasSuffix(path, "-oas.json") { continue } @@ -773,6 +804,7 @@ func (a APIDefinitionLoader) FromDir(dir string) []*APISpec { } return specs } + func (a APIDefinitionLoader) loadDefFromFilePath(filePath string) (*APISpec, error) { log.Info("Loading API Specification from ", filePath) diff --git a/gateway/api_definition_test.go b/gateway/api_definition_test.go index cece0631fb9..5209cd11bb8 100644 --- a/gateway/api_definition_test.go +++ b/gateway/api_definition_test.go @@ -18,10 +18,12 @@ import ( "github.com/stretchr/testify/assert" persistentmodel "github.com/TykTechnologies/storage/persistent/model" + "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/apidef/oas" "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/rpc" + "github.com/TykTechnologies/tyk/storage/kv" "github.com/TykTechnologies/tyk/test" "github.com/TykTechnologies/tyk/user" ) @@ -1602,3 +1604,64 @@ func TestInternalEndpointMW_TT_11126(t *testing.T) { {Path: "/headers", Code: http.StatusForbidden}, }...) } + +func TestAPIDefinitionReplaceSecrets(t *testing.T) { + ts := StartTest(func(globalConf *config.Config) { + globalConf.Secrets = map[string]string{ + "jwt_signing_method": "secrets::jwt_signing_method::value", + "jwt_source": "secrets::jwt_source::value", + } + }) + defer ts.Close() + + t.Setenv("JWT_SIGNING_METHOD", "env::jwt_signing_method::value") + t.Setenv("JWT_SOURCE", "env::jwt_source::value") + + ts.Gw.secretsManagerKVStore = kv.NewSecretsManagerWithClient(kv.NewDummySecretsManagerClient(map[string]string{ + "tyk-apis": "{" + + "\"jwt_signing_method\":\"secretsmanager::jwt_signing_method::value\"," + + "\"jwt_source\":\"secretsmanager::jwt_source::value\"" + + "}", + })) + + tests := []struct { + name string + scheme string + specs []*APISpec + }{ + { + name: "Config", + scheme: "secrets", + specs: BuildAPI(func(spec *APISpec) { + spec.JWTSigningMethod = "secrets://jwt_signing_method" + spec.JWTSource = "secrets://jwt_source" + }), + }, + { + name: "Env", + scheme: "env", + specs: BuildAPI(func(spec *APISpec) { + spec.JWTSigningMethod = "env://JWT_SIGNING_METHOD" + spec.JWTSource = "env://JWT_SOURCE" + }), + }, + { + name: "SecretsManager", + scheme: "secretsmanager", + specs: BuildAPI(func(spec *APISpec) { + spec.JWTSigningMethod = "secretsmanager://jwt_signing_method" + spec.JWTSource = "secretsmanager://jwt_source" + }), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + specs := ts.Gw.LoadAPI(tt.specs...) + for _, spec := range specs { + assert.Equal(t, tt.scheme+"::jwt_signing_method::value", spec.JWTSigningMethod) + assert.Equal(t, tt.scheme+"::jwt_source::value", spec.JWTSource) + } + }) + } +} diff --git a/gateway/mw_auth_key_test.go b/gateway/mw_auth_key_test.go index f1b4b5e0776..3e3b17adef2 100644 --- a/gateway/mw_auth_key_test.go +++ b/gateway/mw_auth_key_test.go @@ -18,6 +18,7 @@ import ( "github.com/TykTechnologies/tyk/config" signaturevalidator "github.com/TykTechnologies/tyk/signature_validator" "github.com/TykTechnologies/tyk/storage" + "github.com/TykTechnologies/tyk/storage/kv" "github.com/TykTechnologies/tyk/test" "github.com/TykTechnologies/tyk/user" @@ -666,3 +667,73 @@ func TestDynamicMTLS(t *testing.T) { }) }) } + +func TestAuthKeyReplaceSecrets(t *testing.T) { + ts := StartTest(func(conf *config.Config) { + conf.Secrets = map[string]string{ + "auth_key_signature_secret": "conf::auth_key_signature_secret::value", + } + }) + defer ts.Close() + + t.Setenv("TYK_SECRET_AUTH_KEY_SIGNATURE_SECRET", "env::auth_key_signature_secret::value") + + ts.Gw.secretsManagerKVStore = kv.NewSecretsManagerWithClient(kv.NewDummySecretsManagerClient(map[string]string{ + "auth_key_signature_secret": "secretsmanager::auth_key_signature_secret::value", + })) + + tests := []struct { + name string + scheme string + specs []*APISpec + }{ + { + name: "Config", + scheme: "conf", + }, + { + name: "Env", + scheme: "env", + }, + { + name: "SecretsManager", + scheme: "secretsmanager", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + specs := BuildAPI(func(spec *APISpec) { + spec.AuthConfigs = map[string]apidef.AuthConfig{ + apidef.AuthTokenType: { + ValidateSignature: true, + Signature: apidef.SignatureConfig{ + Algorithm: "MasherySHA256", + Header: "Signature", + Secret: "$secret_" + tt.scheme + ".auth_key_signature_secret", + }, + }, + } + spec.Proxy.ListenPath = "/" + spec.UseKeylessAccess = false + }) + + ts.Gw.LoadAPI(specs...) + + key := CreateSession(ts.Gw) + hasher := signaturevalidator.MasherySha256Sum{} + signature := hasher.Hash(key, tt.scheme+"::auth_key_signature_secret::value", time.Now().Unix()) + + _, err := ts.Run(t, test.TestCase{ + Method: http.MethodGet, + Path: "/", + Headers: map[string]string{ + "Authorization": key, + "Signature": hex.EncodeToString(signature), + }, + Code: http.StatusOK, + }) + assert.NoError(t, err) + }) + } +} diff --git a/gateway/mw_modify_headers_test.go b/gateway/mw_modify_headers_test.go index 7093be7afd5..40d8126fc47 100644 --- a/gateway/mw_modify_headers_test.go +++ b/gateway/mw_modify_headers_test.go @@ -1,11 +1,15 @@ package gateway import ( + "net/http" "testing" "github.com/stretchr/testify/assert" "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/storage/kv" + "github.com/TykTechnologies/tyk/test" ) func TestTransformHeaders_EnabledForSpec(t *testing.T) { @@ -100,3 +104,90 @@ func TestHeaderInjectionMeta_Enabled(t *testing.T) { h.DeleteHeaders = []string{"a"} assert.True(t, h.Enabled()) } + +func TestTransformHeadersReplaceSecrets(t *testing.T) { + ts := StartTest(func(conf *config.Config) { + conf.Secrets = map[string]string{ + "global_header": "conf::global_header::value", + "path_header": "conf::path_header::value", + } + }) + defer ts.Close() + + t.Setenv("TYK_SECRET_GLOBAL_HEADER", "env::global_header::value") + t.Setenv("TYK_SECRET_PATH_HEADER", "env::path_header::value") + + ts.Gw.secretsManagerKVStore = kv.NewSecretsManagerWithClient(kv.NewDummySecretsManagerClient(map[string]string{ + "global_header": "secretsmanager::global_header::value", + "path_header": "secretsmanager::path_header::value", + })) + + ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + UpdateAPIVersion(spec, "v1", func(v *apidef.VersionInfo) { + v.UseExtendedPaths = true + v.ExtendedPaths.TransformHeader = []apidef.HeaderInjectionMeta{ + { + AddHeaders: map[string]string{ + "Path-Conf": "$secret_conf.path_header", + "Path-Env": "$secret_env.path_header", + "Path-Secretsmanager": "$secret_secretsmanager.path_header", + }, + Method: http.MethodGet, + Path: "/path", + }, + } + v.GlobalHeaders = map[string]string{ + "Global-Conf": "$secret_conf.global_header", + "Global-Env": "$secret_env.global_header", + "Global-Secretsmanager": "$secret_secretsmanager.global_header", + } + }) + }) + + tests := []struct { + name string + globalHeaderMatch string + pathHeaderMatch string + }{ + { + name: "Config", + globalHeaderMatch: `"Global-Conf":"conf::global_header::value"`, + pathHeaderMatch: `"Path-Conf":"conf::path_header::value"`, + }, + { + name: "Env", + globalHeaderMatch: `"Global-Env":"env::global_header::value"`, + pathHeaderMatch: `"Path-Env":"env::path_header::value"`, + }, + { + name: "SecretsManager", + globalHeaderMatch: `"Global-Secretsmanager":"secretsmanager::global_header::value"`, + pathHeaderMatch: `"Path-Secretsmanager":"secretsmanager::path_header::value"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Run("GlobalHeaders", func(t *testing.T) { + _, err := ts.Run(t, test.TestCase{ + Method: http.MethodGet, + Path: "/", + Code: http.StatusOK, + BodyMatch: tt.globalHeaderMatch, + }) + assert.NoError(t, err) + }) + + t.Run("PathHeaders", func(t *testing.T) { + _, err := ts.Run(t, test.TestCase{ + Method: http.MethodGet, + Path: "/path", + Code: http.StatusOK, + BodyMatch: tt.pathHeaderMatch, + }) + assert.NoError(t, err) + }) + }) + } +} diff --git a/gateway/mw_url_rewrite.go b/gateway/mw_url_rewrite.go index b3e75f37505..cf7451f8274 100644 --- a/gateway/mw_url_rewrite.go +++ b/gateway/mw_url_rewrite.go @@ -20,19 +20,21 @@ import ( ) const ( - metaLabel = "$tyk_meta." - contextLabel = "$tyk_context." - consulLabel = "$secret_consul." - vaultLabel = "$secret_vault." - envLabel = "$secret_env." - secretsConfLabel = "$secret_conf." - triggerKeyPrefix = "trigger" - triggerKeySep = "-" + metaLabel = "$tyk_meta." + contextLabel = "$tyk_context." + consulLabel = "$secret_consul." + secretsManagerLabel = "$secret_secretsmanager." + vaultLabel = "$secret_vault." + envLabel = "$secret_env." + secretsConfLabel = "$secret_conf." + triggerKeyPrefix = "trigger" + triggerKeySep = "-" ) var dollarMatch = regexp.MustCompile(`\$\d+`) var contextMatch = regexp.MustCompile(`\$tyk_context.([A-Za-z0-9_\-\.]+)`) var consulMatch = regexp.MustCompile(`\$secret_consul.([A-Za-z0-9\/\-\.]+)`) +var secretsManagerMatch = regexp.MustCompile(`\$secret_secretsmanager\.([A-Za-z0-9\/\-\._]+)`) var vaultMatch = regexp.MustCompile(`\$secret_vault.([A-Za-z0-9\/\-\.]+)`) var envValueMatch = regexp.MustCompile(`\$secret_env.([A-Za-z0-9_\-\.]+)`) var metaMatch = regexp.MustCompile(`\$tyk_meta.([A-Za-z0-9_\-\.]+)`) @@ -211,7 +213,6 @@ func (gw *Gateway) urlRewrite(meta *apidef.URLRewriteMeta, r *http.Request) (str } func (gw *Gateway) replaceTykVariables(r *http.Request, in string, escape bool) string { - if strings.Contains(in, secretsConfLabel) { contextData := ctxGetData(r) vars := secretsConfMatch.FindAllString(in, -1) @@ -251,12 +252,18 @@ func (gw *Gateway) replaceTykVariables(r *http.Request, in string, escape bool) in = gw.replaceVariables(in, vars, session.MetaData, metaLabel, escape) } } + + if strings.Contains(in, secretsManagerLabel) { + contextData := ctxGetData(r) + vars := secretsManagerMatch.FindAllString(in, -1) + in = gw.replaceVariables(in, vars, contextData, secretsManagerLabel, escape) + } + //todo add config_data return in } func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]interface{}, label string, escape bool) string { - emptyStringFn := func(key, in, val string) string { in = strings.Replace(in, val, "", -1) log.WithFields(logrus.Fields{ @@ -272,9 +279,7 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in key := strings.Replace(v, label, "", 1) switch label { - case secretsConfLabel: - secrets := gw.GetConfig().Secrets val, ok := secrets[key] @@ -284,9 +289,7 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in } in = strings.Replace(in, v, val, -1) - case envLabel: - val := os.Getenv(fmt.Sprintf("TYK_SECRET_%s", strings.ToUpper(key))) if val == "" { in = emptyStringFn(key, in, v) @@ -294,9 +297,7 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in } in = strings.Replace(in, v, val, -1) - case vaultLabel: - if err := gw.setUpVault(); err != nil { in = emptyStringFn(key, in, v) continue @@ -309,9 +310,7 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in } in = strings.Replace(in, v, val, -1) - case consulLabel: - if err := gw.setUpConsul(); err != nil { in = emptyStringFn(key, in, v) continue @@ -324,9 +323,20 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in } in = strings.Replace(in, v, val, -1) + case secretsManagerLabel: + if err := gw.setUpSecretsManager(); err != nil { + in = emptyStringFn(key, in, v) + continue + } - default: + val, err := gw.secretsManagerKVStore.Get(key) + if err != nil { + in = strings.Replace(in, v, "", -1) + continue + } + in = strings.Replace(in, v, val, -1) + default: val, ok := vals[key] if ok { valStr := valToStr(val) diff --git a/gateway/mw_url_rewrite_test.go b/gateway/mw_url_rewrite_test.go index 1da2fbc3072..e81fcf28cbe 100644 --- a/gateway/mw_url_rewrite_test.go +++ b/gateway/mw_url_rewrite_test.go @@ -8,7 +8,9 @@ import ( "github.com/stretchr/testify/assert" + "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/ctx" + "github.com/TykTechnologies/tyk/storage/kv" "github.com/TykTechnologies/tyk/test" @@ -127,6 +129,7 @@ func TestRewriter(t *testing.T) { }) } } + func BenchmarkRewriter(b *testing.B) { ts := StartTest(nil) defer ts.Close() @@ -1384,3 +1387,112 @@ func TestURLRewriteMiddleware_CheckHostRewrite(t *testing.T) { }) } } + +func TestTransformMiddlewareReplaceSecrets(t *testing.T) { + ts := StartTest(func(conf *config.Config) { + conf.Secrets = map[string]string{ + "auth_key_signature_secret": "conf::auth_key_signature_secret::value", + "persist_graphql_variable": "conf::persist_graphql_variable::value", + "rate_limit_pattern": "conf::rate_limit_pattern::value", + "request_body": "conf::request_body::value", + "request_headers_global_header_value": "conf::request_headers_global_header_value::value", + "request_headers_path_header_value": "conf::request_headers_path_header_value::value", + "response_headers_global_header_value": "conf::response_headers_global_header_value::value", + "response_headers_path_header_value": "conf::response_headers_path_header_value::value", + "url_rewrite": "/conf/url_rewrite/value", + } + }) + defer ts.Close() + + t.Setenv("TYK_SECRET_AUTH_KEY_SIGNATURE_SECRET", "env::auth_key_signature_secret::value") + t.Setenv("TYK_SECRET_PERSIST_GRAPHQL_VARIABLE", "env::persist_graphql_variable::value") + t.Setenv("TYK_SECRET_RATE_LIMIT_PATTERN", "env::rate_limit_pattern::value") + t.Setenv("TYK_SECRET_REQUEST_BODY", "env::request_body::value") + t.Setenv("TYK_SECRET_REQUEST_HEADERS_GLOBAL_HEADER_VALUE", "env::request_headers_global_header_value::value") + t.Setenv("TYK_SECRET_REQUEST_HEADERS_PATH_HEADER_VALUE", "env::request_headers_path_header_value::value") + t.Setenv("TYK_SECRET_RESPONSE_HEADERS_GLOBAL_HEADER_VALUE", "env::response_headers_global_header_value::value") + t.Setenv("TYK_SECRET_RESPONSE_HEADERS_PATH_HEADER_VALUE", "env::response_headers_path_header_value::value") + t.Setenv("TYK_SECRET_URL_REWRITE", "/env/url_rewrite/value") + + ts.Gw.secretsManagerKVStore = kv.NewSecretsManagerWithClient(kv.NewDummySecretsManagerClient(map[string]string{ + "auth_key_signature_secret": "secretsmanager::auth_key_signature_secret::value", + "persist_graphql_variable": "secretsmanager::persist_graphql_variable::value", + "rate_limit_pattern": "secretsmanager::rate_limit_pattern::value", + "request_body": "secretsmanager::transform_request_body::value", + "request_headers_global_header_value": "secretsmanager::request_headers_global_header_value::value", + "request_headers_path_header_value": "secretsmanager::request_headers_path_header_value::value", + "response_headers_global_header_value": "secretsmanager::response_headers_global_header_value::value", + "response_headers_path_header_value": "secretsmanager::response_headers_path_header_value::value", + "url_rewrite": "/secretsmanager/url_rewrite/value", + })) + + // [x] Auth key signature secret + // [ ] Persist GraphQL variable + // [ ] Rate limit pattern + // [ ] Request body + // [ ] Request headers global header value + // [ ] Request headers path header value + // [ ] Response headers global header value + // [ ] Response headers path header value + // [x] URL rewrite +} + +func TestURLRewriteReplaceSecrets(t *testing.T) { + ts := StartTest(func(conf *config.Config) { + conf.Secrets = map[string]string{ + "url_rewrite": "/conf/url_rewrite/value", + } + }) + defer ts.Close() + + t.Setenv("TYK_SECRET_URL_REWRITE", "/env/url_rewrite/value") + + ts.Gw.secretsManagerKVStore = kv.NewSecretsManagerWithClient(kv.NewDummySecretsManagerClient(map[string]string{ + "url_rewrite": "/secretsmanager/url_rewrite/value", + })) + + tests := []struct { + name string + scheme string + specs []*APISpec + }{ + { + name: "Config", + scheme: "conf", + }, + { + name: "Env", + scheme: "env", + }, + { + name: "SecretsManager", + scheme: "secretsmanager", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + specs := BuildAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/" + UpdateAPIVersion(spec, "Default", func(v *apidef.VersionInfo) { + v.ExtendedPaths.URLRewrite = []apidef.URLRewriteMeta{{ + Path: "/", + Method: http.MethodGet, + MatchPattern: "/", + RewriteTo: "$secret_" + tt.scheme + ".url_rewrite", + }} + }) + }) + + ts.Gw.LoadAPI(specs...) + + _, err := ts.Run(t, test.TestCase{ + Method: http.MethodGet, + Path: "/", + Code: http.StatusOK, + BodyMatch: "\"Url\":\"/" + tt.scheme + "/url_rewrite/value\"", + }) + assert.NoError(t, err) + }) + } +} diff --git a/gateway/server.go b/gateway/server.go index aa20ee19eea..df7eabba137 100644 --- a/gateway/server.go +++ b/gateway/server.go @@ -158,8 +158,9 @@ type Gateway struct { dnsCacheManager dnscache.IDnsCacheManager - consulKVStore kv.Store - vaultKVStore kv.Store + consulKVStore kv.Store + secretsManagerKVStore kv.Store + vaultKVStore kv.Store // signatureVerifier is used to verify signatures with config.PublicKeyPath. signatureVerifier atomic.Pointer[goverify.Verifier] @@ -1477,7 +1478,6 @@ func (gw *Gateway) afterConfSetup() { } func (gw *Gateway) kvStore(value string) (string, error) { - if strings.HasPrefix(value, "secrets://") { key := strings.TrimPrefix(value, "secrets://") log.Debugf("Retrieving %s from secret store in config", key) @@ -1520,6 +1520,17 @@ func (gw *Gateway) kvStore(value string) (string, error) { return gw.vaultKVStore.Get(key) } + if strings.HasPrefix(value, "secretsmanager://") { + key := strings.TrimPrefix(value, "secretsmanager://") + log.Debugf("Retrieving %s from Secrets Manager", key) + if err := gw.setUpSecretsManager(); err != nil { + log.Error("Failed to setup Secrets Manager: ", err) + return value, nil + } + + return gw.secretsManagerKVStore.Get(key) + } + return value, nil } @@ -1553,6 +1564,21 @@ func (gw *Gateway) setUpConsul() error { return err } +func (gw *Gateway) setUpSecretsManager() error { + if gw.secretsManagerKVStore != nil { + return nil + } + + var err error + + gw.secretsManagerKVStore, err = kv.NewSecretsManagerWithConfig(gw.GetConfig().KV.SecretsManager) + if err != nil { + log.Debugf("an error occurred while setting up Secrets Manager... %v", err) + } + + return err +} + var getIpAddress = netutil.GetIpAddress func (gw *Gateway) getHostDetails(file string) { diff --git a/gateway/server_test.go b/gateway/server_test.go index 8bac1296249..056d0b8e943 100644 --- a/gateway/server_test.go +++ b/gateway/server_test.go @@ -14,6 +14,7 @@ import ( "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/internal/netutil" "github.com/TykTechnologies/tyk/internal/otel" + "github.com/TykTechnologies/tyk/storage/kv" "github.com/TykTechnologies/tyk/test" "github.com/TykTechnologies/tyk/user" ) @@ -367,3 +368,148 @@ func TestGatewayGetHostDetails(t *testing.T) { }) } } + +func TestGatewayConfigReplaceSecrets(t *testing.T) { + ts := StartTest(nil) + defer ts.Close() + + t.Setenv("TYK_SECRET_SECRET", "env::secret::value") + t.Setenv("TYK_SECRET_NODE_SECRET", "env::node_secret::value") + t.Setenv("TYK_SECRET_STORAGE_PASSWORD", "env::storage::password::value") + t.Setenv("TYK_SECRET_CACHE_STORAGE_PASSWORD", "env::cache_storage::password::value") + t.Setenv("TYK_SECRET_SECURITY_PRIVATE_CERTIFICATE_ENCODING_SECRET", "env::security::private_certificate_encoding_secret::value") + t.Setenv("TYK_SECRET_DB_APP_CONF_OPTION_CONNECTION_STRING", "env::db_app_conf_options::connection_string::value") + t.Setenv("TYK_SECRET_POLICIES_POLICY_CONNECTION_STRING", "env::policies::policy_connection_string::value") + + ts.Gw.secretsManagerKVStore = kv.NewSecretsManagerWithClient(kv.NewDummySecretsManagerClient(map[string]string{ + "tyk-config": "{" + + "\"secret\":\"secretsmanager::secret::value\"," + + "\"node_secret\":\"secretsmanager::node_secret::value\"," + + "\"storage_password\":\"secretsmanager::storage::password::value\"," + + "\"cache_storage_password\":\"secretsmanager::cache_storage::password::value\"," + + "\"security_private_certificate_encoding_secret\":\"secretsmanager::security::private_certificate_encoding_secret::value\"," + + "\"db_app_conf_options_connection_string\":\"secretsmanager::db_app_conf_options::connection_string::value\"," + + "\"policies_policy_connection_string\":\"secretsmanager::policies::policy_connection_string::value\"" + + "}", + })) + + tests := []struct { + name string + scheme string + conf config.Config + }{ + { + name: "Config", + scheme: "secrets", + conf: config.Config{ + Secrets: map[string]string{ + "secret": "secrets::secret::value", + "node_secret": "secrets::node_secret::value", + "storage_password": "secrets::storage::password::value", + "cache_storage_password": "secrets::cache_storage::password::value", + "security_private_certificate_encoding_secret": "secrets::security::private_certificate_encoding_secret::value", + "db_app_conf_options_connection_string": "secrets::db_app_conf_options::connection_string::value", + "policies_policy_connection_string": "secrets::policies::policy_connection_string::value", + }, + Secret: "secrets://secret", + NodeSecret: "secrets://node_secret", + Storage: config.StorageOptionsConf{ + Password: "secrets://storage_password", + }, + CacheStorage: config.StorageOptionsConf{ + Password: "secrets://cache_storage_password", + }, + Security: config.SecurityConfig{ + PrivateCertificateEncodingSecret: "secrets://security_private_certificate_encoding_secret", + }, + UseDBAppConfigs: true, + DBAppConfOptions: config.DBAppConfOptionsConfig{ + ConnectionString: "secrets://db_app_conf_options_connection_string", + }, + Policies: config.PoliciesConfig{ + PolicySource: "service", + PolicyConnectionString: "secrets://policies_policy_connection_string", + }, + }, + }, + { + name: "Env", + scheme: "env", + conf: config.Config{ + Secret: "env://SECRET", + NodeSecret: "env://NODE_SECRET", + Storage: config.StorageOptionsConf{ + Password: "env://STORAGE_PASSWORD", + }, + CacheStorage: config.StorageOptionsConf{ + Password: "env://CACHE_STORAGE_PASSWORD", + }, + Security: config.SecurityConfig{ + PrivateCertificateEncodingSecret: "env://SECURITY_PRIVATE_CERTIFICATE_ENCODING_SECRET", + }, + UseDBAppConfigs: true, + DBAppConfOptions: config.DBAppConfOptionsConfig{ + ConnectionString: "env://DB_APP_CONF_OPTION_CONNECTION_STRING", + }, + Policies: config.PoliciesConfig{ + PolicySource: "service", + PolicyConnectionString: "env://POLICIES_POLICY_CONNECTION_STRING", + }, + }, + }, + { + name: "SecretsManager", + scheme: "secretsmanager", + conf: config.Config{ + Secret: "secretsmanager://tyk-config.secret", + NodeSecret: "secretsmanager://tyk-config.node_secret", + Storage: config.StorageOptionsConf{ + Password: "secretsmanager://tyk-config.storage_password", + }, + CacheStorage: config.StorageOptionsConf{ + Password: "secretsmanager://tyk-config.cache_storage_password", + }, + Security: config.SecurityConfig{ + PrivateCertificateEncodingSecret: "secretsmanager://tyk-config.security_private_certificate_encoding_secret", + }, + UseDBAppConfigs: true, + DBAppConfOptions: config.DBAppConfOptionsConfig{ + ConnectionString: "secretsmanager://tyk-config.db_app_conf_options_connection_string", + }, + Policies: config.PoliciesConfig{ + PolicySource: "service", + PolicyConnectionString: "secretsmanager://tyk-config.policies_policy_connection_string", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts.Gw.SetConfig(tt.conf) + ts.Gw.afterConfSetup() + + actual := ts.Gw.GetConfig() + + assert.Equal(t, tt.scheme+"::secret::value", actual.Secret) + assert.Equal(t, tt.scheme+"::node_secret::value", actual.NodeSecret) + assert.Equal(t, tt.scheme+"::storage::password::value", actual.Storage.Password) + assert.Equal(t, tt.scheme+"::cache_storage::password::value", actual.CacheStorage.Password) + assert.Equal( + t, + tt.scheme+"::security::private_certificate_encoding_secret::value", + actual.Security.PrivateCertificateEncodingSecret, + ) + assert.Equal( + t, + tt.scheme+"::db_app_conf_options::connection_string::value", + actual.DBAppConfOptions.ConnectionString, + ) + assert.Equal( + t, + tt.scheme+"::policies::policy_connection_string::value", + actual.Policies.PolicyConnectionString, + ) + }) + } +} diff --git a/go.mod b/go.mod index fdf38955646..12a55ece60c 100644 --- a/go.mod +++ b/go.mod @@ -87,6 +87,11 @@ require ( github.com/TykTechnologies/kin-openapi v0.90.0 github.com/TykTechnologies/opentelemetry v0.0.21 github.com/alecthomas/kingpin/v2 v2.4.0 + github.com/aws/aws-sdk-go v1.47.10 + github.com/aws/aws-sdk-go-v2/config v1.9.0 + github.com/aws/aws-sdk-go-v2/credentials v1.6.1 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.0 + github.com/aws/smithy-go v1.21.0 github.com/go-redis/redismock/v9 v9.2.0 github.com/goccy/go-json v0.10.3 github.com/google/go-cmp v0.6.0 @@ -96,6 +101,17 @@ require ( go.uber.org/mock v0.4.0 ) +require ( + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.2.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.6.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.10.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect +) + require ( dario.cat/mergo v1.0.1 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect @@ -111,6 +127,7 @@ require ( github.com/asyncapi/converter-go v0.3.0 // indirect github.com/asyncapi/parser-go v0.4.2 // indirect github.com/asyncapi/spec-json-schemas/v2 v2.14.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.31.0 github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -175,7 +192,7 @@ require ( github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/r3labs/sse/v2 v2.8.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect diff --git a/go.sum b/go.sum index 2b3ab43af4e..9931281b2ea 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,43 @@ github.com/asyncapi/parser-go v0.4.2 h1:+PPo+Sk/u9IPH3JklXFrPUQnr9LURwNJU5jfcLBB github.com/asyncapi/parser-go v0.4.2/go.mod h1:5iAT+irO9xKeBDnIhqT0ev8QJH1dHq4i2oU/UBhhwB8= github.com/asyncapi/spec-json-schemas/v2 v2.14.0 h1:t5Frb4WzaV2bTlcRqXhBdvqTjO6+n/oiPMjvCxYooag= github.com/asyncapi/spec-json-schemas/v2 v2.14.0/go.mod h1:5lFCFtRGfI3WVOla4slifjgPs9x79FY0fqZjgNL495c= +github.com/aws/aws-sdk-go v1.47.10 h1:cvufN7WkD1nlOgpRopsmxKQlFp5X1MfyAw4r7BBORQc= +github.com/aws/aws-sdk-go v1.47.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go-v2 v1.10.0/go.mod h1:U/EyyVvKtzmFeQQcca7eBotKdlpcP2zzU6bXBYcf7CE= +github.com/aws/aws-sdk-go-v2 v1.11.0/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= +github.com/aws/aws-sdk-go-v2 v1.31.0 h1:3V05LbxTSItI5kUqNwhJrrrY1BAXxXt0sN0l72QmG5U= +github.com/aws/aws-sdk-go-v2 v1.31.0/go.mod h1:ztolYtaEUtdpf9Wftr31CJfLVjOnD/CVRkKOOYgF8hA= +github.com/aws/aws-sdk-go-v2/config v1.9.0 h1:SkREVSwi+J8MSdjhJ96jijZm5ZDNleI0E4hHCNivh7s= +github.com/aws/aws-sdk-go-v2/config v1.9.0/go.mod h1:qhK5NNSgo9/nOSMu3HyE60WHXZTWTHTgd5qtIF44vOQ= +github.com/aws/aws-sdk-go-v2/credentials v1.5.0/go.mod h1:kvqTkpzQmzri9PbsiTY+LvwFzM0gY19emlAWwBOJMb0= +github.com/aws/aws-sdk-go-v2/credentials v1.6.1 h1:A39JYth2fFCx+omN/gib/jIppx3rRnt2r7UKPq7Mh5Y= +github.com/aws/aws-sdk-go-v2/credentials v1.6.1/go.mod h1:QyvQk1IYTqBWSi1T6UgT/W8DMxBVa5pVuLFSRLLhGf8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.7.0/go.mod h1:KqEkRkxm/+1Pd/rENRNbQpfblDBYeg5HDSqjB6ks8hA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.0 h1:OpZjuUy8Jt3CA1WgJgBC5Bz+uOjE5Ppx4NFTRaooUuA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.0/go.mod h1:5E1J3/TTYy6z909QNR0QnXGBpfESYGDqd3O0zqONghU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.0/go.mod h1:NO3Q5ZTTQtO2xIg2+xTXYDiT7knSejfeDm7WGDaOo0U= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 h1:kYQ3H1u0ANr9KEKlGs/jTLrBFPo8P8NaH/w7A01NeeM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18/go.mod h1:r506HmK5JDUh9+Mw4CfGJGSSoqIiLCndAuqXuhbv67Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.0/go.mod h1:anlUzBoEWglcUxUQwZA7HQOEVEnQALVZsizAapB2hq8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 h1:Z7IdFUONvTcvS7YuhtVxN99v2cCoHRXOS4mTr0B/pUc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18/go.mod h1:DkKMmksZVVyat+Y+r1dEOgJEfUeA7UngIHWeKsi0yNc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.5 h1:zPxLGWALExNepElO0gYgoqsbqTlt4ZCrhZ7XlfJ+Qlw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.5/go.mod h1:6ZBTuDmvpCOD4Sf1i2/I3PgftlEcDGgvi8ocq64oQEg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.4.0/go.mod h1:X5/JuOxPLU/ogICgDTtnpfaQzdQJO0yKDcpoxWLLJ8Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.0 h1:qGZWS/WgiFY+Zgad2u0gwBHpJxz6Ne401JE7iQI1nKs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.0/go.mod h1:Mq6AEc+oEjCUlBuLiK5YwW4shSOAKCQ3tXN0sQeYoBA= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.0 h1:r+37fBAonXAmRx2MX0naWDKZpAaP2AOQ22cf9Cg71GA= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.0/go.mod h1:WyLS5qwXHtjKAONYZq/4ewdd+hcVsa3LBu77Ow5uj3k= +github.com/aws/aws-sdk-go-v2/service/sso v1.5.0/go.mod h1:GsqaJOJeOfeYD88/2vHWKXegvDRofDqWwC5i48A2kgs= +github.com/aws/aws-sdk-go-v2/service/sso v1.6.0 h1:JDgKIUZOmLFu/Rv6zXLrVTWCmzA0jcTdvsT8iFIKrAI= +github.com/aws/aws-sdk-go-v2/service/sso v1.6.0/go.mod h1:Q/l0ON1annSU+mc0JybDy1Gy6dnJxIcWjphO6qJPzvM= +github.com/aws/aws-sdk-go-v2/service/sts v1.8.0/go.mod h1:dOlm91B439le5y1vtPCk5yJtbx3RdT3hRGYRY8TYKvQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.10.0 h1:1jh8J+JjYRp+QWKOsaZt7rGUgoyrqiiVwIm+w0ymeUw= +github.com/aws/aws-sdk-go-v2/service/sts v1.10.0/go.mod h1:jLKCFqS+1T4i7HDqCP9GM4Uk75YW1cS0o82LdxpMyOE= +github.com/aws/smithy-go v1.8.1/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.21.0 h1:H7L8dtDRk0P1Qm6y0ji7MCYMQObJ5R9CRpyPhRUkLYA= +github.com/aws/smithy-go v1.21.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -402,6 +439,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= diff --git a/storage/kv/dummy.go b/storage/kv/dummy.go new file mode 100644 index 00000000000..026497348ad --- /dev/null +++ b/storage/kv/dummy.go @@ -0,0 +1,41 @@ +package kv + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/smithy-go" +) + +// DummySecretsManagerClient is an in-memory implementation of the SecretsManagerClient interface. +type DummySecretsManagerClient struct { + secrets map[string]string +} + +// NewDummySecretsManagerClient creates a new DummySecretsManagerClient with the given secrets. +func NewDummySecretsManagerClient(secrets map[string]string) *DummySecretsManagerClient { + return &DummySecretsManagerClient{ + secrets: secrets, + } +} + +// GetSecretValue gets the value of a secret. +func (c *DummySecretsManagerClient) GetSecretValue( + _ context.Context, + params *secretsmanager.GetSecretValueInput, + _ ...func(*secretsmanager.Options), +) (*secretsmanager.GetSecretValueOutput, error) { + value, ok := c.secrets[*params.SecretId] + if !ok { + return nil, &smithy.GenericAPIError{ + Code: "ResourceNotFoundException", + Message: "The requested secret was not found.", + } + } + output := &secretsmanager.GetSecretValueOutput{ + Name: params.SecretId, + SecretString: aws.String(value), + } + return output, nil +} diff --git a/storage/kv/secretsmanager.go b/storage/kv/secretsmanager.go new file mode 100644 index 00000000000..f9428f3180c --- /dev/null +++ b/storage/kv/secretsmanager.go @@ -0,0 +1,108 @@ +package kv + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/smithy-go" + "github.com/pkg/errors" + + "github.com/TykTechnologies/tyk/config" +) + +// SecretsManagerClient is an interface for the AWS Secrets Manager client. +type SecretsManagerClient interface { + // GetSecretValue retrieves a secret from AWS Secrets Manager. + GetSecretValue( + ctx context.Context, + params *secretsmanager.GetSecretValueInput, + optFns ...func(*secretsmanager.Options), + ) (*secretsmanager.GetSecretValueOutput, error) +} + +// SecretsManager is an implementation of a KV store which uses AWS Secrets Manager as its backend. +type SecretsManager struct { + client SecretsManagerClient +} + +// NewSecretsManager returns an AWS Secrets Manager KV store with default configuration. +func NewSecretsManager() (*SecretsManager, error) { + return NewSecretsManagerWithConfig(config.SecretsManagerConfig{}) +} + +// NewSecretsManagerWithConfig returns an AWS Secrets Manager KV store with a custom configuration. +func NewSecretsManagerWithConfig(conf config.SecretsManagerConfig) (*SecretsManager, error) { + cfg, err := awsconfig.LoadDefaultConfig( + context.TODO(), + awsconfig.WithRegion(conf.Region), + awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(conf.AccessKeyID, conf.SecretAccessKey, ""), + ), + ) + if err != nil { + return nil, errors.Wrap(err, "error loading default config") + } + + client := secretsmanager.NewFromConfig(cfg) + + return &SecretsManager{client: client}, nil +} + +// NewSecretsManagerWithClient returns a configured AWS Secrets Manager KV store with a custom client. +func NewSecretsManagerWithClient(client SecretsManagerClient) *SecretsManager { + return &SecretsManager{client: client} +} + +// Get retrieves a secret from AWS Secrets Manager. Key is the secret name if the secret is a plain text value (e.g., +// "config") or the secret name followed by the key name if the secret is a JSON document (e.g., "config.node_secret"). +func (s *SecretsManager) Get(key string) (string, error) { + secretId, secretKey := splitKVPathAndKey(key) + + value, err := s.client.GetSecretValue( + context.TODO(), + &secretsmanager.GetSecretValueInput{SecretId: aws.String(secretId)}, + ) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + if apiErr.ErrorCode() == "ResourceNotFoundException" { + return "", ErrKeyNotFound + } + } + + return "", fmt.Errorf("error getting secret value: %w", err) + } + + if secretKey == "" { + return *value.SecretString, nil + } + + jsonValue := make(map[string]interface{}) + err = json.Unmarshal([]byte(*value.SecretString), &jsonValue) + if err != nil { + return "", fmt.Errorf("error unmarshalling secret string: %w", err) + } + + val, ok := jsonValue[secretKey] + if !ok { + return "", ErrKeyNotFound + } + + return fmt.Sprintf("%v", val), nil +} + +func splitKVPathAndKey(raw string) (path string, key string) { + sep := "." + parts := strings.Split(raw, sep) + if len(parts) > 1 { + return parts[0], strings.Join(parts[1:], sep) + } + + return raw, "" +} diff --git a/storage/kv/secretsmanager_test.go b/storage/kv/secretsmanager_test.go new file mode 100644 index 00000000000..69a2faf1362 --- /dev/null +++ b/storage/kv/secretsmanager_test.go @@ -0,0 +1,73 @@ +package kv + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSecretsManagerGet(t *testing.T) { + client := NewDummySecretsManagerClient(map[string]string{ + "key": "key::value", + "path/to/secret": "{\"key\":\"path/to/secret::key::value\", \"key.multiple.levels\":\"path/to/secret::key.multiple.levels::value\"}", + }) + secretsManager := NewSecretsManagerWithClient(client) + + tests := []struct { + name string + key string + expected string + err bool + }{ + + { + name: "Key", + key: "key", + expected: "key::value", + err: false, + }, + { + name: "KeyNotFound", + key: "notfound", + expected: "", + err: true, + }, + { + name: "PathAndKey", + key: "path/to/secret.key", + expected: "path/to/secret::key::value", + err: false, + }, + { + name: "PathAndKeyInvalidJSON", + key: "key.invalid", + expected: "", + err: true, + }, + { + name: "PathAndKeyMultipleLevels", + key: "path/to/secret.key.multiple.levels", + expected: "path/to/secret::key.multiple.levels::value", + err: false, + }, + { + name: "PathAndKeyNotFound", + key: "path/to/secret.notfound", + expected: "", + err: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual, err := secretsManager.Get(tt.key) + if tt.err { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + + assert.Equal(t, tt.expected, actual) + }) + } +}