Add Secrets Manager KV store (TykTechnologies#6563)

cli/linter/schema.json

@@ -1135,6 +1135,20 @@
             }
           }
         },
+        "secrets_manager": {
+          "type": ["object", "null"],
+          "properties": {
+            "access_key_id": {
+              "type": "string"
+            },
+            "secret_access_key": {
+              "type": "string"
+            },
+            "region": {
+              "type": "string"
+            }
+          }
+        },
         "vault": {
           "type": ["object", "null"],
           "properties": {

config/config.go

@@ -1069,8 +1069,9 @@ type Config struct {
 	// This section enables the use of the KV capabilities to substitute configuration values.
 	// See more details https://tyk.io/docs/tyk-configuration-reference/kv-store/
 	KV struct {
-		Consul ConsulConfig `json:"consul"`
-		Vault  VaultConfig  `json:"vault"`
+		Consul         ConsulConfig         `json:"consul"`
+		SecretsManager SecretsManagerConfig `json:"secrets_manager"`
+		Vault          VaultConfig          `json:"vault"`
 	} `json:"kv"`
 
 	// Secrets are key-value pairs that can be accessed in the dashboard via "secrets://"
@@ -1213,6 +1214,18 @@ type ConsulConfig struct {
 	} `json:"tls_config"`
 }
 
+// SecretsManagerConfig is used to configure the AWS Secrets Manager client.
+type SecretsManagerConfig struct {
+	// AccessKeyID is the AWS access key ID.
+	AccessKeyID string `json:"access_key_id"`
+
+	// SecretAccessKey is the AWS secret access key.
+	SecretAccessKey string `json:"secret_access_key"`
+
+	// Region is the AWS region.
+	Region string `json:"region"`
+}
+
 // GetEventTriggers returns event triggers. There was a typo in the json tag.
 // To maintain backward compatibility, this solution is chosen.
 func (c Config) GetEventTriggers() map[apidef.TykEvent][]TykEventHandler {

config/config_test.go

@@ -353,3 +353,23 @@ func TestPortsWhiteListDecoder(t *testing.T) {
 	assert.Contains(t, tlsWhiteList.Ports, 6000, "tls should have 6000 port")
 	assert.Contains(t, tlsWhiteList.Ports, 6015, "tls should have 6015 port")
 }
+
+func TestConfigKVSecretsManager(t *testing.T) {
+	var c Config
+
+	err := os.Setenv("TYK_GW_KV_SECRETSMANAGER_ACCESSKEYID", "AKIAIOSFODNN7EXAMPLE")
+	assert.NoError(t, err)
+
+	err = os.Setenv("TYK_GW_KV_SECRETSMANAGER_SECRETACCESSKEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+	assert.NoError(t, err)
+
+	err = os.Setenv("TYK_GW_KV_SECRETSMANAGER_REGION", "us-east-1")
+	assert.NoError(t, err)
+
+	err = envconfig.Process("TYK_GW", &c)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "AKIAIOSFODNN7EXAMPLE", c.KV.SecretsManager.AccessKeyID)
+	assert.Equal(t, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", c.KV.SecretsManager.SecretAccessKey)
+	assert.Equal(t, "us-east-1", c.KV.SecretsManager.Region)
+}

docs/kv.md

@@ -0,0 +1,91 @@
+# External Key-Value Storage
+
+Tyk Gateway supports storing configuration data in key-value (KV) stores such as AWS Secrets Manager, Consul, and Vault
+and then referencing these values the gateway configuration or API definitions deployed on the gateway.
+
+## Supported KV Stores
+
+Tyk Gateway supports the following KV stores:
+
+- [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/)
+- [Consul](https://www.consul.io/)
+- [Vault](https://www.vaultproject.io/)
+- Tyk Config Secrets
+- Environment Variables
+
+## Accessing KV Store Data
+
+You can configure Tyk Gateway to retrieve values from KV stores in the following places:
+
+- Tyk Gateway configuration file (tyk.conf)
+- API definitions
+- Transform Middleware
+
+### Gateway Configuration
+
+You can retrieve values from KV stores for the following fields Tyk Gateway configuration fields:
+
+- `secret`
+- `node_secret`
+- `storage.password`
+- `cache_storage.password`
+- `security.private_certificate_encoding_secret`
+- `db_app_conf_options.connection_string`
+- `policies.policy_connection_string`
+
+Use the following notation to reference KV store values in the gateway configuration file:
+
+| Store                       | Notation                                                    |
+|-----------------------------|-------------------------------------------------------------|
+| AWS Secrets Manager \[1\]   | `secretsmanager://{key}` or `secretsmanager://{path}.{key}` |
+| Consul                      | `consul://{key}`                                            |
+| Vault  \[1\]                | `vault://{path}.{key}`                                      |
+| Tyk config secrets          | `secrets://{key}`                                           |
+| Environment variables \[2\] | `env://{key}`                                               |
+
+\[1\] Path key value must be a JSON document to support `{path}.{key}` references (e.g., `{"key": "value"}`).\
+\[2\] Gateway configuration environment variable names must be prefixed with `TYK_SECRET_{KEY}` (e.g.,
+`TYK_SECRET_MY_API_SECRET` is referenced as `env://MY_API_SECRET`). Environment variables names must be uppercase.
+
+### API Definition
+
+Use the following notation to reference KV store values in API definition fields:
+
+| Store                       | Notation                 |
+|-----------------------------|--------------------------|
+| AWS Secrets Manager \[1\]   | `secretsmanager://{key}` |
+| Consul \[1\]                | `consul://{key}`         |
+| Vault \[1\]                 | `vault://{key}`          |
+| Tyk config secrets          | `secrets://{key}`        |
+| Environment variables \[2\] | `env://{key}`            |
+
+\[1\] External API definition secrets must be stored in a JSON document in the KV store with the key `tyk-apis`.
+In this case, `{key}` refers to the JSON document key (e.g., `{"key": "value"}`).\
+\[2\] API definition environment variables can be any name (e.g. `MY_API_SECRET` is referenced as
+`env://MY_API_SECRET`). They **do not** require the prefix `TYK_SECRET_{KEY}`.
+
+### Transform Middleware
+
+You can retrieve values from KV stores for the following transform middleware:
+
+- Authentication Token (signature secret)
+- Persist GraphQL Operation (request path)
+- Rate Limiting (rate limit pattern)
+- Request Body
+- Request Headers
+- Response Headers
+- URL Rewrite
+
+Use the following notation to reference KV store values in transform middleware:
+
+| Store                       | Notation                                                                |
+|-----------------------------|-------------------------------------------------------------------------|
+| AWS Secrets Manager \[1\]   | `$secret_secretsmanager.{key}` or `$secret_secretsmanager.{path}.{key}` |
+| Consul                      | `$secret_consul.{key}`                                                  |
+| Vault  \[1\]                | `$secret_vault.{path}.{key}`                                            |
+| Tyk config secrets          | `$secret_conf.{key}`                                                    |
+| Environment variables \[2\] | `$secret_env.{key}`                                                     |
+
+\[1\] Path key value must be a JSON document to support `{path}.{key}` references (e.g., `{"key": "value"}`).\
+\[2\] Transform middleware environment variable names must be prefixed with `TYK_SECRET_{KEY}` (e.g.,
+`TYK_SECRET_MY_API_SECRET` is referenced as `$secret_env.my_api_secret`).

gateway/api_definition.go

@@ -34,7 +34,7 @@ import (
 
 	"github.com/cenk/backoff"
 
-	sprig "github.com/Masterminds/sprig/v3"
+	"github.com/Masterminds/sprig/v3"
 
 	"github.com/sirupsen/logrus"
 
@@ -558,12 +558,13 @@ func (a APIDefinitionLoader) FromDashboardService(endpoint string) ([]*APISpec,
 var envRegex = regexp.MustCompile(`env://([^"]+)`)
 
 const (
-	prefixEnv       = "env://"
-	prefixSecrets   = "secrets://"
-	prefixConsul    = "consul://"
-	prefixVault     = "vault://"
-	prefixKeys      = "tyk-apis"
-	vaultSecretPath = "secret/data/"
+	prefixEnv            = "env://"
+	prefixSecrets        = "secrets://"
+	prefixConsul         = "consul://"
+	prefixSecretsManager = "secretsmanager://"
+	prefixVault          = "vault://"
+	prefixKeys           = "tyk-apis"
+	vaultSecretPath      = "secret/data/"
 )
 
 func (a APIDefinitionLoader) replaceSecrets(in []byte) []byte {
@@ -597,6 +598,12 @@ func (a APIDefinitionLoader) replaceSecrets(in []byte) []byte {
 		}
 	}
 
+	if strings.Contains(input, prefixSecretsManager) {
+		if err := a.replaceSecretsManagerSecrets(&input); err != nil {
+			log.WithError(err).Error("Couldn't replace Secrets Manager secrets")
+		}
+	}
+
 	if strings.Contains(input, prefixVault) {
 		if err := a.replaceVaultSecrets(&input); err != nil {
 			log.WithError(err).Error("Couldn't replace vault secrets")
@@ -624,6 +631,29 @@ func (a APIDefinitionLoader) replaceConsulSecrets(input *string) error {
 	return nil
 }
 
+func (a APIDefinitionLoader) replaceSecretsManagerSecrets(input *string) error {
+	if err := a.Gw.setUpSecretsManager(); err != nil {
+		return err
+	}
+
+	value, err := a.Gw.secretsManagerKVStore.Get(prefixKeys)
+	if err != nil {
+		return err
+	}
+
+	jsonValue := make(map[string]interface{})
+	err = json.Unmarshal([]byte(value), &jsonValue)
+	if err != nil {
+		return fmt.Errorf("error unmarshalling secret string: %w", err)
+	}
+
+	for k, v := range jsonValue {
+		*input = strings.Replace(*input, prefixSecretsManager+k, fmt.Sprintf("%v", v), -1)
+	}
+
+	return nil
+}
+
 func (a APIDefinitionLoader) replaceVaultSecrets(input *string) error {
 	if err := a.Gw.setUpVault(); err != nil {
 		return err
@@ -651,7 +681,7 @@ func (a APIDefinitionLoader) replaceVaultSecrets(input *string) error {
 	return nil
 }
 
-// FromCloud will connect and download ApiDefintions from a Mongo DB instance.
+// FromRPC will connect and download API definitions.
 func (a APIDefinitionLoader) FromRPC(store RPCDataLoader, orgId string, gw *Gateway) ([]*APISpec, error) {
 	if rpc.IsEmergencyMode() {
 		return gw.LoadDefinitionsFromRPCBackup()
@@ -759,6 +789,7 @@ func (a APIDefinitionLoader) FromDir(dir string) []*APISpec {
 	// Grab json files from directory
 	paths, _ := filepath.Glob(filepath.Join(dir, "*.json"))
 	for _, path := range paths {
+		// TODO(#6563): Why aren't OAS definitions loaded here? This prevents secrets from being replaced in OAS definitions.
 		if strings.HasSuffix(path, "-oas.json") {
 			continue
 		}
@@ -773,6 +804,7 @@ func (a APIDefinitionLoader) FromDir(dir string) []*APISpec {
 	}
 	return specs
 }
+
 func (a APIDefinitionLoader) loadDefFromFilePath(filePath string) (*APISpec, error) {
 	log.Info("Loading API Specification from ", filePath)
 

gateway/api_definition_test.go

@@ -18,10 +18,12 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	persistentmodel "github.com/TykTechnologies/storage/persistent/model"
+
 	"github.com/TykTechnologies/tyk/apidef"
 	"github.com/TykTechnologies/tyk/apidef/oas"
 	"github.com/TykTechnologies/tyk/config"
 	"github.com/TykTechnologies/tyk/rpc"
+	"github.com/TykTechnologies/tyk/storage/kv"
 	"github.com/TykTechnologies/tyk/test"
 	"github.com/TykTechnologies/tyk/user"
 )
@@ -1602,3 +1604,64 @@ func TestInternalEndpointMW_TT_11126(t *testing.T) {
 		{Path: "/headers", Code: http.StatusForbidden},
 	}...)
 }
+
+func TestAPIDefinitionReplaceSecrets(t *testing.T) {
+	ts := StartTest(func(globalConf *config.Config) {
+		globalConf.Secrets = map[string]string{
+			"jwt_signing_method": "secrets::jwt_signing_method::value",
+			"jwt_source":         "secrets::jwt_source::value",
+		}
+	})
+	defer ts.Close()
+
+	t.Setenv("JWT_SIGNING_METHOD", "env::jwt_signing_method::value")
+	t.Setenv("JWT_SOURCE", "env::jwt_source::value")
+
+	ts.Gw.secretsManagerKVStore = kv.NewSecretsManagerWithClient(kv.NewDummySecretsManagerClient(map[string]string{
+		"tyk-apis": "{" +
+			"\"jwt_signing_method\":\"secretsmanager::jwt_signing_method::value\"," +
+			"\"jwt_source\":\"secretsmanager::jwt_source::value\"" +
+			"}",
+	}))
+
+	tests := []struct {
+		name   string
+		scheme string
+		specs  []*APISpec
+	}{
+		{
+			name:   "Config",
+			scheme: "secrets",
+			specs: BuildAPI(func(spec *APISpec) {
+				spec.JWTSigningMethod = "secrets://jwt_signing_method"
+				spec.JWTSource = "secrets://jwt_source"
+			}),
+		},
+		{
+			name:   "Env",
+			scheme: "env",
+			specs: BuildAPI(func(spec *APISpec) {
+				spec.JWTSigningMethod = "env://JWT_SIGNING_METHOD"
+				spec.JWTSource = "env://JWT_SOURCE"
+			}),
+		},
+		{
+			name:   "SecretsManager",
+			scheme: "secretsmanager",
+			specs: BuildAPI(func(spec *APISpec) {
+				spec.JWTSigningMethod = "secretsmanager://jwt_signing_method"
+				spec.JWTSource = "secretsmanager://jwt_source"
+			}),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			specs := ts.Gw.LoadAPI(tt.specs...)
+			for _, spec := range specs {
+				assert.Equal(t, tt.scheme+"::jwt_signing_method::value", spec.JWTSigningMethod)
+				assert.Equal(t, tt.scheme+"::jwt_source::value", spec.JWTSource)
+			}
+		})
+	}
+}