Add Secrets Manager KV store (TykTechnologies#6563)

cli/linter/schema.json

@@ -1135,6 +1135,20 @@
             }
           }
         },
+        "secrets_manager": {
+          "type": ["object", "null"],
+          "properties": {
+            "access_key_id": {
+              "type": "string"
+            },
+            "secret_access_key": {
+              "type": "string"
+            },
+            "region": {
+              "type": "string"
+            }
+          }
+        },
         "vault": {
           "type": ["object", "null"],
           "properties": {

config/config.go

@@ -1069,8 +1069,9 @@ type Config struct {
 	// This section enables the use of the KV capabilities to substitute configuration values.
 	// See more details https://tyk.io/docs/tyk-configuration-reference/kv-store/
 	KV struct {
-		Consul ConsulConfig `json:"consul"`
-		Vault  VaultConfig  `json:"vault"`
+		Consul         ConsulConfig         `json:"consul"`
+		SecretsManager SecretsManagerConfig `json:"secrets_manager"`
+		Vault          VaultConfig          `json:"vault"`
 	} `json:"kv"`
 
 	// Secrets are key-value pairs that can be accessed in the dashboard via "secrets://"
@@ -1213,6 +1214,18 @@ type ConsulConfig struct {
 	} `json:"tls_config"`
 }
 
+// SecretsManagerConfig is used to configure the AWS Secrets Manager client.
+type SecretsManagerConfig struct {
+	// AccessKeyID is the AWS access key ID.
+	AccessKeyID string `json:"access_key_id"`
+
+	// SecretAccessKey is the AWS secret access key.
+	SecretAccessKey string `json:"secret_access_key"`
+
+	// Region is the AWS region.
+	Region string `json:"region"`
+}
+
 // GetEventTriggers returns event triggers. There was a typo in the json tag.
 // To maintain backward compatibility, this solution is chosen.
 func (c Config) GetEventTriggers() map[apidef.TykEvent][]TykEventHandler {

config/config_test.go

@@ -353,3 +353,23 @@ func TestPortsWhiteListDecoder(t *testing.T) {
 	assert.Contains(t, tlsWhiteList.Ports, 6000, "tls should have 6000 port")
 	assert.Contains(t, tlsWhiteList.Ports, 6015, "tls should have 6015 port")
 }
+
+func TestConfigKVSecretsManager(t *testing.T) {
+	var c Config
+
+	err := os.Setenv("TYK_GW_KV_SECRETSMANAGER_ACCESSKEYID", "AKIAIOSFODNN7EXAMPLE")
+	assert.NoError(t, err)
+
+	err = os.Setenv("TYK_GW_KV_SECRETSMANAGER_SECRETACCESSKEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+	assert.NoError(t, err)
+
+	err = os.Setenv("TYK_GW_KV_SECRETSMANAGER_REGION", "us-east-1")
+	assert.NoError(t, err)
+
+	err = envconfig.Process("TYK_GW", &c)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "AKIAIOSFODNN7EXAMPLE", c.KV.SecretsManager.AccessKeyID)
+	assert.Equal(t, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", c.KV.SecretsManager.SecretAccessKey)
+	assert.Equal(t, "us-east-1", c.KV.SecretsManager.Region)
+}

gateway/api_definition.go

@@ -558,12 +558,13 @@ func (a APIDefinitionLoader) FromDashboardService(endpoint string) ([]*APISpec,
 var envRegex = regexp.MustCompile(`env://([^"]+)`)
 
 const (
-	prefixEnv       = "env://"
-	prefixSecrets   = "secrets://"
-	prefixConsul    = "consul://"
-	prefixVault     = "vault://"
-	prefixKeys      = "tyk-apis"
-	vaultSecretPath = "secret/data/"
+	prefixEnv            = "env://"
+	prefixSecrets        = "secrets://"
+	prefixConsul         = "consul://"
+	prefixSecretsManager = "secretsmanager://"
+	prefixVault          = "vault://"
+	prefixKeys           = "tyk-apis"
+	vaultSecretPath      = "secret/data/"
 )
 
 func (a APIDefinitionLoader) replaceSecrets(in []byte) []byte {
@@ -597,6 +598,12 @@ func (a APIDefinitionLoader) replaceSecrets(in []byte) []byte {
 		}
 	}
 
+	if strings.Contains(input, prefixSecretsManager) {
+		if err := a.replaceSecretsManagerSecrets(&input); err != nil {
+			log.WithError(err).Error("Couldn't replace Secrets Manager secrets")
+		}
+	}
+
 	if strings.Contains(input, prefixVault) {
 		if err := a.replaceVaultSecrets(&input); err != nil {
 			log.WithError(err).Error("Couldn't replace vault secrets")
@@ -624,6 +631,29 @@ func (a APIDefinitionLoader) replaceConsulSecrets(input *string) error {
 	return nil
 }
 
+func (a APIDefinitionLoader) replaceSecretsManagerSecrets(input *string) error {
+	if err := a.Gw.setUpSecretsManager(); err != nil {
+		return err
+	}
+
+	value, err := a.Gw.secretsManagerKVStore.Get(prefixKeys)
+	if err != nil {
+		return err
+	}
+
+	jsonValue := make(map[string]interface{})
+	err = json.Unmarshal([]byte(value), &jsonValue)
+	if err != nil {
+		return fmt.Errorf("error unmarshalling secret string: %w", err)
+	}
+
+	for k, v := range jsonValue {
+		*input = strings.Replace(*input, prefixSecretsManager+k, fmt.Sprintf("%v", v), -1)
+	}
+
+	return nil
+}
+
 func (a APIDefinitionLoader) replaceVaultSecrets(input *string) error {
 	if err := a.Gw.setUpVault(); err != nil {
 		return err

gateway/api_definition_test.go

@@ -2,6 +2,7 @@ package gateway
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -22,6 +23,7 @@ import (
 	"github.com/TykTechnologies/tyk/apidef/oas"
 	"github.com/TykTechnologies/tyk/config"
 	"github.com/TykTechnologies/tyk/rpc"
+	"github.com/TykTechnologies/tyk/storage/kv"
 	"github.com/TykTechnologies/tyk/test"
 	"github.com/TykTechnologies/tyk/user"
 )
@@ -1602,3 +1604,30 @@ func TestInternalEndpointMW_TT_11126(t *testing.T) {
 		{Path: "/headers", Code: http.StatusForbidden},
 	}...)
 }
+
+func TestReplaceSecretsManagerSecrets(t *testing.T) {
+	client := kv.NewDummySecretsManagerClient(map[string]string{
+		"tyk-apis": "{\"jwt_source\":\"https://localhost/.well-known/openid-configuration\"}",
+	})
+	store := kv.NewSecretsManagerWithClient(client)
+
+	gw := NewGateway(config.Config{}, context.Background())
+	gw.secretsManagerKVStore = store
+	loader := APIDefinitionLoader{Gw: gw}
+
+	specs := BuildAPI(func(spec *APISpec) { spec.JWTSource = "secretsmanager://jwt_source" })
+
+	jsonSpec, err := json.Marshal(specs[0])
+	assert.NoError(t, err)
+	actual := string(jsonSpec)
+
+	err = loader.replaceSecretsManagerSecrets(&actual)
+	assert.NoError(t, err)
+
+	specs[0].JWTSource = "https://localhost/.well-known/openid-configuration"
+	jsonSpec, err = json.Marshal(specs[0])
+	assert.NoError(t, err)
+	expected := string(jsonSpec)
+
+	assert.Equal(t, expected, actual)
+}

gateway/mw_url_rewrite.go

@@ -20,19 +20,21 @@ import (
 )
 
 const (
-	metaLabel        = "$tyk_meta."
-	contextLabel     = "$tyk_context."
-	consulLabel      = "$secret_consul."
-	vaultLabel       = "$secret_vault."
-	envLabel         = "$secret_env."
-	secretsConfLabel = "$secret_conf."
-	triggerKeyPrefix = "trigger"
-	triggerKeySep    = "-"
+	metaLabel           = "$tyk_meta."
+	contextLabel        = "$tyk_context."
+	consulLabel         = "$secret_consul."
+	secretsManagerLabel = "$secret_secretsmanager."
+	vaultLabel          = "$secret_vault."
+	envLabel            = "$secret_env."
+	secretsConfLabel    = "$secret_conf."
+	triggerKeyPrefix    = "trigger"
+	triggerKeySep       = "-"
 )
 
 var dollarMatch = regexp.MustCompile(`\$\d+`)
 var contextMatch = regexp.MustCompile(`\$tyk_context.([A-Za-z0-9_\-\.]+)`)
 var consulMatch = regexp.MustCompile(`\$secret_consul.([A-Za-z0-9\/\-\.]+)`)
+var secretsManagerMatch = regexp.MustCompile(`\$secret_secretsmanager\.([A-Za-z0-9\/\-\._]+)`)
 var vaultMatch = regexp.MustCompile(`\$secret_vault.([A-Za-z0-9\/\-\.]+)`)
 var envValueMatch = regexp.MustCompile(`\$secret_env.([A-Za-z0-9_\-\.]+)`)
 var metaMatch = regexp.MustCompile(`\$tyk_meta.([A-Za-z0-9_\-\.]+)`)
@@ -211,7 +213,6 @@ func (gw *Gateway) urlRewrite(meta *apidef.URLRewriteMeta, r *http.Request) (str
 }
 
 func (gw *Gateway) replaceTykVariables(r *http.Request, in string, escape bool) string {
-
 	if strings.Contains(in, secretsConfLabel) {
 		contextData := ctxGetData(r)
 		vars := secretsConfMatch.FindAllString(in, -1)
@@ -251,12 +252,18 @@ func (gw *Gateway) replaceTykVariables(r *http.Request, in string, escape bool)
 			in = gw.replaceVariables(in, vars, session.MetaData, metaLabel, escape)
 		}
 	}
+
+	if strings.Contains(in, secretsManagerLabel) {
+		contextData := ctxGetData(r)
+		vars := secretsManagerMatch.FindAllString(in, -1)
+		in = gw.replaceVariables(in, vars, contextData, secretsManagerLabel, escape)
+	}
+
 	//todo add config_data
 	return in
 }
 
 func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]interface{}, label string, escape bool) string {
-
 	emptyStringFn := func(key, in, val string) string {
 		in = strings.Replace(in, val, "", -1)
 		log.WithFields(logrus.Fields{
@@ -272,9 +279,7 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in
 		key := strings.Replace(v, label, "", 1)
 
 		switch label {
-
 		case secretsConfLabel:
-
 			secrets := gw.GetConfig().Secrets
 
 			val, ok := secrets[key]
@@ -284,19 +289,15 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in
 			}
 
 			in = strings.Replace(in, v, val, -1)
-
 		case envLabel:
-
 			val := os.Getenv(fmt.Sprintf("TYK_SECRET_%s", strings.ToUpper(key)))
 			if val == "" {
 				in = emptyStringFn(key, in, v)
 				continue
 			}
 
 			in = strings.Replace(in, v, val, -1)
-
 		case vaultLabel:
-
 			if err := gw.setUpVault(); err != nil {
 				in = emptyStringFn(key, in, v)
 				continue
@@ -309,9 +310,7 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in
 			}
 
 			in = strings.Replace(in, v, val, -1)
-
 		case consulLabel:
-
 			if err := gw.setUpConsul(); err != nil {
 				in = emptyStringFn(key, in, v)
 				continue
@@ -324,9 +323,20 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in
 			}
 
 			in = strings.Replace(in, v, val, -1)
+		case secretsManagerLabel:
+			if err := gw.setUpSecretsManager(); err != nil {
+				in = emptyStringFn(key, in, v)
+				continue
+			}
 
-		default:
+			val, err := gw.secretsManagerKVStore.Get(key)
+			if err != nil {
+				in = strings.Replace(in, v, "", -1)
+				continue
+			}
 
+			in = strings.Replace(in, v, val, -1)
+		default:
 			val, ok := vals[key]
 			if ok {
 				valStr := valToStr(val)

gateway/mw_url_rewrite_test.go

@@ -2,13 +2,16 @@ package gateway
 
 import (
 	"bytes"
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/TykTechnologies/tyk/config"
 	"github.com/TykTechnologies/tyk/ctx"
+	"github.com/TykTechnologies/tyk/storage/kv"
 
 	"github.com/TykTechnologies/tyk/test"
 
@@ -1384,3 +1387,64 @@ func TestURLRewriteMiddleware_CheckHostRewrite(t *testing.T) {
 		})
 	}
 }
+
+func TestReplaceTykVariables(t *testing.T) {
+	client := kv.NewDummySecretsManagerClient(map[string]string{
+		"secret":         "secret::value",
+		"path":           "{\"key\":\"path::key::value\"}",
+		"path-to-secret": "path-to-secret::value",
+		"path/to/secret": "path/to/secret::value",
+		"path_to_secret": "path_to_secret::value",
+	})
+	store := kv.NewSecretsManagerWithClient(client)
+
+	gw := NewGateway(config.Config{}, context.Background())
+	gw.secretsManagerKVStore = store
+
+	tests := []struct {
+		name     string
+		in       string
+		escape   bool
+		expected string
+	}{
+		{
+			name:     "SecretsManager/Path",
+			in:       "$secret_secretsmanager.secret",
+			escape:   false,
+			expected: "secret::value",
+		},
+		{
+			name:     "SecretsManager/PathAndKey",
+			in:       "$secret_secretsmanager.path.key",
+			escape:   false,
+			expected: "path::key::value",
+		},
+		{
+			name:     "SecretsManager/PathWithDash",
+			in:       "$secret_secretsmanager.path-to-secret",
+			escape:   false,
+			expected: "path-to-secret::value",
+		},
+		{
+			name:     "SecretsManager/PathWithForwardSlash",
+			in:       "$secret_secretsmanager.path/to/secret",
+			escape:   false,
+			expected: "path/to/secret::value",
+		},
+		{
+			name:     "SecretsManager/PathWithUnderscore",
+			in:       "$secret_secretsmanager.path_to_secret",
+			escape:   false,
+			expected: "path_to_secret::value",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			req, err := http.NewRequestWithContext(context.TODO(), "GET", "/test/replace/tyk/variables", nil)
+			assert.NoError(t, err)
+			actual := gw.replaceTykVariables(req, tt.in, tt.escape)
+			assert.Equal(t, tt.expected, actual)
+		})
+	}
+}