From 75338ac728caa2a898a67ed51ae5cc22dc94f878 Mon Sep 17 00:00:00 2001 From: Pravin Pushkar Date: Tue, 1 Aug 2023 09:20:01 +0530 Subject: [PATCH 01/22] Azure App Config: fix time format to time.duration (#3004) Signed-off-by: Pravin Pushkar Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Bernd Verst Co-authored-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- configuration/azure/appconfig/appconfig.go | 69 +++------ .../azure/appconfig/appconfig_test.go | 132 ++++++++++-------- configuration/azure/appconfig/metadata.go | 70 ++++++++-- configuration/azure/appconfig/metadata.yaml | 32 ++--- 4 files changed, 160 insertions(+), 143 deletions(-) diff --git a/configuration/azure/appconfig/appconfig.go b/configuration/azure/appconfig/appconfig.go index a18f5d367c..7e52d8e819 100644 --- a/configuration/azure/appconfig/appconfig.go +++ b/configuration/azure/appconfig/appconfig.go @@ -74,21 +74,21 @@ func NewAzureAppConfigurationStore(logger logger.Logger) configuration.Store { } // Init does metadata and connection parsing. -func (r *ConfigurationStore) Init(_ context.Context, metadata configuration.Metadata) error { - m, err := parseMetadata(metadata) +func (r *ConfigurationStore) Init(_ context.Context, md configuration.Metadata) error { + r.metadata = metadata{} + err := r.metadata.Parse(r.logger, md) if err != nil { return err } - r.metadata = m coreClientOpts := azcore.ClientOptions{ Telemetry: policy.TelemetryOptions{ ApplicationID: "dapr-" + logger.DaprVersion, }, Retry: policy.RetryOptions{ - MaxRetries: int32(m.MaxRetries), - RetryDelay: m.internalMaxRetryDelay, - MaxRetryDelay: m.internalMaxRetryDelay, + MaxRetries: int32(r.metadata.MaxRetries), + RetryDelay: r.metadata.MaxRetryDelay, + MaxRetryDelay: r.metadata.MaxRetryDelay, }, } @@ -103,7 +103,7 @@ func (r *ConfigurationStore) Init(_ context.Context, metadata configuration.Meta } } else { var settings azauth.EnvironmentSettings - settings, err = azauth.NewEnvironmentSettings(metadata.Properties) + settings, err = azauth.NewEnvironmentSettings(md.Properties) if err != nil { return err } @@ -123,43 +123,6 @@ func (r *ConfigurationStore) Init(_ context.Context, metadata configuration.Meta return nil } -func parseMetadata(meta configuration.Metadata) (metadata, error) { - m := metadata{ - MaxRetries: defaultMaxRetries, - internalMaxRetryDelay: defaultMaxRetryDelay, - internalRetryDelay: defaultRetryDelay, - internalSubscribePollInterval: defaultSubscribePollInterval, - internalRequestTimeout: defaultRequestTimeout, - } - decodeErr := contribMetadata.DecodeMetadata(meta.Properties, &m) - if decodeErr != nil { - return m, decodeErr - } - - if m.ConnectionString != "" && m.Host != "" { - return m, fmt.Errorf("azure appconfig error: can't set both %s and %s fields in metadata", host, connectionString) - } - - if m.ConnectionString == "" && m.Host == "" { - return m, fmt.Errorf("azure appconfig error: specify %s or %s field in metadata", host, connectionString) - } - - if m.MaxRetryDelay != nil { - m.internalMaxRetryDelay = time.Duration(*m.MaxRetryDelay) - } - if m.RetryDelay != nil { - m.internalRetryDelay = time.Duration(*m.RetryDelay) - } - if m.SubscribePollInterval != nil { - m.internalSubscribePollInterval = time.Duration(*m.SubscribePollInterval) - } - if m.RequestTimeout != nil { - m.internalRequestTimeout = time.Duration(*m.RequestTimeout) - } - - return m, nil -} - func (r *ConfigurationStore) Get(ctx context.Context, req *configuration.GetRequest) (*configuration.GetResponse, error) { keys := req.Keys var items map[string]*configuration.Item @@ -216,7 +179,7 @@ func (r *ConfigurationStore) getAll(ctx context.Context, req *configuration.GetR nil) for allSettingsPgr.More() { - timeoutContext, cancel := context.WithTimeout(ctx, r.metadata.internalRequestTimeout) + timeoutContext, cancel := context.WithTimeout(ctx, r.metadata.RequestTimeout) defer cancel() if revResp, err := allSettingsPgr.NextPage(timeoutContext); err == nil { for _, setting := range revResp.Settings { @@ -248,11 +211,11 @@ func (r *ConfigurationStore) getLabelFromMetadata(metadata map[string]string) *s func (r *ConfigurationStore) Subscribe(ctx context.Context, req *configuration.SubscribeRequest, handler configuration.UpdateHandler) (string, error) { sentinelKey := r.getSentinelKeyFromMetadata(req.Metadata) if sentinelKey == "" { - return "", fmt.Errorf("azure appconfig error: sentinel key is not provided in metadata") + return "", fmt.Errorf("sentinel key is not provided in metadata") } uuid, err := uuid.NewRandom() if err != nil { - return "", fmt.Errorf("azure appconfig error: failed to generate uuid, error is %w", err) + return "", fmt.Errorf("failed to generate uuid, error is %w", err) } subscribeID := uuid.String() childContext, cancel := context.WithCancel(ctx) @@ -277,7 +240,7 @@ func (r *ConfigurationStore) doSubscribe(ctx context.Context, req *configuration if errors.Is(err, context.Canceled) { return } - r.logger.Debugf("azure appconfig error: fail to get sentinel key or sentinel's key %s value is unchanged: %s", sentinelKey, err) + r.logger.Debugf("Failed to get sentinel key or sentinel's key %s value is unchanged: %s", sentinelKey, err) } else { // if sentinel key has changed then update the Etag value. etagVal = resp.ETag @@ -289,7 +252,7 @@ func (r *ConfigurationStore) doSubscribe(ctx context.Context, req *configuration if errors.Is(err, context.Canceled) { return } - r.logger.Errorf("azure appconfig error: fail to get configuration key changes: %s", err) + r.logger.Errorf("Failed to get configuration key changes: %s", err) } else { r.handleSubscribedChange(ctx, handler, items, id) } @@ -297,13 +260,13 @@ func (r *ConfigurationStore) doSubscribe(ctx context.Context, req *configuration select { case <-ctx.Done(): return - case <-time.After(r.metadata.internalSubscribePollInterval): + case <-time.After(r.metadata.SubscribePollInterval): } } } func (r *ConfigurationStore) getSettings(ctx context.Context, key string, getSettingsOptions *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { - timeoutContext, cancel := context.WithTimeout(ctx, r.metadata.internalRequestTimeout) + timeoutContext, cancel := context.WithTimeout(ctx, r.metadata.RequestTimeout) defer cancel() resp, err := r.client.GetSetting(timeoutContext, key, getSettingsOptions) return resp, err @@ -316,7 +279,7 @@ func (r *ConfigurationStore) handleSubscribedChange(ctx context.Context, handler } err := handler(ctx, e) if err != nil { - r.logger.Errorf("azure appconfig error: fail to call handler to notify event for configuration update subscribe: %s", err) + r.logger.Errorf("Failed to call handler to notify event for configuration update subscribe: %s", err) } } @@ -334,7 +297,7 @@ func (r *ConfigurationStore) Unsubscribe(ctx context.Context, req *configuration cancelContext.(context.CancelFunc)() return nil } - return fmt.Errorf("azure appconfig error: subscription with id %s does not exist", req.ID) + return fmt.Errorf("subscription with id %s does not exist", req.ID) } // GetComponentMetadata returns the metadata of the component. diff --git a/configuration/azure/appconfig/appconfig_test.go b/configuration/azure/appconfig/appconfig_test.go index 371f0e1f7d..31a49b8a10 100644 --- a/configuration/azure/appconfig/appconfig_test.go +++ b/configuration/azure/appconfig/appconfig_test.go @@ -22,6 +22,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/dapr/components-contrib/configuration" mdata "github.com/dapr/components-contrib/metadata" @@ -31,6 +32,11 @@ import ( type MockConfigurationStore struct{} +const ( + testMaxRetryDelay = "120s" + testSubscribePollIntervalAndReqTimeout = "30s" +) + func (m *MockConfigurationStore) GetSetting(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { if key == "testKey" || key == "test_sentinel_key" { settings := azappconfig.Setting{} @@ -192,10 +198,10 @@ func TestInit(t *testing.T) { testProperties := make(map[string]string) testProperties[host] = "testHost" testProperties[maxRetries] = "3" - testProperties[retryDelay] = "4000000000" - testProperties[maxRetryDelay] = "120000000000" - testProperties[subscribePollInterval] = "30000000000" - testProperties[requestTimeout] = "30000000000" + testProperties[retryDelay] = "4s" + testProperties[maxRetryDelay] = testMaxRetryDelay + testProperties[subscribePollInterval] = testSubscribePollIntervalAndReqTimeout + testProperties[requestTimeout] = testSubscribePollIntervalAndReqTimeout m := configuration.Metadata{Base: mdata.Base{ Properties: testProperties, @@ -207,20 +213,20 @@ func TestInit(t *testing.T) { assert.True(t, ok) assert.Equal(t, testProperties[host], cs.metadata.Host) assert.Equal(t, 3, cs.metadata.MaxRetries) - assert.Equal(t, time.Second*4, cs.metadata.internalRetryDelay) - assert.Equal(t, time.Second*120, cs.metadata.internalMaxRetryDelay) - assert.Equal(t, time.Second*30, cs.metadata.internalSubscribePollInterval) - assert.Equal(t, time.Second*30, cs.metadata.internalRequestTimeout) + assert.Equal(t, time.Second*4, cs.metadata.RetryDelay) + assert.Equal(t, time.Second*120, cs.metadata.MaxRetryDelay) + assert.Equal(t, time.Second*30, cs.metadata.SubscribePollInterval) + assert.Equal(t, time.Second*30, cs.metadata.RequestTimeout) }) t.Run("Init with valid appConfigConnectionString metadata", func(t *testing.T) { testProperties := make(map[string]string) testProperties[connectionString] = "Endpoint=https://foo.azconfig.io;Id=osOX-l9-s0:sig;Secret=00000000000000000000000000000000000000000000" testProperties[maxRetries] = "3" - testProperties[retryDelay] = "4000000000" - testProperties[maxRetryDelay] = "120000000000" - testProperties[subscribePollInterval] = "30000000000" - testProperties[requestTimeout] = "30000000000" + testProperties[retryDelay] = "4s" + testProperties[maxRetryDelay] = testMaxRetryDelay + testProperties[subscribePollInterval] = testSubscribePollIntervalAndReqTimeout + testProperties[requestTimeout] = testSubscribePollIntervalAndReqTimeout m := configuration.Metadata{Base: mdata.Base{ Properties: testProperties, @@ -232,76 +238,78 @@ func TestInit(t *testing.T) { assert.True(t, ok) assert.Equal(t, testProperties[connectionString], cs.metadata.ConnectionString) assert.Equal(t, 3, cs.metadata.MaxRetries) - assert.Equal(t, time.Second*4, cs.metadata.internalRetryDelay) - assert.Equal(t, time.Second*120, cs.metadata.internalMaxRetryDelay) - assert.Equal(t, time.Second*30, cs.metadata.internalSubscribePollInterval) - assert.Equal(t, time.Second*30, cs.metadata.internalRequestTimeout) + assert.Equal(t, time.Second*4, cs.metadata.RetryDelay) + assert.Equal(t, time.Second*120, cs.metadata.MaxRetryDelay) + assert.Equal(t, time.Second*30, cs.metadata.SubscribePollInterval) + assert.Equal(t, time.Second*30, cs.metadata.RequestTimeout) }) } -func Test_parseMetadata(t *testing.T) { +func TestParseMetadata(t *testing.T) { t.Run(fmt.Sprintf("parse metadata with %s", host), func(t *testing.T) { testProperties := make(map[string]string) testProperties[host] = "testHost" testProperties[maxRetries] = "3" - testProperties[retryDelay] = "4000000000" - testProperties[maxRetryDelay] = "120000000000" - testProperties[subscribePollInterval] = "30000000000" - testProperties[requestTimeout] = "30000000000" + testProperties[retryDelay] = "4s" + testProperties[maxRetryDelay] = testMaxRetryDelay + testProperties[subscribePollInterval] = testSubscribePollIntervalAndReqTimeout + testProperties[requestTimeout] = testSubscribePollIntervalAndReqTimeout meta := configuration.Metadata{Base: mdata.Base{ Properties: testProperties, }} want := metadata{ - Host: "testHost", - MaxRetries: 3, - internalRetryDelay: time.Second * 4, - internalMaxRetryDelay: time.Second * 120, - internalSubscribePollInterval: time.Second * 30, - internalRequestTimeout: time.Second * 30, + Host: "testHost", + MaxRetries: 3, + RetryDelay: time.Second * 4, + MaxRetryDelay: time.Second * 120, + SubscribePollInterval: time.Second * 30, + RequestTimeout: time.Second * 30, } - m, _ := parseMetadata(meta) - assert.NotNil(t, m) + m := metadata{} + err := m.Parse(logger.NewLogger("test"), meta) + require.NoError(t, err) assert.Equal(t, want.Host, m.Host) assert.Equal(t, want.MaxRetries, m.MaxRetries) - assert.Equal(t, want.internalRetryDelay, m.internalRetryDelay) - assert.Equal(t, want.internalMaxRetryDelay, m.internalMaxRetryDelay) - assert.Equal(t, want.internalSubscribePollInterval, m.internalSubscribePollInterval) - assert.Equal(t, want.internalRequestTimeout, m.internalRequestTimeout) + assert.Equal(t, want.RetryDelay, m.RetryDelay) + assert.Equal(t, want.MaxRetryDelay, m.MaxRetryDelay) + assert.Equal(t, want.SubscribePollInterval, m.SubscribePollInterval) + assert.Equal(t, want.RequestTimeout, m.RequestTimeout) }) t.Run(fmt.Sprintf("parse metadata with %s", connectionString), func(t *testing.T) { testProperties := make(map[string]string) testProperties[connectionString] = "testConnectionString" testProperties[maxRetries] = "3" - testProperties[retryDelay] = "4000000000" - testProperties[maxRetryDelay] = "120000000000" - testProperties[subscribePollInterval] = "30000000000" - testProperties[requestTimeout] = "30000000000" + testProperties[retryDelay] = "4s" + testProperties[maxRetryDelay] = testMaxRetryDelay + testProperties[subscribePollInterval] = testSubscribePollIntervalAndReqTimeout + testProperties[requestTimeout] = testSubscribePollIntervalAndReqTimeout meta := configuration.Metadata{Base: mdata.Base{ Properties: testProperties, }} want := metadata{ - ConnectionString: "testConnectionString", - MaxRetries: 3, - internalRetryDelay: time.Second * 4, - internalMaxRetryDelay: time.Second * 120, - internalSubscribePollInterval: time.Second * 30, - internalRequestTimeout: time.Second * 30, + ConnectionString: "testConnectionString", + MaxRetries: 3, + RetryDelay: time.Second * 4, + MaxRetryDelay: time.Second * 120, + SubscribePollInterval: time.Second * 30, + RequestTimeout: time.Second * 30, } - m, _ := parseMetadata(meta) - assert.NotNil(t, m) + m := metadata{} + err := m.Parse(logger.NewLogger("test"), meta) + require.NoError(t, err) assert.Equal(t, want.ConnectionString, m.ConnectionString) assert.Equal(t, want.MaxRetries, m.MaxRetries) - assert.Equal(t, want.internalRetryDelay, m.internalRetryDelay) - assert.Equal(t, want.internalMaxRetryDelay, m.internalMaxRetryDelay) - assert.Equal(t, want.internalSubscribePollInterval, m.internalSubscribePollInterval) - assert.Equal(t, want.internalRequestTimeout, m.internalRequestTimeout) + assert.Equal(t, want.RetryDelay, m.RetryDelay) + assert.Equal(t, want.MaxRetryDelay, m.MaxRetryDelay) + assert.Equal(t, want.SubscribePollInterval, m.SubscribePollInterval) + assert.Equal(t, want.RequestTimeout, m.RequestTimeout) }) t.Run(fmt.Sprintf("both %s and %s fields set in metadata", host, connectionString), func(t *testing.T) { @@ -309,17 +317,18 @@ func Test_parseMetadata(t *testing.T) { testProperties[host] = "testHost" testProperties[connectionString] = "testConnectionString" testProperties[maxRetries] = "3" - testProperties[retryDelay] = "4000000000" - testProperties[maxRetryDelay] = "120000000000" - testProperties[subscribePollInterval] = "30000000000" - testProperties[requestTimeout] = "30000000000" + testProperties[retryDelay] = "4s" + testProperties[maxRetryDelay] = testMaxRetryDelay + testProperties[subscribePollInterval] = testSubscribePollIntervalAndReqTimeout + testProperties[requestTimeout] = testSubscribePollIntervalAndReqTimeout meta := configuration.Metadata{Base: mdata.Base{ Properties: testProperties, }} - _, err := parseMetadata(meta) - assert.Error(t, err) + m := metadata{} + err := m.Parse(logger.NewLogger("test"), meta) + require.Error(t, err) }) t.Run(fmt.Sprintf("both %s and %s fields not set in metadata", host, connectionString), func(t *testing.T) { @@ -327,17 +336,18 @@ func Test_parseMetadata(t *testing.T) { testProperties[host] = "" testProperties[connectionString] = "" testProperties[maxRetries] = "3" - testProperties[retryDelay] = "4000000000" - testProperties[maxRetryDelay] = "120000000000" - testProperties[subscribePollInterval] = "30000000000" - testProperties[requestTimeout] = "30000000000" + testProperties[retryDelay] = "4s" + testProperties[maxRetryDelay] = testMaxRetryDelay + testProperties[subscribePollInterval] = testSubscribePollIntervalAndReqTimeout + testProperties[requestTimeout] = testSubscribePollIntervalAndReqTimeout meta := configuration.Metadata{Base: mdata.Base{ Properties: testProperties, }} - _, err := parseMetadata(meta) - assert.Error(t, err) + m := metadata{} + err := m.Parse(logger.NewLogger("test"), meta) + require.Error(t, err) }) } diff --git a/configuration/azure/appconfig/metadata.go b/configuration/azure/appconfig/metadata.go index 75d7de35a4..79b12b963e 100644 --- a/configuration/azure/appconfig/metadata.go +++ b/configuration/azure/appconfig/metadata.go @@ -13,19 +13,63 @@ limitations under the License. package appconfig -import "time" +import ( + "fmt" + "time" + + "github.com/dapr/components-contrib/configuration" + contribMetadata "github.com/dapr/components-contrib/metadata" + "github.com/dapr/kit/logger" +) type metadata struct { - Host string `mapstructure:"host"` - ConnectionString string `mapstructure:"connectionString"` - MaxRetries int `mapstructure:"maxRetries"` - MaxRetryDelay *int `mapstructure:"maxRetryDelay"` - RetryDelay *int `mapstructure:"retryDelay"` - SubscribePollInterval *int `mapstructure:"subscribePollInterval"` - RequestTimeout *int `mapstructure:"requestTimeout"` - - internalRequestTimeout time.Duration `mapstructure:"-"` - internalMaxRetryDelay time.Duration `mapstructure:"-"` - internalSubscribePollInterval time.Duration `mapstructure:"-"` - internalRetryDelay time.Duration `mapstructure:"-"` + Host string `mapstructure:"host"` + ConnectionString string `mapstructure:"connectionString"` + MaxRetries int `mapstructure:"maxRetries"` + MaxRetryDelay time.Duration `mapstructure:"maxRetryDelay"` + RetryDelay time.Duration `mapstructure:"retryDelay"` + SubscribePollInterval time.Duration `mapstructure:"subscribePollInterval"` + RequestTimeout time.Duration `mapstructure:"requestTimeout"` +} + +func (m *metadata) Parse(log logger.Logger, meta configuration.Metadata) error { + // Set defaults + m.MaxRetries = defaultMaxRetries + m.MaxRetryDelay = defaultMaxRetryDelay + m.RetryDelay = defaultRetryDelay + m.SubscribePollInterval = defaultSubscribePollInterval + m.RequestTimeout = defaultRequestTimeout + + // Decode the metadata + decodeErr := contribMetadata.DecodeMetadata(meta.Properties, m) + if decodeErr != nil { + return decodeErr + } + + // Validate options + if m.ConnectionString != "" && m.Host != "" { + return fmt.Errorf("azure appconfig error: can't set both %s and %s fields in metadata", host, connectionString) + } + + if m.ConnectionString == "" && m.Host == "" { + return fmt.Errorf("azure appconfig error: specify %s or %s field in metadata", host, connectionString) + } + + // In Dapr 1.11, these properties accepted nanoseconds as integers + // If users pass values larger than 10^6 (before, 1ms; now, 10^6 seconds), they probably set the metadata property for 1.11 in nanoseconds and that's not what they want here + // TODO: Remove this in Dapr 1.13 + if m.MaxRetryDelay > time.Millisecond*time.Second { //nolint:durationcheck + log.Warnf("[WARN] Property 'maxRetryDelay' is %v, which is probably incorrect. If you are upgrading from Dapr 1.11, please note that the property is now a Go duration rather than a number of nanoseconds", m.MaxRetryDelay) + } + if m.RetryDelay > time.Millisecond*time.Second { //nolint:durationcheck + log.Warnf("[WARN] Property 'retryDelay' is %v, which is probably incorrect. If you are upgrading from Dapr 1.11, please note that the property is now a Go duration rather than a number of nanoseconds", m.RetryDelay) + } + if m.SubscribePollInterval > time.Millisecond*time.Second { //nolint:durationcheck + log.Warnf("[WARN] Property 'subscribePollInterval' is %v, which is probably incorrect. If you are upgrading from Dapr 1.11, please note that the property is now a Go duration rather than a number of nanoseconds", m.SubscribePollInterval) + } + if m.RequestTimeout > time.Millisecond*time.Second { //nolint:durationcheck + log.Warnf("[WARN] Property 'requestTimeout' is %v, which is probably incorrect. If you are upgrading from Dapr 1.11, please note that the property is now a Go duration rather than a number of nanoseconds", m.RequestTimeout) + } + + return nil } diff --git a/configuration/azure/appconfig/metadata.yaml b/configuration/azure/appconfig/metadata.yaml index 040b72fb5d..c379fdf8b8 100644 --- a/configuration/azure/appconfig/metadata.yaml +++ b/configuration/azure/appconfig/metadata.yaml @@ -35,22 +35,22 @@ metadata: default: '3' example: '10' - name: retryDelay - description: "Specifies the initial amount of delay to use before retrying an operation, in nanoseconds. The delay increases exponentially with each retry up to the maximum specified by MaxRetryDelay. Defaults to 4 seconds. -1 disables delay between retries." - type: number - default: '4000000000' - example: '5000000000' + description: "Specifies the initial amount of delay to use before retrying an operation. The delay increases exponentially with each retry up to the maximum specified by MaxRetryDelay. Defaults to 4 seconds." + type: duration + default: '4s' + example: '5s' - name: maxRetryDelay - description: "Specifies the maximum delay allowed before retrying an operation, in nanoseconds. Typically the value is greater than or equal to the value specified in RetryDelay. Defaults to 120 seconds. -1 disables the limit." - type: number - default: '120000000000' - example: '180000000000' + description: "Specifies the maximum delay allowed before retrying an operation. Typically the value is greater than or equal to the value specified in RetryDelay. Defaults to 2 minutes." + type: duration + default: '2m' + example: '3m' - name: subscribePollInterval - description: "Specifies the poll interval for polling the subscribed keys for any changes, in nanoseconds. Default polling interval is set to 24 hours." - type: number - default: '86400000000000' - example: '240000000000' + description: "Specifies the poll interval for polling the subscribed keys for any changes. Default polling interval is set to 24 hours." + type: duration + default: '24h' + example: '5m' - name: requesttimeout - description: "Specifies the time allowed to pass until a request is failed, in nanoseconds. Default timeout is set to 15 seconds." - type: number - default: '15000000000' - example: '30000000000' \ No newline at end of file + description: "Specifies the time allowed to pass until a request is failed. Default timeout is set to 15 seconds." + type: duration + default: '15s' + example: '30s' \ No newline at end of file From 5278890290b765075466ec1a4bcbf1385eb403aa Mon Sep 17 00:00:00 2001 From: Filinto Duran Date: Mon, 31 Jul 2023 23:04:37 -0500 Subject: [PATCH 02/22] Add rethinkdb closer (#3019) Signed-off-by: Filinto Duran --- state/rethinkdb/rethinkdb.go | 7 +++++++ state/rethinkdb/rethinkdb_test.go | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/state/rethinkdb/rethinkdb.go b/state/rethinkdb/rethinkdb.go index b725ea17bc..481ef740e7 100644 --- a/state/rethinkdb/rethinkdb.go +++ b/state/rethinkdb/rethinkdb.go @@ -311,3 +311,10 @@ func (s *RethinkDB) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) return } + +func (s *RethinkDB) Close() error { + if s.session == nil { + return nil + } + return s.session.Close() +} diff --git a/state/rethinkdb/rethinkdb_test.go b/state/rethinkdb/rethinkdb_test.go index 0d24b7e014..a4b60f12c7 100644 --- a/state/rethinkdb/rethinkdb_test.go +++ b/state/rethinkdb/rethinkdb_test.go @@ -17,6 +17,7 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "testing" "time" @@ -163,6 +164,9 @@ func TestRethinkDBStateStoreRongRun(t *testing.T) { if err := db.Init(context.Background(), m); err != nil { t.Fatalf("error initializing db: %v", err) } + closer, ok := db.(io.Closer) + assert.True(t, ok) + defer assert.NoError(t, closer.Close()) for i := 0; i < 1000; i++ { testBulk(t, db, i) From c288c519b9e123f0a8c985f581f45974ece4f103 Mon Sep 17 00:00:00 2001 From: Filinto Duran Date: Mon, 31 Jul 2023 23:13:46 -0500 Subject: [PATCH 03/22] Add closer to jetstream state (#3021) Signed-off-by: Filinto Duran --- state/jetstream/jetstream.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/state/jetstream/jetstream.go b/state/jetstream/jetstream.go index fedb1e2f68..6c721757f6 100644 --- a/state/jetstream/jetstream.go +++ b/state/jetstream/jetstream.go @@ -16,8 +16,10 @@ package jetstream import ( "context" "errors" + "io" "reflect" "strings" + "sync/atomic" jsoniter "github.com/json-iterator/go" "github.com/nats-io/nats.go" @@ -37,6 +39,7 @@ type StateStore struct { json jsoniter.API bucket nats.KeyValue logger logger.Logger + closed atomic.Bool } type jetstreamMetadata struct { @@ -175,3 +178,12 @@ func (js *StateStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) return } + +func (js *StateStore) Close() error { + if js.closed.CompareAndSwap(false, true) && js.nc != nil { + js.nc.Close() + } + return nil +} + +var _ io.Closer = (*StateStore)(nil) From 69df184f51e666e21bf37d98026bb7c4e993f3a5 Mon Sep 17 00:00:00 2001 From: Roberto Rojas Date: Tue, 1 Aug 2023 02:31:01 -0400 Subject: [PATCH 04/22] [AWS PubSub SNSSQS] Adds Component Metadata Schema (#2905) Signed-off-by: Roberto J Rojas Signed-off-by: Roberto Rojas Co-authored-by: Artur Souza Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> --- .../builtin-authentication-profiles.yaml | 8 + pubsub/aws/snssqs/metadata.go | 14 +- pubsub/aws/snssqs/metadata.yaml | 146 ++++++++++++++++++ 3 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 pubsub/aws/snssqs/metadata.yaml diff --git a/.build-tools/builtin-authentication-profiles.yaml b/.build-tools/builtin-authentication-profiles.yaml index 81e07c3635..548fbf5352 100644 --- a/.build-tools/builtin-authentication-profiles.yaml +++ b/.build-tools/builtin-authentication-profiles.yaml @@ -13,6 +13,14 @@ aws: required: true sensitive: true example: '"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"' + - name: sessionToken + required: false + sensitive: true + description: | + AWS session token to use. A session token is only required if you are using + temporary security credentials. + example: '"TOKEN"' + type: string - title: "AWS: Credentials from Environment Variables" description: Use AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY from the environment diff --git a/pubsub/aws/snssqs/metadata.go b/pubsub/aws/snssqs/metadata.go index 3b6796b783..2f3016ccec 100644 --- a/pubsub/aws/snssqs/metadata.go +++ b/pubsub/aws/snssqs/metadata.go @@ -11,20 +11,22 @@ import ( ) type snsSqsMetadata struct { - // aws endpoint for the component to use. - Endpoint string `mapstructure:"endpoint"` + // Ignored by metadata parser because included in built-in authentication profile // access key to use for accessing sqs/sns. - AccessKey string `mapstructure:"accessKey"` + AccessKey string `json:"accessKey" mapstructure:"accessKey" mdignore:"true"` // secret key to use for accessing sqs/sns. - SecretKey string `mapstructure:"secretKey"` + SecretKey string `json:"secretKey" mapstructure:"secretKey" mdignore:"true"` // aws session token to use. - SessionToken string `mapstructure:"sessionToken"` + SessionToken string `mapstructure:"sessionToken" mdignore:"true"` + + // aws endpoint for the component to use. + Endpoint string `mapstructure:"endpoint"` // aws region in which SNS/SQS should create resources. Region string `mapstructure:"region"` // aws partition in which SNS/SQS should create resources. internalPartition string `mapstructure:"-"` // name of the queue for this application. The is provided by the runtime as "consumerID". - SqsQueueName string `mapstructure:"consumerID"` + SqsQueueName string `mapstructure:"consumerID" mdignore:"true"` // name of the dead letter queue for this application. SqsDeadLettersQueueName string `mapstructure:"sqsDeadLettersQueueName"` // flag to SNS and SQS FIFO. diff --git a/pubsub/aws/snssqs/metadata.yaml b/pubsub/aws/snssqs/metadata.yaml new file mode 100644 index 0000000000..ffdf2397a1 --- /dev/null +++ b/pubsub/aws/snssqs/metadata.yaml @@ -0,0 +1,146 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: pubsub +name: aws.snssqs +version: v1 +status: stable +title: "AWS SNS/SQS" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-pubsub/setup-aws-snssqs/ +capabilities: + - ttl +builtinAuthenticationProfiles: + - name: "aws" +metadata: + - name: region + required: true + description: | + The AWS region where the SNS/SQS assets are located or be created in. See the `Supported AWS services per region` page. + Ensure that SNS and SQS are available in that region. + url: + title: "Supported AWS services per region" + url: "https://aws.amazon.com/about-aws/global-infrastructure/regional-product-services/?p=ugi&l=na" + example: '"us-east-1"' + type: string + - name: endpoint + required: false + description: | + AWS endpoint for the component to use, to connect to emulators. + Do not use this when running against production AWS. + example: '"http://localhost:4566"' + type: string + - name: messageVisibilityTimeout + required: false + description: | + Amount of time in seconds that a message is hidden from receive requests after + it is sent to a subscriber. + type: number + default: '10' + example: '10' + - name: messageReceiveLimit + required: false + description: | + Maximun number of attempts the message will be re-delivered after processing failures. + The sqsDeadLettersQueueName is a SQS dead-letters queue to move the message to + once the maximun number of attempts have been reached. + type: number + default: '10' + example: '10' + - name: messageRetryLimit + required: false + description: | + Number of times to resend a message after processing of that message fails + before removing that message from the queue. + type: number + default: '10' + example: '10' + - name: sqsDeadLettersQueueName + required: false + description: | + Name of the dead letters queue for this application. + example: '"myapp-dlq"' + type: string + - name: messageWaitTimeSeconds + required: false + description: | + The duration (in seconds) for which the call waits for a message to arrive + in the queue before returning. If a message is available, the call returns + sooner than messageWaitTimeSeconds. If no messages are available and the + wait time expires, the call returns successfully with an empty list of messages. + type: number + default: '1' + example: '1' + - name: messageMaxNumber + required: false + description: | + Maximum number of messages to receive from the queue at a time. + type: number + default: '10' + example: '10' + - name: fifo + description: | + Use SQS FIFO queue to provide message ordering and deduplication. + See `Amazon SQS FIFO (First-In-First-Out) queues` further details. + url: + title: "Amazon SQS FIFO (First-In-First-Out) queues" + url: "https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html" + type: bool + default: 'false' + example: '"true", "false"' + - name: fifoMessageGroupID + required: false + description: | + If fifo is enabled, instructs Dapr to use a custom Message Group ID + for the pubsub deployment. This is not mandatory as Dapr creates a + custom Message Group ID for each producer, thus ensuring ordering + of messages per a Dapr producer. + See Message Group ID Property documentation. + url: + title: "Message Group ID Property documentation" + url: "https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html" + example: '"app1-mgi"' + type: string + - name: disableEntityManagement + description: | + When set to true, SNS topics, SQS queues and the SQS subscriptions to + SNS do not get created automatically. + type: bool + default: 'false' + example: '"true", "false"' + - name: disableDeleteOnRetryLimit + description: | + When set to true, after retrying and failing of messageRetryLimit + times processing a message, reset the message visibility timeout + so that other consumers can try processing, instead of deleting + the message from SQS (the default behvior). + type: bool + default: 'false' + example: '"true", "false"' + - name: assetsManagementTimeoutSeconds + required: false + description: | + Amount of time in seconds, for an AWS asset management operation, + before it times out and cancelled. Asset management operations + are any operations performed on STS, SNS and SQS, except message + publish and consume operations that implement the default Dapr + component retry behavior. The value can be set to any non-negative + float/integer. + type: number + default: '1' + example: '0.5, 10' + - name: concurrencyMode + required: false + description: | + When messages are received in bulk from SQS, call the subscriber + sequentially (“single” message at a time), or + concurrently (in “parallel”). + default: '"parallel"' + example: '"single", "parallel"' + type: string + - name: accountId + required: false + description: | + The AWS account ID. Resolved automatically if not provided. + example: '""' + type: string \ No newline at end of file From 5f695a41c8d5a978a94aa8e2057e4dc1ad6e5558 Mon Sep 17 00:00:00 2001 From: Roberto Rojas Date: Tue, 1 Aug 2023 10:31:51 -0400 Subject: [PATCH 05/22] [AWS Bindings S3] Adds Component Metadata Schema (#2904) Signed-off-by: Roberto J Rojas Signed-off-by: Roberto Rojas Signed-off-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Artur Souza Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> --- bindings/aws/s3/metadata.yaml | 78 +++++++++++++++++++++++++++++++++++ bindings/aws/s3/s3.go | 10 ++--- 2 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 bindings/aws/s3/metadata.yaml diff --git a/bindings/aws/s3/metadata.yaml b/bindings/aws/s3/metadata.yaml new file mode 100644 index 0000000000..dd573c5466 --- /dev/null +++ b/bindings/aws/s3/metadata.yaml @@ -0,0 +1,78 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: bindings +name: aws.s3 +version: v1 +status: stable +title: "AWS S3" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-bindings/s3/ +binding: + output: true + operations: + - name: create + description: "Create blob" + - name: get + description: "Get blob" + - name: delete + description: "Delete blob" + - name: list + description: "List blob" +capabilities: [] +builtinAuthenticationProfiles: + - name: "aws" +metadata: + - name: bucket + required: true + description: | + The name of the S3 bucket to write to. + example: '"bucket"' + type: string + - name: region + required: true + description: | + The specific AWS region where the S3 bucket is located. + example: '"us-east-1"' + type: string + - name: endpoint + required: false + description: | + AWS endpoint for the component to use, to connect to S3-compatible services or emulators. + Do not use this when running against production AWS. + example: '"http://localhost:4566"' + type: string + - name: forcePathStyle + description: | + Currently Amazon S3 SDK supports virtual-hosted-style and path-style access. + When false (the default), uses virtual-hosted-style format, i.e.: `https://./`. + When true, uses path-style format, i.e.: `https:////`. + type: bool + default: 'false' + example: '"true", "false"' + - name: decodeBase64 + description: | + Configuration to decode base64 file content before saving to bucket storage. + (In case of saving a file with binary content). + type: bool + default: 'false' + example: '"true", "false"' + - name: encodeBase64 + description: | + Configuration to encode base64 file content before returning the content. + (In case of opening a file with binary content). + type: bool + default: 'false' + example: '"true", "false"' + - name: disableSSL + description: | + Allows to connect to non-`https://` endpoints. + type: bool + default: 'false' + example: '"true", "false"' + - name: insecureSSL + description: | + When connecting to `https://` endpoints, accepts self-signed or invalid certificates. + type: bool + default: 'false' + example: '"true", "false"' \ No newline at end of file diff --git a/bindings/aws/s3/s3.go b/bindings/aws/s3/s3.go index ff84145e74..19fb60af70 100644 --- a/bindings/aws/s3/s3.go +++ b/bindings/aws/s3/s3.go @@ -63,20 +63,20 @@ type AWSS3 struct { type s3Metadata struct { // Ignored by metadata parser because included in built-in authentication profile - AccessKey string `json:"accessKey" mapstructure:"accessKey" mdignore:"true"` - SecretKey string `json:"secretKey" mapstructure:"secretKey" mdignore:"true"` + AccessKey string `json:"accessKey" mapstructure:"accessKey" mdignore:"true"` + SecretKey string `json:"secretKey" mapstructure:"secretKey" mdignore:"true"` + SessionToken string `json:"sessionToken" mapstructure:"sessionToken" mdignore:"true"` Region string `json:"region" mapstructure:"region"` Endpoint string `json:"endpoint" mapstructure:"endpoint"` - SessionToken string `json:"sessionToken" mapstructure:"sessionToken"` Bucket string `json:"bucket" mapstructure:"bucket"` DecodeBase64 bool `json:"decodeBase64,string" mapstructure:"decodeBase64"` EncodeBase64 bool `json:"encodeBase64,string" mapstructure:"encodeBase64"` ForcePathStyle bool `json:"forcePathStyle,string" mapstructure:"forcePathStyle"` DisableSSL bool `json:"disableSSL,string" mapstructure:"disableSSL"` InsecureSSL bool `json:"insecureSSL,string" mapstructure:"insecureSSL"` - FilePath string `mapstructure:"filePath"` - PresignTTL string `mapstructure:"presignTTL"` + FilePath string `json:"filePath" mapstructure:"filePath" mdignore:"true"` + PresignTTL string `json:"presignTTL" mapstructure:"presignTTL" mdignore:"true"` } type createResponse struct { From 5d009c06189942f15538ef63c984cf4ee80d442e Mon Sep 17 00:00:00 2001 From: Shivam Kumar Singh Date: Tue, 1 Aug 2023 23:30:30 +0530 Subject: [PATCH 06/22] #3018: Add `get-embedding` operation to Azure OpenAI (#3022) Signed-off-by: Shivam Singh --- bindings/azure/openai/metadata.yaml | 6 +-- bindings/azure/openai/openai.go | 68 ++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/bindings/azure/openai/metadata.yaml b/bindings/azure/openai/metadata.yaml index f579916940..7822f5ec05 100644 --- a/bindings/azure/openai/metadata.yaml +++ b/bindings/azure/openai/metadata.yaml @@ -31,8 +31,4 @@ metadata: - name: endpoint required: true description: "Endpoint of the Azure OpenAI service" - example: '"https://myopenai.openai.azure.com"' - - name: deploymentID - required: true - description: "ID of the model deployment in the Azure OpenAI service" - example: '"my-model"' + example: '"https://myopenai.openai.azure.com"' \ No newline at end of file diff --git a/bindings/azure/openai/openai.go b/bindings/azure/openai/openai.go index 81743b5c81..263277fbbc 100644 --- a/bindings/azure/openai/openai.go +++ b/bindings/azure/openai/openai.go @@ -33,6 +33,7 @@ import ( const ( CompletionOperation bindings.OperationKind = "completion" ChatCompletionOperation bindings.OperationKind = "chat-completion" + GetEmbeddingOperation bindings.OperationKind = "get-embedding" APIKey = "apiKey" DeploymentID = "deploymentID" @@ -50,22 +51,20 @@ const ( // AzOpenAI represents OpenAI output binding. type AzOpenAI struct { - logger logger.Logger - client *azopenai.Client - deploymentID string + logger logger.Logger + client *azopenai.Client } type openAIMetadata struct { // APIKey is the API key for the Azure OpenAI API. APIKey string `mapstructure:"apiKey"` - // DeploymentID is the deployment ID for the Azure OpenAI API. - DeploymentID string `mapstructure:"deploymentID"` // Endpoint is the endpoint for the Azure OpenAI API. Endpoint string `mapstructure:"endpoint"` } // ChatMessages type for chat completion API. type ChatMessages struct { + DeploymentID string `json:"deploymentID"` Messages []Message `json:"messages"` Temperature float32 `json:"temperature"` MaxTokens int32 `json:"maxTokens"` @@ -84,6 +83,7 @@ type Message struct { // Prompt type for completion API. type Prompt struct { + DeploymentID string `json:"deploymentID"` Prompt string `json:"prompt"` Temperature float32 `json:"temperature"` MaxTokens int32 `json:"maxTokens"` @@ -94,6 +94,11 @@ type Prompt struct { Stop []string `json:"stop"` } +type EmbeddingMessage struct { + DeploymentID string `json:"deploymentID"` + Message string `json:"message"` +} + // NewOpenAI returns a new OpenAI output binding. func NewOpenAI(logger logger.Logger) bindings.OutputBinding { return &AzOpenAI{ @@ -111,9 +116,6 @@ func (p *AzOpenAI) Init(ctx context.Context, meta bindings.Metadata) error { if m.Endpoint == "" { return fmt.Errorf("required metadata not set: %s", Endpoint) } - if m.DeploymentID == "" { - return fmt.Errorf("required metadata not set: %s", DeploymentID) - } if m.APIKey != "" { // use API key authentication @@ -144,7 +146,6 @@ func (p *AzOpenAI) Init(ctx context.Context, meta bindings.Metadata) error { return fmt.Errorf("error creating Azure OpenAI client: %w", err) } } - p.deploymentID = m.DeploymentID return nil } @@ -154,6 +155,7 @@ func (p *AzOpenAI) Operations() []bindings.OperationKind { return []bindings.OperationKind{ ChatCompletionOperation, CompletionOperation, + GetEmbeddingOperation, } } @@ -188,6 +190,14 @@ func (p *AzOpenAI) Invoke(ctx context.Context, req *bindings.InvokeRequest) (res responseAsBytes, _ := json.Marshal(response) resp.Data = responseAsBytes + case GetEmbeddingOperation: + response, err := p.getEmbedding(ctx, req.Data, req.Metadata) + if err != nil { + return nil, fmt.Errorf("error performing get embedding operation: %w", err) + } + responseAsBytes, _ := json.Marshal(response) + resp.Data = responseAsBytes + default: return nil, fmt.Errorf( "invalid operation type: %s. Expected %s, %s", @@ -220,12 +230,16 @@ func (p *AzOpenAI) completion(ctx context.Context, message []byte, metadata map[ return nil, fmt.Errorf("prompt is required for completion operation") } + if prompt.DeploymentID == "" { + return nil, fmt.Errorf("required metadata not set: %s", DeploymentID) + } + if len(prompt.Stop) == 0 { prompt.Stop = nil } resp, err := p.client.GetCompletions(ctx, azopenai.CompletionsOptions{ - DeploymentID: p.deploymentID, + DeploymentID: prompt.DeploymentID, Prompt: []string{prompt.Prompt}, MaxTokens: &prompt.MaxTokens, Temperature: &prompt.Temperature, @@ -268,6 +282,10 @@ func (p *AzOpenAI) chatCompletion(ctx context.Context, messageRequest []byte, me return nil, fmt.Errorf("messages are required for chat-completion operation") } + if messages.DeploymentID == "" { + return nil, fmt.Errorf("required metadata not set: %s", DeploymentID) + } + if len(messages.Stop) == 0 { messages.Stop = nil } @@ -286,7 +304,7 @@ func (p *AzOpenAI) chatCompletion(ctx context.Context, messageRequest []byte, me } res, err := p.client.GetChatCompletions(ctx, azopenai.ChatCompletionsOptions{ - DeploymentID: p.deploymentID, + DeploymentID: messages.DeploymentID, MaxTokens: maxTokens, Temperature: &messages.Temperature, TopP: &messages.TopP, @@ -312,6 +330,34 @@ func (p *AzOpenAI) chatCompletion(ctx context.Context, messageRequest []byte, me return response, nil } +func (p *AzOpenAI) getEmbedding(ctx context.Context, messageRequest []byte, metadata map[string]string) (response []float32, err error) { + message := EmbeddingMessage{} + err = json.Unmarshal(messageRequest, &message) + if err != nil { + return nil, fmt.Errorf("error unmarshalling the input object: %w", err) + } + + if message.DeploymentID == "" { + return nil, fmt.Errorf("required metadata not set: %s", DeploymentID) + } + + res, err := p.client.GetEmbeddings(ctx, azopenai.EmbeddingsOptions{ + DeploymentID: message.DeploymentID, + Input: []string{message.Message}, + }, nil) + if err != nil { + return nil, fmt.Errorf("error getting embedding api: %w", err) + } + + // No embedding returned. + if len(res.Data) == 0 { + return []float32{}, nil + } + + response = res.Data[0].Embedding + return response, nil +} + // Close Az OpenAI instance. func (p *AzOpenAI) Close() error { p.client = nil From af5bacebf4877e0c10ba860737ce8a7ccb0ee02c Mon Sep 17 00:00:00 2001 From: Josh van Leeuwen Date: Tue, 1 Aug 2023 19:38:33 +0100 Subject: [PATCH 07/22] Adds OIDC authentication support for pubsub Apache Pulsar (#3023) Signed-off-by: joshvanl Co-authored-by: Yaron Schneider --- .github/workflows/certification.yml | 2 +- .../authentication/oidc/clientcredentials.go | 213 +++++++++++ .../oidc/clientcredentials_test.go | 266 ++++++++++++++ pubsub/pulsar/metadata.go | 10 +- pubsub/pulsar/pulsar.go | 59 +++- .../{ => auth-none}/consumer_five/pulsar.yaml | 0 .../{ => auth-none}/consumer_four/pulsar.yaml | 0 .../{ => auth-none}/consumer_one/pulsar.yml | 0 .../{ => auth-none}/consumer_six/pulsar.yaml | 0 .../{ => auth-none}/consumer_three/pulsar.yml | 0 .../{ => auth-none}/consumer_two/pulsar.yml | 0 .../auth-oidc/consumer_five/pulsar.yml.tmpl | 34 ++ .../auth-oidc/consumer_four/pulsar.yml.tmpl | 30 ++ .../auth-oidc/consumer_one/pulsar.yml.tmpl | 28 ++ .../auth-oidc/consumer_six/pulsar.yaml.tmpl | 34 ++ .../auth-oidc/consumer_three/pulsar.yml.tmpl | 28 ++ .../auth-oidc/consumer_two/pulsar.yml.tmpl | 28 ++ .../pubsub/pulsar/config/.gitignore | 1 + .../docker-compose_auth-mock-oidc-server.yaml | 14 + .../docker-compose_auth-none.yaml} | 2 +- .../config/docker-compose_auth-oidc.yaml.tmpl | 96 +++++ .../pulsar/config/pulsar_auth-oidc.conf | 39 ++ .../pubsub/pulsar/pulsar_test.go | 334 ++++++++++++++---- 23 files changed, 1132 insertions(+), 86 deletions(-) create mode 100644 internal/authentication/oidc/clientcredentials.go create mode 100644 internal/authentication/oidc/clientcredentials_test.go rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_five/pulsar.yaml (100%) rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_four/pulsar.yaml (100%) rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_one/pulsar.yml (100%) rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_six/pulsar.yaml (100%) rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_three/pulsar.yml (100%) rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_two/pulsar.yml (100%) create mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_five/pulsar.yml.tmpl create mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_four/pulsar.yml.tmpl create mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_one/pulsar.yml.tmpl create mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_six/pulsar.yaml.tmpl create mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_three/pulsar.yml.tmpl create mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_two/pulsar.yml.tmpl create mode 100644 tests/certification/pubsub/pulsar/config/.gitignore create mode 100644 tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oidc-server.yaml rename tests/certification/pubsub/pulsar/{docker-compose.yml => config/docker-compose_auth-none.yaml} (97%) create mode 100644 tests/certification/pubsub/pulsar/config/docker-compose_auth-oidc.yaml.tmpl create mode 100644 tests/certification/pubsub/pulsar/config/pulsar_auth-oidc.conf diff --git a/.github/workflows/certification.yml b/.github/workflows/certification.yml index dfa2fe39ba..5482b986a1 100644 --- a/.github/workflows/certification.yml +++ b/.github/workflows/certification.yml @@ -258,7 +258,7 @@ jobs: set +e gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json \ --junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.xml --format standard-quiet -- \ - -coverprofile=cover.out -covermode=set -tags=certtests -coverpkg=${{ matrix.source-pkg }} + -coverprofile=cover.out -covermode=set -tags=certtests -timeout=30m -coverpkg=${{ matrix.source-pkg }} status=$? echo "Completed certification tests for ${{ matrix.component }} ... " if test $status -ne 0; then diff --git a/internal/authentication/oidc/clientcredentials.go b/internal/authentication/oidc/clientcredentials.go new file mode 100644 index 0000000000..902f78b561 --- /dev/null +++ b/internal/authentication/oidc/clientcredentials.go @@ -0,0 +1,213 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidc + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net/http" + "net/url" + "sync" + "sync/atomic" + "time" + + "golang.org/x/oauth2" + ccreds "golang.org/x/oauth2/clientcredentials" + "k8s.io/utils/clock" + + "github.com/dapr/kit/logger" +) + +const ( + oidcScopeOpenID = "openid" +) + +type ClientCredentialsOptions struct { + Logger logger.Logger + TokenURL string + ClientID string + ClientSecret string + Scopes []string + Audiences []string + CAPEM []byte +} + +// ClientCredentials is an OAuth2 Token Source that uses the client_credentials +// grant type to fetch a token. +type ClientCredentials struct { + log logger.Logger + currentToken *oauth2.Token + httpClient *http.Client + fetchTokenFn func(context.Context) (*oauth2.Token, error) + + lock sync.RWMutex + wg sync.WaitGroup + closeCh chan struct{} + closed atomic.Bool + clock clock.Clock +} + +func NewClientCredentials(ctx context.Context, opts ClientCredentialsOptions) (*ClientCredentials, error) { + conf, httpClient, err := toConfig(opts) + if err != nil { + return nil, err + } + + token, err := conf.Token(context.WithValue(ctx, oauth2.HTTPClient, httpClient)) + if err != nil { + return nil, fmt.Errorf("error fetching initial oidc client_credentials token: %w", err) + } + + opts.Logger.Info("Fetched initial oidc client_credentials token") + + return &ClientCredentials{ + log: opts.Logger, + currentToken: token, + httpClient: httpClient, + closeCh: make(chan struct{}), + clock: clock.RealClock{}, + fetchTokenFn: conf.Token, + }, nil +} + +func (c *ClientCredentials) Run(ctx context.Context) { + c.log.Info("Running oidc client_credentials token renewer") + renewDuration := c.tokenRenewDuration() + + c.wg.Add(1) + go func() { + defer func() { + c.log.Info("Stopped oidc client_credentials token renewer") + c.wg.Done() + }() + + for { + select { + case <-c.closeCh: + return + case <-ctx.Done(): + return + case <-c.clock.After(renewDuration): + } + + c.log.Debug("Renewing client credentials token") + + token, err := c.fetchTokenFn(context.WithValue(ctx, oauth2.HTTPClient, c.httpClient)) + if err != nil { + c.log.Errorf("Error fetching renewed oidc client_credentials token, retrying in 30 seconds: %s", err) + renewDuration = time.Second * 30 + continue + } + + c.lock.Lock() + c.currentToken = token + c.lock.Unlock() + renewDuration = c.tokenRenewDuration() + } + }() +} + +func toConfig(opts ClientCredentialsOptions) (*ccreds.Config, *http.Client, error) { + scopes := opts.Scopes + if len(scopes) == 0 { + // If no scopes are provided, then the default is to use the 'openid' scope + // since that is always required for OIDC so implicitly add it. + scopes = []string{oidcScopeOpenID} + } + + var oidcScopeFound bool + for _, scope := range scopes { + if scope == oidcScopeOpenID { + oidcScopeFound = true + break + } + } + if !oidcScopeFound { + return nil, nil, fmt.Errorf("oidc client_credentials token source requires the %q scope", oidcScopeOpenID) + } + + tokenURL, err := url.Parse(opts.TokenURL) + if err != nil { + return nil, nil, fmt.Errorf("error parsing token URL: %w", err) + } + if tokenURL.Scheme != "https" { + return nil, nil, fmt.Errorf("OIDC token provider URL requires 'https' scheme: %q", tokenURL) + } + + conf := &ccreds.Config{ + ClientID: opts.ClientID, + ClientSecret: opts.ClientSecret, + TokenURL: opts.TokenURL, + Scopes: scopes, + } + + if len(opts.Audiences) == 0 { + return nil, nil, errors.New("oidc client_credentials token source requires at least one audience") + } + + conf.EndpointParams = url.Values{"audience": opts.Audiences} + + // If caPool is nil, then the Go TLS library will use the system's root CA. + var caPool *x509.CertPool + if len(opts.CAPEM) > 0 { + caPool = x509.NewCertPool() + if !caPool.AppendCertsFromPEM(opts.CAPEM) { + return nil, nil, errors.New("failed to parse CA PEM") + } + } + + return conf, &http.Client{ + Timeout: time.Second * 30, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: caPool, + }, + }, + }, nil +} + +func (c *ClientCredentials) Close() { + defer c.wg.Wait() + + if c.closed.CompareAndSwap(false, true) { + close(c.closeCh) + } +} + +func (c *ClientCredentials) Token() (string, error) { + c.lock.RLock() + defer c.lock.RUnlock() + + if c.closed.Load() { + return "", errors.New("client_credentials token source is closed") + } + + if !c.currentToken.Valid() { + return "", errors.New("client_credentials token source is invalid") + } + + return c.currentToken.AccessToken, nil +} + +// tokenRenewTime returns the duration when the token should be renewed, which is +// half of the token's lifetime. +func (c *ClientCredentials) tokenRenewDuration() time.Duration { + c.lock.RLock() + defer c.lock.RUnlock() + return c.currentToken.Expiry.Sub(c.clock.Now()) / 2 +} diff --git a/internal/authentication/oidc/clientcredentials_test.go b/internal/authentication/oidc/clientcredentials_test.go new file mode 100644 index 0000000000..8a090037ac --- /dev/null +++ b/internal/authentication/oidc/clientcredentials_test.go @@ -0,0 +1,266 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidc + +import ( + "context" + "errors" + "net/url" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "golang.org/x/oauth2" + ccreds "golang.org/x/oauth2/clientcredentials" + "k8s.io/utils/clock" + clocktesting "k8s.io/utils/clock/testing" + + "github.com/dapr/kit/logger" +) + +func TestRun(t *testing.T) { + var lock sync.Mutex + clock := clocktesting.NewFakeClock(time.Now()) + called := make(chan struct{}, 1) + var retErr error = nil + + fetchTokenFn := func(context.Context) (*oauth2.Token, error) { + lock.Lock() + defer lock.Unlock() + called <- struct{}{} + return &oauth2.Token{ + Expiry: clock.Now().Add(time.Minute), + }, retErr + } + + t.Run("should return when context is cancelled", func(t *testing.T) { + c := &ClientCredentials{ + log: logger.NewLogger("test"), + clock: clock, + fetchTokenFn: fetchTokenFn, + closeCh: make(chan struct{}), + currentToken: &oauth2.Token{ + Expiry: clock.Now(), + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + + c.Run(ctx) + cancel() + + select { + case <-called: + t.Fatal("should not have called fetchTokenFn") + default: + } + }) + + t.Run("should return when closed", func(t *testing.T) { + c := &ClientCredentials{ + log: logger.NewLogger("test"), + clock: clock, + fetchTokenFn: fetchTokenFn, + closeCh: make(chan struct{}), + currentToken: &oauth2.Token{ + Expiry: clock.Now(), + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c.Run(ctx) + + // Should be able to close multiple times. + c.Close() + c.Close() + + select { + case <-called: + t.Fatal("should not have called fetchTokenFn") + case <-c.closeCh: + case <-time.After(time.Second * 5): + t.Fatal("should have closed run") + } + }) + + t.Run("should renew token when ready for renewal", func(t *testing.T) { + c := &ClientCredentials{ + log: logger.NewLogger("test"), + clock: clock, + fetchTokenFn: fetchTokenFn, + closeCh: make(chan struct{}), + currentToken: &oauth2.Token{Expiry: clock.Now().Add(time.Minute * 2)}, + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + c.Run(ctx) + + assert.Eventually(t, clock.HasWaiters, time.Second*5, time.Millisecond*10) + clock.Step(time.Minute * 1) + + select { + case <-called: + case <-time.After(time.Second * 5): + t.Fatal("should have called") + } + }) + + t.Run("should call renew again after 30 seconds when it fails", func(t *testing.T) { + c := &ClientCredentials{ + log: logger.NewLogger("test"), + clock: clock, + fetchTokenFn: fetchTokenFn, + closeCh: make(chan struct{}), + currentToken: &oauth2.Token{ + Expiry: clock.Now(), + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c.Run(ctx) + + assert.Eventually(t, clock.HasWaiters, time.Second*5, time.Millisecond*10) + clock.Step(time.Minute * 1) + + select { + case <-called: + case <-time.After(time.Second * 5): + t.Fatal("should have called") + } + + lock.Lock() + retErr = errors.New("test error") + lock.Unlock() + + assert.Eventually(t, clock.HasWaiters, time.Second*5, time.Millisecond*10) + clock.Step(time.Minute * 1) + + select { + case <-called: + case <-time.After(time.Second * 5): + t.Fatal("should have called") + } + + assert.Eventually(t, clock.HasWaiters, time.Second*5, time.Millisecond*10) + clock.Step(time.Second * 30) + + select { + case <-called: + case <-time.After(time.Second * 5): + t.Fatal("should have called") + } + + c.Close() + + select { + case <-c.closeCh: + case <-time.After(time.Second * 5): + t.Fatal("should have closed run") + } + }) +} + +func Test_tokenRenewDuration(t *testing.T) { + c := &ClientCredentials{ + clock: clock.RealClock{}, + currentToken: &oauth2.Token{ + Expiry: time.Now(), + }, + } + assert.InDelta(t, c.tokenRenewDuration(), time.Duration(0), float64(time.Second*5)) + + c = &ClientCredentials{ + clock: clock.RealClock{}, + currentToken: &oauth2.Token{ + Expiry: time.Now().Add(time.Hour), + }, + } + assert.InDelta(t, c.tokenRenewDuration(), time.Minute*30, float64(time.Second*5)) +} + +func Test_toConfig(t *testing.T) { + tests := map[string]struct { + opts ClientCredentialsOptions + expConfig *ccreds.Config + expErr bool + }{ + "openid not in scopes should error": { + opts: ClientCredentialsOptions{ + TokenURL: "https://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Scopes: []string{"profile"}, + Audiences: []string{"audience"}, + }, + expErr: true, + }, + "non-https endpoint should error": { + opts: ClientCredentialsOptions{ + TokenURL: "http://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Audiences: []string{"audience"}, + }, + expErr: true, + }, + "bad CA certificate should error": { + opts: ClientCredentialsOptions{ + TokenURL: "https://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Audiences: []string{"audience"}, + CAPEM: []byte("ca-pem"), + }, + expErr: true, + }, + "no audiences should error": { + opts: ClientCredentialsOptions{ + TokenURL: "https://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + }, + expErr: true, + }, + "should default scope": { + opts: ClientCredentialsOptions{ + TokenURL: "https://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Audiences: []string{"audience"}, + }, + expConfig: &ccreds.Config{ + ClientID: "client-id", + ClientSecret: "client-secret", + TokenURL: "https://localhost:8080", + Scopes: []string{"openid"}, + EndpointParams: url.Values{"audience": []string{"audience"}}, + }, + expErr: false, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + config, _, err := toConfig(test.opts) + assert.Equalf(t, test.expErr, err != nil, "%v", err) + assert.Equal(t, test.expConfig, config) + }) + } +} diff --git a/pubsub/pulsar/metadata.go b/pubsub/pulsar/metadata.go index 5bddedb6d6..aae2e20bb2 100644 --- a/pubsub/pulsar/metadata.go +++ b/pubsub/pulsar/metadata.go @@ -26,12 +26,20 @@ type pulsarMetadata struct { Tenant string `mapstructure:"tenant"` Namespace string `mapstructure:"namespace"` Persistent bool `mapstructure:"persistent"` - Token string `mapstructure:"token"` RedeliveryDelay time.Duration `mapstructure:"redeliveryDelay"` internalTopicSchemas map[string]schemaMetadata `mapstructure:"-"` PublicKey string `mapstructure:"publicKey"` PrivateKey string `mapstructure:"privateKey"` Keys string `mapstructure:"keys"` + + AuthType string `mapstructure:"authType"` + Token string `mapstructure:"token"` + OIDCTokenCAPEM string `mapstructure:"oidcTokenCAPEM"` + OIDCTokenURL string `mapstructure:"oidcTokenURL"` + OIDCClientID string `mapstructure:"oidcClientID"` + OIDCClientSecret string `mapstructure:"oidcClientSecret"` + OIDCAudiences []string `mapstructure:"oidcAudiences"` + OIDCScopes []string `mapstructure:"oidcScopes"` } type schemaMetadata struct { diff --git a/pubsub/pulsar/pulsar.go b/pubsub/pulsar/pulsar.go index 8846f2dc50..5df4216912 100644 --- a/pubsub/pulsar/pulsar.go +++ b/pubsub/pulsar/pulsar.go @@ -25,12 +25,12 @@ import ( "sync/atomic" "time" - "github.com/hamba/avro/v2" - "github.com/apache/pulsar-client-go/pulsar" "github.com/apache/pulsar-client-go/pulsar/crypto" + "github.com/hamba/avro/v2" lru "github.com/hashicorp/golang-lru/v2" + "github.com/dapr/components-contrib/internal/authentication/oidc" "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/pubsub" "github.com/dapr/kit/logger" @@ -55,6 +55,10 @@ const ( protoProtocol = "proto" partitionKey = "partitionKey" + authTypeNone = "none" + authTypeToken = "token" + authTypeOIDC = "oidc" + defaultTenant = "public" defaultNamespace = "default" cachedNumProducer = 10 @@ -94,13 +98,14 @@ const ( type ProcessMode string type Pulsar struct { - logger logger.Logger - client pulsar.Client - metadata pulsarMetadata - cache *lru.Cache[string, pulsar.Producer] - closed atomic.Bool - closeCh chan struct{} - wg sync.WaitGroup + logger logger.Logger + client pulsar.Client + metadata pulsarMetadata + oidcProvider *oidc.ClientCredentials + cache *lru.Cache[string, pulsar.Producer] + closed atomic.Bool + closeCh chan struct{} + wg sync.WaitGroup } func NewPulsar(l logger.Logger) pubsub.PubSub { @@ -157,7 +162,7 @@ func parsePulsarMetadata(meta pubsub.Metadata) (*pulsarMetadata, error) { return &m, nil } -func (p *Pulsar) Init(_ context.Context, metadata pubsub.Metadata) error { +func (p *Pulsar) Init(ctx context.Context, metadata pubsub.Metadata) error { m, err := parsePulsarMetadata(metadata) if err != nil { return err @@ -173,9 +178,37 @@ func (p *Pulsar) Init(_ context.Context, metadata pubsub.Metadata) error { ConnectionTimeout: 30 * time.Second, TLSAllowInsecureConnection: !m.EnableTLS, } - if m.Token != "" { + + switch m.AuthType { + case "": + // To ensure backward compatibility, if authType is not set but the token + // is we fallthrough to token auth. + if m.Token == "" { + break + } + fallthrough + case authTypeToken: options.Authentication = pulsar.NewAuthenticationToken(m.Token) + case authTypeOIDC: + var cc *oidc.ClientCredentials + cc, err = oidc.NewClientCredentials(ctx, oidc.ClientCredentialsOptions{ + Logger: p.logger, + TokenURL: m.OIDCTokenURL, + CAPEM: []byte(m.OIDCTokenCAPEM), + ClientID: m.OIDCClientID, + ClientSecret: m.OIDCClientSecret, + Scopes: m.OIDCScopes, + Audiences: m.OIDCAudiences, + }) + if err != nil { + return fmt.Errorf("could not instantiate oidc token provider: %v", err) + } + + options.Authentication = pulsar.NewAuthenticationTokenFromSupplier(cc.Token) + p.oidcProvider = cc + p.oidcProvider.Run(ctx) } + client, err := pulsar.NewClient(options) if err != nil { return fmt.Errorf("could not instantiate pulsar client: %v", err) @@ -490,6 +523,10 @@ func (p *Pulsar) Close() error { } p.client.Close() + if p.oidcProvider != nil { + p.oidcProvider.Close() + } + return nil } diff --git a/tests/certification/pubsub/pulsar/components/consumer_five/pulsar.yaml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_five/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_five/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_five/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/consumer_four/pulsar.yaml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_four/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_four/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_four/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/consumer_one/pulsar.yml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_one/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_one/pulsar.yml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_one/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/components/consumer_six/pulsar.yaml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_six/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_six/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_six/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/consumer_three/pulsar.yml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_three/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_three/pulsar.yml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_three/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/components/consumer_two/pulsar.yml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_two/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_two/pulsar.yml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_two/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_five/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_five/pulsar.yml.tmpl new file mode 100644 index 0000000000..1b9189d6ce --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_five/pulsar.yml.tmpl @@ -0,0 +1,34 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification5 + - name: redeliveryDelay + value: 200ms + - name: publicKey + value: public.key + - name: privateKey + value: private.key + - name: keys + value: myapp.key + - name: authType + value: oidc + - name: oidcTokenURL + value: https://localhost:8085/issuer1/token + - name: oidcClientID + value: foo + - name: oidcClientSecret + value: bar + - name: oidcScopes + value: openid + - name: oidcAudiences + value: pulsar + - name: oidcTokenCAPEM + value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_four/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_four/pulsar.yml.tmpl new file mode 100644 index 0000000000..e6caab2fb3 --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_four/pulsar.yml.tmpl @@ -0,0 +1,30 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification4 + - name: redeliveryDelay + value: 200ms + - name: certification-pubsub-topic-active.jsonschema + value: "{\"type\":\"record\",\"name\":\"Example\",\"namespace\":\"test\",\"fields\":[{\"name\":\"ID\",\"type\":\"int\"},{\"name\":\"Name\",\"type\":\"string\"}]}" + - name: authType + value: oidc + - name: oidcTokenURL + value: https://localhost:8085/issuer1/token + - name: oidcClientID + value: foo + - name: oidcClientSecret + value: bar + - name: oidcScopes + value: openid + - name: oidcAudiences + value: pulsar + - name: oidcTokenCAPEM + value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_one/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_one/pulsar.yml.tmpl new file mode 100644 index 0000000000..40e3ac6f33 --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_one/pulsar.yml.tmpl @@ -0,0 +1,28 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification1 + - name: redeliveryDelay + value: 200ms + - name: authType + value: oidc + - name: oidcTokenURL + value: https://localhost:8085/issuer1/token + - name: oidcClientID + value: foo + - name: oidcClientSecret + value: bar + - name: oidcScopes + value: openid + - name: oidcAudiences + value: pulsar + - name: oidcTokenCAPEM + value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_six/pulsar.yaml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_six/pulsar.yaml.tmpl new file mode 100644 index 0000000000..67f4ff56d1 --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_six/pulsar.yaml.tmpl @@ -0,0 +1,34 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification5 + - name: redeliveryDelay + value: 200ms + - name: publicKey + value: "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1KDAM4L8RtJ+nLaXBrBh\nzVpvTemsKVZoAct8A+ShepOHT9lgHOCGLFGWNla6K6j+b3AV/P/fAAhwj82vwTDd\nruXSflvSdmYeFAw3Ypphc1A5oM53wSRWhg63potBNWqdDzj8ApYgqjpmjYSQdL5/\na3golb36GYFrY0MLFTv7wZ87pmMIPsOgGIcPbCHker2fRZ34WXYLb1hkeUpwx4eK\njpwcg35gccvR6o/UhbKAuc60V1J9Wof2sNgtlRaQej45wnpjWYzZrIyk5qUbn0Qi\nCdpIrXvYtANq0Id6gP8zJvUEdPIgNuYxEmVCl9jI+8eGI6peD0qIt8U80hf9axhJ\n3QIDAQAB\n-----END PUBLIC KEY-----\n" + - name: privateKey + value: "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA1KDAM4L8RtJ+nLaXBrBhzVpvTemsKVZoAct8A+ShepOHT9lg\nHOCGLFGWNla6K6j+b3AV/P/fAAhwj82vwTDdruXSflvSdmYeFAw3Ypphc1A5oM53\nwSRWhg63potBNWqdDzj8ApYgqjpmjYSQdL5/a3golb36GYFrY0MLFTv7wZ87pmMI\nPsOgGIcPbCHker2fRZ34WXYLb1hkeUpwx4eKjpwcg35gccvR6o/UhbKAuc60V1J9\nWof2sNgtlRaQej45wnpjWYzZrIyk5qUbn0QiCdpIrXvYtANq0Id6gP8zJvUEdPIg\nNuYxEmVCl9jI+8eGI6peD0qIt8U80hf9axhJ3QIDAQABAoIBAQCKuHnM4ac/eXM7\nQPDVX1vfgyHc3hgBPCtNCHnXfGFRvFBqavKGxIElBvGOcBS0CWQ+Rg1Ca5kMx3TQ\njSweSYhH5A7pe3Sa5FK5V6MGxJvRhMSkQi/lJZUBjzaIBJA9jln7pXzdHx8ekE16\nBMPONr6g2dr4nuI9o67xKrtfViwRDGaG6eh7jIMlEqMMc6WqyhvI67rlVDSTHFKX\njlMcozJ3IT8BtTzKg2Tpy7ReVuJEpehum8yn1ZVdAnotBDJxI07DC1cbOP4M2fHM\ngfgPYWmchauZuTeTFu4hrlY5jg0/WLs6by8r/81+vX3QTNvejX9UdTHMSIfQdX82\nAfkCKUVhAoGBAOvGv+YXeTlPRcYC642x5iOyLQm+BiSX4jKtnyJiTU2s/qvvKkIu\nxAOk3OtniT9NaUAHEZE9tI71dDN6IgTLQlAcPCzkVh6Sc5eG0MObqOO7WOMCWBkI\nlaAKKBbd6cGDJkwGCJKnx0pxC9f8R4dw3fmXWgWAr8ENiekMuvjSfjZ5AoGBAObd\ns2L5uiUPTtpyh8WZ7rEvrun3djBhzi+d7rgxEGdditeiLQGKyZbDPMSMBuus/5wH\nwfi0xUq50RtYDbzQQdC3T/C20oHmZbjWK5mDaLRVzWS89YG/NT2Q8eZLBstKqxkx\ngoT77zoUDfRy+CWs1xvXzgxagD5Yg8/OrCuXOqWFAoGAPIw3r6ELknoXEvihASxU\nS4pwInZYIYGXpygLG8teyrnIVOMAWSqlT8JAsXtPNaBtjPHDwyazfZrvEmEk51JD\nX0tA8M5ah1NYt+r5JaKNxp3P/8wUT6lyszyoeubWJsnFRfSusuq/NRC+1+KDg/aq\nKnSBu7QGbm9JoT2RrmBv5RECgYBRn8Lj1I1muvHTNDkiuRj2VniOSirkUkA2/6y+\nPMKi+SS0tqcY63v4rNCYYTW1L7Yz8V44U5mJoQb4lvpMbolGhPljjxAAU3hVkItb\nvGVRlSCIZHKczADD4rJUDOS7DYxO3P1bjUN4kkyYx+lKUMDBHFzCa2D6Kgt4dobS\n5qYajQKBgQC7u7MFPkkEMqNqNGu5erytQkBq1v1Ipmf9rCi3iIj4XJLopxMgw0fx\n6jwcwNInl72KzoUBLnGQ9PKGVeBcgEgdI+a+tq+1TJo6Ta+hZSx+4AYiKY18eRKG\neNuER9NOcSVJ7Eqkcw4viCGyYDm2vgNV9HJ0VlAo3RDh8x5spEN+mg==\n-----END RSA PRIVATE KEY-----\n" + - name: keys + value: myapp.key + - name: authType + value: oidc + - name: oidcTokenURL + value: https://localhost:8085/issuer1/token + - name: oidcClientID + value: foo + - name: oidcClientSecret + value: bar + - name: oidcScopes + value: openid + - name: oidcAudiences + value: pulsar + - name: oidcTokenCAPEM + value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_three/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_three/pulsar.yml.tmpl new file mode 100644 index 0000000000..d1dc3e1095 --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_three/pulsar.yml.tmpl @@ -0,0 +1,28 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification3 + - name: redeliveryDelay + value: 200ms + - name: authType + value: oidc + - name: oidcTokenURL + value: https://localhost:8085/issuer1/token + - name: oidcClientID + value: foo + - name: oidcClientSecret + value: bar + - name: oidcScopes + value: openid + - name: oidcAudiences + value: pulsar + - name: oidcTokenCAPEM + value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_two/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_two/pulsar.yml.tmpl new file mode 100644 index 0000000000..2b4834f17c --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_two/pulsar.yml.tmpl @@ -0,0 +1,28 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification2 + - name: redeliveryDelay + value: 200ms + - name: authType + value: oidc + - name: oidcTokenURL + value: https://localhost:8085/issuer1/token + - name: oidcClientID + value: foo + - name: oidcClientSecret + value: bar + - name: oidcScopes + value: openid + - name: oidcAudiences + value: pulsar + - name: oidcTokenCAPEM + value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/config/.gitignore b/tests/certification/pubsub/pulsar/config/.gitignore new file mode 100644 index 0000000000..3af0ccb687 --- /dev/null +++ b/tests/certification/pubsub/pulsar/config/.gitignore @@ -0,0 +1 @@ +/data diff --git a/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oidc-server.yaml b/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oidc-server.yaml new file mode 100644 index 0000000000..220011d949 --- /dev/null +++ b/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oidc-server.yaml @@ -0,0 +1,14 @@ +# We run in network_mode: "host" so `localhost` is the same for both the host +# and containers. This is required as the mock server uses the SNI hostname to +# build the issuer URL. +version: '3' +services: + mock-oauth2-server: + image: ghcr.io/navikt/mock-oauth2-server:1.0.0 + container_name: mock-oauth2-server + restart: on-failure + network_mode: "host" + environment: + - PORT=8085 + - LOG_LEVEL=DEBUG + - 'JSON_CONFIG={"interactiveLogin":false,"httpServer":{"type":"NettyWrapper","ssl":{}},"tokenCallbacks":[{"issuerId":"issuer1","tokenExpiry":120,"requestMappings":[{"requestParam":"scope","match":"openid","claims":{"sub":"foo","aud":["pulsar"]}}]}]}' diff --git a/tests/certification/pubsub/pulsar/docker-compose.yml b/tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml similarity index 97% rename from tests/certification/pubsub/pulsar/docker-compose.yml rename to tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml index 5ae0535596..7857b77961 100644 --- a/tests/certification/pubsub/pulsar/docker-compose.yml +++ b/tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml @@ -16,4 +16,4 @@ services: - pulsarconf:/pulsar/conf volumes: pulsardata: - pulsarconf: \ No newline at end of file + pulsarconf: diff --git a/tests/certification/pubsub/pulsar/config/docker-compose_auth-oidc.yaml.tmpl b/tests/certification/pubsub/pulsar/config/docker-compose_auth-oidc.yaml.tmpl new file mode 100644 index 0000000000..256ba89677 --- /dev/null +++ b/tests/certification/pubsub/pulsar/config/docker-compose_auth-oidc.yaml.tmpl @@ -0,0 +1,96 @@ +# We run the pulsar services individually as OIDC doesn't seem to work in +# standalone mode. OIDC is also only available from pulsar v3 onwards. We use +# host networking as the mock OAuth server uses the SNI host name to determine +# the host name of the OIDC issuer URL, so we need to have the mock server +# reachable by localhost from both the pulsar services and the host network. +version: '3' +services: + # Start zookeeper + zookeeper: + image: apachepulsar/pulsar:3.0.0 + container_name: zookeeper + restart: on-failure + network_mode: "host" + environment: + - metadataStoreUrl=zk:localhost:2181 + - metricsProvider.httpPort=7000 + - PULSAR_MEM=-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m + command: > + bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \ + bin/generate-zookeeper-config.sh conf/zookeeper.conf && \ + exec bin/pulsar zookeeper" + healthcheck: + test: ["CMD", "bin/pulsar-zookeeper-ruok.sh"] + interval: 1s + timeout: 5s + retries: 300 + + # Init cluster metadata + pulsar-init: + container_name: pulsar-init + image: apachepulsar/pulsar:3.0.0 + network_mode: "host" + env_file: + - ./pulsar_auth-oidc.conf + command: > + bin/pulsar initialize-cluster-metadata \ + --cluster cluster-a \ + --zookeeper localhost:2181 \ + --configuration-store localhost:2181 \ + --web-service-url http://localhost:8080 \ + --broker-service-url pulsar://localhost:6650 + depends_on: + zookeeper: + condition: service_healthy + + # Start bookie + bookie: + image: apachepulsar/pulsar:3.0.0 + container_name: bookie + restart: on-failure + network_mode: "host" + environment: + - clusterName=cluster-a + - zkServers=localhost:2181 + - metadataServiceUri=metadata-store:zk:localhost:2181 + # otherwise every time we run docker compose uo or down we fail to start due to Cookie + # See: https://github.com/apache/bookkeeper/blob/405e72acf42bb1104296447ea8840d805094c787/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Cookie.java#L57-68 + - advertisedAddress=localhost + - BOOKIE_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m + env_file: + - ./pulsar_auth-oidc.conf + volumes: + - "{{ .TmpDir }}:/pulsar/conf/dapr" + depends_on: + zookeeper: + condition: service_healthy + pulsar-init: + condition: service_completed_successfully + command: bash -c "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + + # Start broker + broker: + image: apachepulsar/pulsar:3.0.0 + container_name: broker + restart: on-failure + network_mode: "host" + env_file: + - ./pulsar_auth-oidc.conf + volumes: + - "{{ .TmpDir }}:/pulsar/conf/dapr" + environment: + - metadataStoreUrl=zk:localhost:2181 + - zookeeperServers=localhost:2181 + - clusterName=cluster-a + - managedLedgerDefaultEnsembleSize=1 + - managedLedgerDefaultWriteQuorum=1 + - managedLedgerDefaultAckQuorum=1 + - advertisedAddress=localhost + - advertisedListeners=external:pulsar://127.0.0.1:6650 + - PULSAR_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m + depends_on: + zookeeper: + condition: service_healthy + bookie: + condition: service_started + command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" diff --git a/tests/certification/pubsub/pulsar/config/pulsar_auth-oidc.conf b/tests/certification/pubsub/pulsar/config/pulsar_auth-oidc.conf new file mode 100644 index 0000000000..1bef86ea45 --- /dev/null +++ b/tests/certification/pubsub/pulsar/config/pulsar_auth-oidc.conf @@ -0,0 +1,39 @@ +# Configuration to enable authentication +authenticationEnabled=true +authenticationProviders=org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID + +# Required settings for AuthenticationProviderOpenID +# A comma separated list of allowed, or trusted, token issuers. The token issuer is the URL of the token issuer. +PULSAR_PREFIX_openIDAllowedTokenIssuers=https://localhost:8085/issuer1 +# The list of allowed audiences for the token. The audience is the intended recipient of the token. A token with +# at least one of these audience claims will pass the audience validation check. +PULSAR_PREFIX_openIDAllowedAudiences=pulsar + +# Optional settings (values shown are the defaults) +# The path to the file containing the trusted certificate(s) of the token issuer(s). If not set, uses the default +# trust store of the JVM. Note: in version 3.0.0, the default only applies when this setting is not an environment +# variable and is not in the configuration file. +PULSAR_PREFIX_openIDTokenIssuerTrustCertsFilePath=/pulsar/conf/dapr/ca.pem +# The JWT's claim to use for the role/principal during authorization. +PULSAR_PREFIX_openIDRoleClaim=sub +# The leeway, in seconds, to use when validating the token's expiration time. +PULSAR_PREFIX_openIDAcceptedTimeLeewaySeconds=0 + +# Cache settings +PULSAR_PREFIX_openIDCacheSize=5 +PULSAR_PREFIX_openIDCacheRefreshAfterWriteSeconds=64800 +PULSAR_PREFIX_openIDCacheExpirationSeconds=86400 +PULSAR_PREFIX_openIDHttpConnectionTimeoutMillis=10000 +PULSAR_PREFIX_openIDHttpReadTimeoutMillis=10000 + +# The number of seconds to wait before refreshing the JWKS when a token presents a key ID (kid claim) that is not +# in the cache. This setting is available from Pulsar 3.0.1 and is documented below. +PULSAR_PREFIX_openIDKeyIdCacheMissRefreshSeconds=300 + +# Whether to require that issuers use HTTPS. It is part of the OIDC spec to use HTTPS, so the default is true. +# This setting is for testing purposes and is not recommended for any production environment. +#PULSAR_PREFIX_openIDRequireIssuersUseHttps=false + +# A setting describing how to handle discovery of the OpenID Connect configuration document when the issuer is not +# in the list of allowed issuers. This setting is documented below. +PULSAR_PREFIX_openIDFallbackDiscoveryMode=DISABLED diff --git a/tests/certification/pubsub/pulsar/pulsar_test.go b/tests/certification/pubsub/pulsar/pulsar_test.go index 9c7fd3e405..8d6f1cd80c 100644 --- a/tests/certification/pubsub/pulsar/pulsar_test.go +++ b/tests/certification/pubsub/pulsar/pulsar_test.go @@ -16,17 +16,28 @@ package pulsar_test import ( "bytes" "context" + "crypto/tls" "encoding/json" + "encoding/pem" "fmt" + "io" + "io/fs" "io/ioutil" "net/http" + "os" + "os/exec" + "path/filepath" + "strings" "testing" + "text/template" "time" "github.com/google/uuid" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "go.uber.org/multierr" + "github.com/dapr/components-contrib/internal/authentication/oidc" pubsub_pulsar "github.com/dapr/components-contrib/pubsub/pulsar" pubsub_loader "github.com/dapr/dapr/pkg/components/pubsub" @@ -54,21 +65,23 @@ const ( appID1 = "app-1" appID2 = "app-2" - numMessages = 10 - appPort = 8000 - portOffset = 2 - messageKey = "partitionKey" - pubsubName = "messagebus" - topicActiveName = "certification-pubsub-topic-active" - topicPassiveName = "certification-pubsub-topic-passive" - topicToBeCreated = "certification-topic-per-test-run" - topicDefaultName = "certification-topic-default" - topicMultiPartitionName = "certification-topic-multi-partition8" - partition0 = "partition-0" - partition1 = "partition-1" - clusterName = "pulsarcertification" - dockerComposeYAML = "docker-compose.yml" - pulsarURL = "localhost:6650" + numMessages = 10 + appPort = 8001 + portOffset = 2 + messageKey = "partitionKey" + pubsubName = "messagebus" + topicActiveName = "certification-pubsub-topic-active" + topicPassiveName = "certification-pubsub-topic-passive" + topicToBeCreated = "certification-topic-per-test-run" + topicDefaultName = "certification-topic-default" + topicMultiPartitionName = "certification-topic-multi-partition8" + partition0 = "partition-0" + partition1 = "partition-1" + clusterName = "pulsarcertification" + dockerComposeAuthNoneYAML = "./config/docker-compose_auth-none.yaml" + dockerComposeAuthOIDCYAML = "./config/docker-compose_auth-oidc.yaml.tmpl" + dockerComposeMockOAuthYAML = "./config/docker-compose_auth-mock-oidc-server.yaml" + pulsarURL = "localhost:6650" subscribeTypeKey = "subscribeType" @@ -82,6 +95,103 @@ const ( processModeSync = "sync" ) +type pulsarSuite struct { + suite.Suite + + authType string + oidcCAPEM []byte + dockerComposeYAML string + componentsPath string + services []string +} + +func TestPulsar(t *testing.T) { + t.Run("Auth:None", func(t *testing.T) { + suite.Run(t, &pulsarSuite{ + authType: "none", + dockerComposeYAML: dockerComposeAuthNoneYAML, + componentsPath: "./components/auth-none", + services: []string{"standalone"}, + }) + }) + + t.Run("Auth:OIDC", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.Chmod(dir, 0o777)) + + t.Log("Starting OIDC server...") + out, err := exec.Command( + "docker-compose", + "-p", "oidc", + "-f", dockerComposeMockOAuthYAML, + "up", "-d").CombinedOutput() + require.NoError(t, err, string(out)) + t.Log(string(out)) + + t.Cleanup(func() { + t.Log("Stopping OIDC server...") + out, err = exec.Command( + "docker-compose", + "-p", "oidc", + "-f", dockerComposeMockOAuthYAML, + "down", "-v", + "--remove-orphans").CombinedOutput() + require.NoError(t, err, string(out)) + t.Log(string(out)) + }) + + t.Log("Waiting for OAuth server to be ready...") + oauthCA := peerCertificate(t, "localhost:8085") + t.Log("OAuth server is ready") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.pem"), oauthCA, 0o644)) + outf, err := os.OpenFile("./config/pulsar_auth-oidc.conf", os.O_RDONLY, 0o644) + require.NoError(t, err) + inf, err := os.OpenFile(filepath.Join(dir, "pulsar_auth-oidc.conf"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + require.NoError(t, err) + _, err = io.Copy(inf, outf) + require.NoError(t, err) + outf.Close() + inf.Close() + + td := struct { + TmpDir string + OIDCCAPEM string + }{ + TmpDir: dir, + OIDCCAPEM: strings.ReplaceAll(string(oauthCA), "\n", "\\n"), + } + + tmpl, err := template.New("").ParseFiles(dockerComposeAuthOIDCYAML) + require.NoError(t, err) + f, err := os.OpenFile(filepath.Join(dir, "docker-compose.yaml"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + require.NoError(t, err) + require.NoError(t, tmpl.ExecuteTemplate(f, "docker-compose_auth-oidc.yaml.tmpl", td)) + + require.NoError(t, filepath.Walk("./components/auth-oidc", func(path string, info fs.FileInfo, err error) error { + if info.IsDir() { + return nil + } + tmpl, err := template.New("").ParseFiles(path) + require.NoError(t, err) + path = strings.TrimSuffix(path, ".tmpl") + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, path)), 0o755)) + f, err := os.OpenFile(filepath.Join(dir, path), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + require.NoError(t, err) + require.NoError(t, tmpl.ExecuteTemplate(f, filepath.Base(path)+".tmpl", td)) + return nil + })) + + suite.Run(t, &pulsarSuite{ + oidcCAPEM: oauthCA, + authType: "oidc", + dockerComposeYAML: filepath.Join(dir, "docker-compose.yaml"), + componentsPath: filepath.Join(dir, "components/auth-oidc"), + services: []string{"zookeeper", "pulsar-init", "bookie", "broker"}, + }) + }) +} + func subscriberApplication(appID string, topicName string, messagesWatcher *watcher.Watcher) app.SetupFn { return func(ctx flow.Context, s common.Service) error { // Simulate periodic errors. @@ -196,7 +306,8 @@ func assertMessages(timeout time.Duration, messageWatchers ...*watcher.Watcher) } } -func TestPulsar(t *testing.T) { +func (p *pulsarSuite) TestPulsar() { + t := p.T() consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -244,10 +355,10 @@ func TestPulsar(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -267,7 +378,7 @@ func TestPulsar(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -280,7 +391,7 @@ func TestPulsar(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -295,7 +406,8 @@ func TestPulsar(t *testing.T) { Run() } -func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { +func (p *pulsarSuite) TestPulsarMultipleSubsSameConsumerIDs() { + t := p.T() consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -312,10 +424,10 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -335,7 +447,7 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -348,7 +460,7 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -362,7 +474,9 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { Run() } -func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { +func (p *pulsarSuite) TestPulsarMultipleSubsDifferentConsumerIDs() { + t := p.T() + consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -376,10 +490,10 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -401,7 +515,7 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -414,7 +528,7 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -427,7 +541,8 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { Run() } -func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { +func (p *pulsarSuite) TestPulsarMultiplePubSubsDifferentConsumerIDs() { + t := p.T() consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -445,10 +560,10 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -470,7 +585,7 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -483,7 +598,7 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -498,7 +613,8 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { Run() } -func TestPulsarNonexistingTopic(t *testing.T) { +func (p *pulsarSuite) TestPulsarNonexistingTopic() { + t := p.T() consumerGroup1 := watcher.NewUnordered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -511,10 +627,10 @@ func TestPulsarNonexistingTopic(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset*3), subscriberApplication(appID1, topicToBeCreated, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -537,7 +653,7 @@ func TestPulsarNonexistingTopic(t *testing.T) { })). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset*3), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*3), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*3), @@ -550,7 +666,8 @@ func TestPulsarNonexistingTopic(t *testing.T) { Run() } -func TestPulsarNetworkInterruption(t *testing.T) { +func (p *pulsarSuite) TestPulsarNetworkInterruption() { + t := p.T() consumerGroup1 := watcher.NewUnordered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -563,10 +680,10 @@ func TestPulsarNetworkInterruption(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -589,7 +706,7 @@ func TestPulsarNetworkInterruption(t *testing.T) { })). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -603,7 +720,8 @@ func TestPulsarNetworkInterruption(t *testing.T) { Run() } -func TestPulsarPersitant(t *testing.T) { +func (p *pulsarSuite) TestPulsarPersitant() { + t := p.T() consumerGroup1 := watcher.NewUnordered() flow.New(t, "pulsar certification persistant test"). @@ -611,10 +729,10 @@ func TestPulsarPersitant(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -636,23 +754,24 @@ func TestPulsarPersitant(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), componentRuntimeOptions(), )). Step("publish messages to topic1", publishMessages(nil, sidecarName1, topicActiveName, consumerGroup1)). - Step("stop pulsar server", dockercompose.Stop(clusterName, dockerComposeYAML, "standalone")). + Step("stop pulsar server", dockercompose.Stop(clusterName, p.dockerComposeYAML, p.services...)). Step("wait", flow.Sleep(5*time.Second)). - Step("start pulsar server", dockercompose.Start(clusterName, dockerComposeYAML, "standalone")). + Step("start pulsar server", dockercompose.Start(clusterName, p.dockerComposeYAML, p.services...)). Step("wait", flow.Sleep(10*time.Second)). Step("verify if app1 has received messages published to active topic", assertMessages(10*time.Second, consumerGroup1)). Step("reset", flow.Reset(consumerGroup1)). Run() } -func TestPulsarDelay(t *testing.T) { +func (p *pulsarSuite) TestPulsarDelay() { + t := p.T() consumerGroup1 := watcher.NewUnordered() date := time.Now() @@ -682,10 +801,10 @@ func TestPulsarDelay(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -707,7 +826,7 @@ func TestPulsarDelay(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_three"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_three")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -730,7 +849,8 @@ type schemaTest struct { Name string `json:"name"` } -func TestPulsarSchema(t *testing.T) { +func (p *pulsarSuite) TestPulsarSchema() { + t := p.T() consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -772,10 +892,10 @@ func TestPulsarSchema(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -795,7 +915,7 @@ func TestPulsarSchema(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_four"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_four")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -818,7 +938,7 @@ func componentRuntimeOptions() []runtime.Option { } } -func createMultiPartitionTopic(tenant, namespace, topic string, partition int) flow.Runnable { +func (p *pulsarSuite) createMultiPartitionTopic(tenant, namespace, topic string, partition int) flow.Runnable { return func(ctx flow.Context) error { reqURL := fmt.Sprintf("http://localhost:8080/admin/v2/persistent/%s/%s/%s/partitions", tenant, namespace, topic) @@ -838,6 +958,19 @@ func createMultiPartitionTopic(tenant, namespace, topic string, partition int) f req.Header.Set("Content-Type", "application/json") + if p.authType == "oidc" { + cc, err := p.oidcClientCredentials() + if err != nil { + return err + } + token, err := cc.Token() + if err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + } + rsp, err := http.DefaultClient.Do(req) if err != nil { @@ -858,7 +991,8 @@ func createMultiPartitionTopic(tenant, namespace, topic string, partition int) f } } -func TestPulsarPartitionedOrderingProcess(t *testing.T) { +func (p *pulsarSuite) TestPulsarPartitionedOrderingProcess() { + t := p.T() consumerGroup1 := watcher.NewOrdered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -867,14 +1001,14 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { } flow.New(t, "pulsar certification - process message in order with partitioned-topic"). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset), subscriberApplicationWithoutError(appID1, topicMultiPartitionName, consumerGroup1))). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -897,10 +1031,10 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { return err })). Step("create multi-partition topic explicitly", retry.Do(10*time.Second, 30, - createMultiPartitionTopic("public", "default", topicMultiPartitionName, 4))). + p.createMultiPartitionTopic("public", "default", topicMultiPartitionName, 4))). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -913,7 +1047,7 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset*3), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*3), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*3), @@ -927,7 +1061,8 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { Run() } -func TestPulsarEncryptionFromFile(t *testing.T) { +func (p *pulsarSuite) TestPulsarEncryptionFromFile() { + t := p.T() consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -969,10 +1104,10 @@ func TestPulsarEncryptionFromFile(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -992,7 +1127,7 @@ func TestPulsarEncryptionFromFile(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_five"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_five")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -1004,7 +1139,8 @@ func TestPulsarEncryptionFromFile(t *testing.T) { Run() } -func TestPulsarEncryptionFromData(t *testing.T) { +func (p *pulsarSuite) TestPulsarEncryptionFromData() { + t := p.T() consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -1046,10 +1182,10 @@ func TestPulsarEncryptionFromData(t *testing.T) { // Run subscriberApplication app2 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -1069,7 +1205,7 @@ func TestPulsarEncryptionFromData(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_six"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_six")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -1080,3 +1216,57 @@ func TestPulsarEncryptionFromData(t *testing.T) { Step("reset", flow.Reset(consumerGroup1)). Run() } + +func (p *pulsarSuite) client(t *testing.T) (pulsar.Client, error) { + t.Helper() + + opts := pulsar.ClientOptions{ + URL: "pulsar://localhost:6650", + } + switch p.authType { + case "oidc": + cc, err := p.oidcClientCredentials() + require.NoError(t, err) + opts.Authentication = pulsar.NewAuthenticationTokenFromSupplier(cc.Token) + default: + } + + return pulsar.NewClient(opts) +} + +func (p *pulsarSuite) oidcClientCredentials() (*oidc.ClientCredentials, error) { + cc, err := oidc.NewClientCredentials(context.Background(), oidc.ClientCredentialsOptions{ + Logger: logger.NewLogger("dapr.test.readiness"), + TokenURL: "https://localhost:8085/issuer1/token", + ClientID: "foo", + ClientSecret: "bar", + Scopes: []string{"openid"}, + Audiences: []string{"pulsar"}, + CAPEM: p.oidcCAPEM, + }) + if err != nil { + return nil, err + } + + return cc, nil +} + +func peerCertificate(t *testing.T, hostport string) []byte { + conf := &tls.Config{InsecureSkipVerify: true} + + for { + time.Sleep(1 * time.Second) + + conn, err := tls.Dial("tcp", hostport, conf) + if err != nil { + t.Log(err) + continue + } + + defer conn.Close() + + certs := conn.ConnectionState().PeerCertificates + require.Len(t, certs, 1, "expected 1 peer certificate") + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certs[0].Raw}) + } +} From c43af14d313a81dd34c2e8b9fb0fa1c084b3c6ad Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Tue, 1 Aug 2023 11:54:01 -0700 Subject: [PATCH 08/22] Revert "Adds OIDC authentication support for pubsub Apache Pulsar (#3023)" This reverts commit af5bacebf4877e0c10ba860737ce8a7ccb0ee02c. --- .github/workflows/certification.yml | 2 +- .../authentication/oidc/clientcredentials.go | 213 ----------- .../oidc/clientcredentials_test.go | 266 -------------- pubsub/pulsar/metadata.go | 10 +- pubsub/pulsar/pulsar.go | 59 +--- .../auth-oidc/consumer_five/pulsar.yml.tmpl | 34 -- .../auth-oidc/consumer_four/pulsar.yml.tmpl | 30 -- .../auth-oidc/consumer_one/pulsar.yml.tmpl | 28 -- .../auth-oidc/consumer_six/pulsar.yaml.tmpl | 34 -- .../auth-oidc/consumer_three/pulsar.yml.tmpl | 28 -- .../auth-oidc/consumer_two/pulsar.yml.tmpl | 28 -- .../{auth-none => }/consumer_five/pulsar.yaml | 0 .../{auth-none => }/consumer_four/pulsar.yaml | 0 .../{auth-none => }/consumer_one/pulsar.yml | 0 .../{auth-none => }/consumer_six/pulsar.yaml | 0 .../{auth-none => }/consumer_three/pulsar.yml | 0 .../{auth-none => }/consumer_two/pulsar.yml | 0 .../pubsub/pulsar/config/.gitignore | 1 - .../docker-compose_auth-mock-oidc-server.yaml | 14 - .../config/docker-compose_auth-oidc.yaml.tmpl | 96 ----- .../pulsar/config/pulsar_auth-oidc.conf | 39 -- ...pose_auth-none.yaml => docker-compose.yml} | 2 +- .../pubsub/pulsar/pulsar_test.go | 334 ++++-------------- 23 files changed, 86 insertions(+), 1132 deletions(-) delete mode 100644 internal/authentication/oidc/clientcredentials.go delete mode 100644 internal/authentication/oidc/clientcredentials_test.go delete mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_five/pulsar.yml.tmpl delete mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_four/pulsar.yml.tmpl delete mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_one/pulsar.yml.tmpl delete mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_six/pulsar.yaml.tmpl delete mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_three/pulsar.yml.tmpl delete mode 100644 tests/certification/pubsub/pulsar/components/auth-oidc/consumer_two/pulsar.yml.tmpl rename tests/certification/pubsub/pulsar/components/{auth-none => }/consumer_five/pulsar.yaml (100%) rename tests/certification/pubsub/pulsar/components/{auth-none => }/consumer_four/pulsar.yaml (100%) rename tests/certification/pubsub/pulsar/components/{auth-none => }/consumer_one/pulsar.yml (100%) rename tests/certification/pubsub/pulsar/components/{auth-none => }/consumer_six/pulsar.yaml (100%) rename tests/certification/pubsub/pulsar/components/{auth-none => }/consumer_three/pulsar.yml (100%) rename tests/certification/pubsub/pulsar/components/{auth-none => }/consumer_two/pulsar.yml (100%) delete mode 100644 tests/certification/pubsub/pulsar/config/.gitignore delete mode 100644 tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oidc-server.yaml delete mode 100644 tests/certification/pubsub/pulsar/config/docker-compose_auth-oidc.yaml.tmpl delete mode 100644 tests/certification/pubsub/pulsar/config/pulsar_auth-oidc.conf rename tests/certification/pubsub/pulsar/{config/docker-compose_auth-none.yaml => docker-compose.yml} (97%) diff --git a/.github/workflows/certification.yml b/.github/workflows/certification.yml index 5482b986a1..dfa2fe39ba 100644 --- a/.github/workflows/certification.yml +++ b/.github/workflows/certification.yml @@ -258,7 +258,7 @@ jobs: set +e gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json \ --junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.xml --format standard-quiet -- \ - -coverprofile=cover.out -covermode=set -tags=certtests -timeout=30m -coverpkg=${{ matrix.source-pkg }} + -coverprofile=cover.out -covermode=set -tags=certtests -coverpkg=${{ matrix.source-pkg }} status=$? echo "Completed certification tests for ${{ matrix.component }} ... " if test $status -ne 0; then diff --git a/internal/authentication/oidc/clientcredentials.go b/internal/authentication/oidc/clientcredentials.go deleted file mode 100644 index 902f78b561..0000000000 --- a/internal/authentication/oidc/clientcredentials.go +++ /dev/null @@ -1,213 +0,0 @@ -/* -Copyright 2021 The Dapr Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package oidc - -import ( - "context" - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "net/http" - "net/url" - "sync" - "sync/atomic" - "time" - - "golang.org/x/oauth2" - ccreds "golang.org/x/oauth2/clientcredentials" - "k8s.io/utils/clock" - - "github.com/dapr/kit/logger" -) - -const ( - oidcScopeOpenID = "openid" -) - -type ClientCredentialsOptions struct { - Logger logger.Logger - TokenURL string - ClientID string - ClientSecret string - Scopes []string - Audiences []string - CAPEM []byte -} - -// ClientCredentials is an OAuth2 Token Source that uses the client_credentials -// grant type to fetch a token. -type ClientCredentials struct { - log logger.Logger - currentToken *oauth2.Token - httpClient *http.Client - fetchTokenFn func(context.Context) (*oauth2.Token, error) - - lock sync.RWMutex - wg sync.WaitGroup - closeCh chan struct{} - closed atomic.Bool - clock clock.Clock -} - -func NewClientCredentials(ctx context.Context, opts ClientCredentialsOptions) (*ClientCredentials, error) { - conf, httpClient, err := toConfig(opts) - if err != nil { - return nil, err - } - - token, err := conf.Token(context.WithValue(ctx, oauth2.HTTPClient, httpClient)) - if err != nil { - return nil, fmt.Errorf("error fetching initial oidc client_credentials token: %w", err) - } - - opts.Logger.Info("Fetched initial oidc client_credentials token") - - return &ClientCredentials{ - log: opts.Logger, - currentToken: token, - httpClient: httpClient, - closeCh: make(chan struct{}), - clock: clock.RealClock{}, - fetchTokenFn: conf.Token, - }, nil -} - -func (c *ClientCredentials) Run(ctx context.Context) { - c.log.Info("Running oidc client_credentials token renewer") - renewDuration := c.tokenRenewDuration() - - c.wg.Add(1) - go func() { - defer func() { - c.log.Info("Stopped oidc client_credentials token renewer") - c.wg.Done() - }() - - for { - select { - case <-c.closeCh: - return - case <-ctx.Done(): - return - case <-c.clock.After(renewDuration): - } - - c.log.Debug("Renewing client credentials token") - - token, err := c.fetchTokenFn(context.WithValue(ctx, oauth2.HTTPClient, c.httpClient)) - if err != nil { - c.log.Errorf("Error fetching renewed oidc client_credentials token, retrying in 30 seconds: %s", err) - renewDuration = time.Second * 30 - continue - } - - c.lock.Lock() - c.currentToken = token - c.lock.Unlock() - renewDuration = c.tokenRenewDuration() - } - }() -} - -func toConfig(opts ClientCredentialsOptions) (*ccreds.Config, *http.Client, error) { - scopes := opts.Scopes - if len(scopes) == 0 { - // If no scopes are provided, then the default is to use the 'openid' scope - // since that is always required for OIDC so implicitly add it. - scopes = []string{oidcScopeOpenID} - } - - var oidcScopeFound bool - for _, scope := range scopes { - if scope == oidcScopeOpenID { - oidcScopeFound = true - break - } - } - if !oidcScopeFound { - return nil, nil, fmt.Errorf("oidc client_credentials token source requires the %q scope", oidcScopeOpenID) - } - - tokenURL, err := url.Parse(opts.TokenURL) - if err != nil { - return nil, nil, fmt.Errorf("error parsing token URL: %w", err) - } - if tokenURL.Scheme != "https" { - return nil, nil, fmt.Errorf("OIDC token provider URL requires 'https' scheme: %q", tokenURL) - } - - conf := &ccreds.Config{ - ClientID: opts.ClientID, - ClientSecret: opts.ClientSecret, - TokenURL: opts.TokenURL, - Scopes: scopes, - } - - if len(opts.Audiences) == 0 { - return nil, nil, errors.New("oidc client_credentials token source requires at least one audience") - } - - conf.EndpointParams = url.Values{"audience": opts.Audiences} - - // If caPool is nil, then the Go TLS library will use the system's root CA. - var caPool *x509.CertPool - if len(opts.CAPEM) > 0 { - caPool = x509.NewCertPool() - if !caPool.AppendCertsFromPEM(opts.CAPEM) { - return nil, nil, errors.New("failed to parse CA PEM") - } - } - - return conf, &http.Client{ - Timeout: time.Second * 30, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - MinVersion: tls.VersionTLS12, - RootCAs: caPool, - }, - }, - }, nil -} - -func (c *ClientCredentials) Close() { - defer c.wg.Wait() - - if c.closed.CompareAndSwap(false, true) { - close(c.closeCh) - } -} - -func (c *ClientCredentials) Token() (string, error) { - c.lock.RLock() - defer c.lock.RUnlock() - - if c.closed.Load() { - return "", errors.New("client_credentials token source is closed") - } - - if !c.currentToken.Valid() { - return "", errors.New("client_credentials token source is invalid") - } - - return c.currentToken.AccessToken, nil -} - -// tokenRenewTime returns the duration when the token should be renewed, which is -// half of the token's lifetime. -func (c *ClientCredentials) tokenRenewDuration() time.Duration { - c.lock.RLock() - defer c.lock.RUnlock() - return c.currentToken.Expiry.Sub(c.clock.Now()) / 2 -} diff --git a/internal/authentication/oidc/clientcredentials_test.go b/internal/authentication/oidc/clientcredentials_test.go deleted file mode 100644 index 8a090037ac..0000000000 --- a/internal/authentication/oidc/clientcredentials_test.go +++ /dev/null @@ -1,266 +0,0 @@ -/* -Copyright 2021 The Dapr Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package oidc - -import ( - "context" - "errors" - "net/url" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "golang.org/x/oauth2" - ccreds "golang.org/x/oauth2/clientcredentials" - "k8s.io/utils/clock" - clocktesting "k8s.io/utils/clock/testing" - - "github.com/dapr/kit/logger" -) - -func TestRun(t *testing.T) { - var lock sync.Mutex - clock := clocktesting.NewFakeClock(time.Now()) - called := make(chan struct{}, 1) - var retErr error = nil - - fetchTokenFn := func(context.Context) (*oauth2.Token, error) { - lock.Lock() - defer lock.Unlock() - called <- struct{}{} - return &oauth2.Token{ - Expiry: clock.Now().Add(time.Minute), - }, retErr - } - - t.Run("should return when context is cancelled", func(t *testing.T) { - c := &ClientCredentials{ - log: logger.NewLogger("test"), - clock: clock, - fetchTokenFn: fetchTokenFn, - closeCh: make(chan struct{}), - currentToken: &oauth2.Token{ - Expiry: clock.Now(), - }, - } - - ctx, cancel := context.WithCancel(context.Background()) - - c.Run(ctx) - cancel() - - select { - case <-called: - t.Fatal("should not have called fetchTokenFn") - default: - } - }) - - t.Run("should return when closed", func(t *testing.T) { - c := &ClientCredentials{ - log: logger.NewLogger("test"), - clock: clock, - fetchTokenFn: fetchTokenFn, - closeCh: make(chan struct{}), - currentToken: &oauth2.Token{ - Expiry: clock.Now(), - }, - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - c.Run(ctx) - - // Should be able to close multiple times. - c.Close() - c.Close() - - select { - case <-called: - t.Fatal("should not have called fetchTokenFn") - case <-c.closeCh: - case <-time.After(time.Second * 5): - t.Fatal("should have closed run") - } - }) - - t.Run("should renew token when ready for renewal", func(t *testing.T) { - c := &ClientCredentials{ - log: logger.NewLogger("test"), - clock: clock, - fetchTokenFn: fetchTokenFn, - closeCh: make(chan struct{}), - currentToken: &oauth2.Token{Expiry: clock.Now().Add(time.Minute * 2)}, - } - - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - c.Run(ctx) - - assert.Eventually(t, clock.HasWaiters, time.Second*5, time.Millisecond*10) - clock.Step(time.Minute * 1) - - select { - case <-called: - case <-time.After(time.Second * 5): - t.Fatal("should have called") - } - }) - - t.Run("should call renew again after 30 seconds when it fails", func(t *testing.T) { - c := &ClientCredentials{ - log: logger.NewLogger("test"), - clock: clock, - fetchTokenFn: fetchTokenFn, - closeCh: make(chan struct{}), - currentToken: &oauth2.Token{ - Expiry: clock.Now(), - }, - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - c.Run(ctx) - - assert.Eventually(t, clock.HasWaiters, time.Second*5, time.Millisecond*10) - clock.Step(time.Minute * 1) - - select { - case <-called: - case <-time.After(time.Second * 5): - t.Fatal("should have called") - } - - lock.Lock() - retErr = errors.New("test error") - lock.Unlock() - - assert.Eventually(t, clock.HasWaiters, time.Second*5, time.Millisecond*10) - clock.Step(time.Minute * 1) - - select { - case <-called: - case <-time.After(time.Second * 5): - t.Fatal("should have called") - } - - assert.Eventually(t, clock.HasWaiters, time.Second*5, time.Millisecond*10) - clock.Step(time.Second * 30) - - select { - case <-called: - case <-time.After(time.Second * 5): - t.Fatal("should have called") - } - - c.Close() - - select { - case <-c.closeCh: - case <-time.After(time.Second * 5): - t.Fatal("should have closed run") - } - }) -} - -func Test_tokenRenewDuration(t *testing.T) { - c := &ClientCredentials{ - clock: clock.RealClock{}, - currentToken: &oauth2.Token{ - Expiry: time.Now(), - }, - } - assert.InDelta(t, c.tokenRenewDuration(), time.Duration(0), float64(time.Second*5)) - - c = &ClientCredentials{ - clock: clock.RealClock{}, - currentToken: &oauth2.Token{ - Expiry: time.Now().Add(time.Hour), - }, - } - assert.InDelta(t, c.tokenRenewDuration(), time.Minute*30, float64(time.Second*5)) -} - -func Test_toConfig(t *testing.T) { - tests := map[string]struct { - opts ClientCredentialsOptions - expConfig *ccreds.Config - expErr bool - }{ - "openid not in scopes should error": { - opts: ClientCredentialsOptions{ - TokenURL: "https://localhost:8080", - ClientID: "client-id", - ClientSecret: "client-secret", - Scopes: []string{"profile"}, - Audiences: []string{"audience"}, - }, - expErr: true, - }, - "non-https endpoint should error": { - opts: ClientCredentialsOptions{ - TokenURL: "http://localhost:8080", - ClientID: "client-id", - ClientSecret: "client-secret", - Audiences: []string{"audience"}, - }, - expErr: true, - }, - "bad CA certificate should error": { - opts: ClientCredentialsOptions{ - TokenURL: "https://localhost:8080", - ClientID: "client-id", - ClientSecret: "client-secret", - Audiences: []string{"audience"}, - CAPEM: []byte("ca-pem"), - }, - expErr: true, - }, - "no audiences should error": { - opts: ClientCredentialsOptions{ - TokenURL: "https://localhost:8080", - ClientID: "client-id", - ClientSecret: "client-secret", - }, - expErr: true, - }, - "should default scope": { - opts: ClientCredentialsOptions{ - TokenURL: "https://localhost:8080", - ClientID: "client-id", - ClientSecret: "client-secret", - Audiences: []string{"audience"}, - }, - expConfig: &ccreds.Config{ - ClientID: "client-id", - ClientSecret: "client-secret", - TokenURL: "https://localhost:8080", - Scopes: []string{"openid"}, - EndpointParams: url.Values{"audience": []string{"audience"}}, - }, - expErr: false, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - config, _, err := toConfig(test.opts) - assert.Equalf(t, test.expErr, err != nil, "%v", err) - assert.Equal(t, test.expConfig, config) - }) - } -} diff --git a/pubsub/pulsar/metadata.go b/pubsub/pulsar/metadata.go index aae2e20bb2..5bddedb6d6 100644 --- a/pubsub/pulsar/metadata.go +++ b/pubsub/pulsar/metadata.go @@ -26,20 +26,12 @@ type pulsarMetadata struct { Tenant string `mapstructure:"tenant"` Namespace string `mapstructure:"namespace"` Persistent bool `mapstructure:"persistent"` + Token string `mapstructure:"token"` RedeliveryDelay time.Duration `mapstructure:"redeliveryDelay"` internalTopicSchemas map[string]schemaMetadata `mapstructure:"-"` PublicKey string `mapstructure:"publicKey"` PrivateKey string `mapstructure:"privateKey"` Keys string `mapstructure:"keys"` - - AuthType string `mapstructure:"authType"` - Token string `mapstructure:"token"` - OIDCTokenCAPEM string `mapstructure:"oidcTokenCAPEM"` - OIDCTokenURL string `mapstructure:"oidcTokenURL"` - OIDCClientID string `mapstructure:"oidcClientID"` - OIDCClientSecret string `mapstructure:"oidcClientSecret"` - OIDCAudiences []string `mapstructure:"oidcAudiences"` - OIDCScopes []string `mapstructure:"oidcScopes"` } type schemaMetadata struct { diff --git a/pubsub/pulsar/pulsar.go b/pubsub/pulsar/pulsar.go index 5df4216912..8846f2dc50 100644 --- a/pubsub/pulsar/pulsar.go +++ b/pubsub/pulsar/pulsar.go @@ -25,12 +25,12 @@ import ( "sync/atomic" "time" + "github.com/hamba/avro/v2" + "github.com/apache/pulsar-client-go/pulsar" "github.com/apache/pulsar-client-go/pulsar/crypto" - "github.com/hamba/avro/v2" lru "github.com/hashicorp/golang-lru/v2" - "github.com/dapr/components-contrib/internal/authentication/oidc" "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/pubsub" "github.com/dapr/kit/logger" @@ -55,10 +55,6 @@ const ( protoProtocol = "proto" partitionKey = "partitionKey" - authTypeNone = "none" - authTypeToken = "token" - authTypeOIDC = "oidc" - defaultTenant = "public" defaultNamespace = "default" cachedNumProducer = 10 @@ -98,14 +94,13 @@ const ( type ProcessMode string type Pulsar struct { - logger logger.Logger - client pulsar.Client - metadata pulsarMetadata - oidcProvider *oidc.ClientCredentials - cache *lru.Cache[string, pulsar.Producer] - closed atomic.Bool - closeCh chan struct{} - wg sync.WaitGroup + logger logger.Logger + client pulsar.Client + metadata pulsarMetadata + cache *lru.Cache[string, pulsar.Producer] + closed atomic.Bool + closeCh chan struct{} + wg sync.WaitGroup } func NewPulsar(l logger.Logger) pubsub.PubSub { @@ -162,7 +157,7 @@ func parsePulsarMetadata(meta pubsub.Metadata) (*pulsarMetadata, error) { return &m, nil } -func (p *Pulsar) Init(ctx context.Context, metadata pubsub.Metadata) error { +func (p *Pulsar) Init(_ context.Context, metadata pubsub.Metadata) error { m, err := parsePulsarMetadata(metadata) if err != nil { return err @@ -178,37 +173,9 @@ func (p *Pulsar) Init(ctx context.Context, metadata pubsub.Metadata) error { ConnectionTimeout: 30 * time.Second, TLSAllowInsecureConnection: !m.EnableTLS, } - - switch m.AuthType { - case "": - // To ensure backward compatibility, if authType is not set but the token - // is we fallthrough to token auth. - if m.Token == "" { - break - } - fallthrough - case authTypeToken: + if m.Token != "" { options.Authentication = pulsar.NewAuthenticationToken(m.Token) - case authTypeOIDC: - var cc *oidc.ClientCredentials - cc, err = oidc.NewClientCredentials(ctx, oidc.ClientCredentialsOptions{ - Logger: p.logger, - TokenURL: m.OIDCTokenURL, - CAPEM: []byte(m.OIDCTokenCAPEM), - ClientID: m.OIDCClientID, - ClientSecret: m.OIDCClientSecret, - Scopes: m.OIDCScopes, - Audiences: m.OIDCAudiences, - }) - if err != nil { - return fmt.Errorf("could not instantiate oidc token provider: %v", err) - } - - options.Authentication = pulsar.NewAuthenticationTokenFromSupplier(cc.Token) - p.oidcProvider = cc - p.oidcProvider.Run(ctx) } - client, err := pulsar.NewClient(options) if err != nil { return fmt.Errorf("could not instantiate pulsar client: %v", err) @@ -523,10 +490,6 @@ func (p *Pulsar) Close() error { } p.client.Close() - if p.oidcProvider != nil { - p.oidcProvider.Close() - } - return nil } diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_five/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_five/pulsar.yml.tmpl deleted file mode 100644 index 1b9189d6ce..0000000000 --- a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_five/pulsar.yml.tmpl +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: messagebus -spec: - type: pubsub.pulsar - version: v1 - metadata: - - name: host - value: "localhost:6650" - - name: consumerID - value: certification5 - - name: redeliveryDelay - value: 200ms - - name: publicKey - value: public.key - - name: privateKey - value: private.key - - name: keys - value: myapp.key - - name: authType - value: oidc - - name: oidcTokenURL - value: https://localhost:8085/issuer1/token - - name: oidcClientID - value: foo - - name: oidcClientSecret - value: bar - - name: oidcScopes - value: openid - - name: oidcAudiences - value: pulsar - - name: oidcTokenCAPEM - value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_four/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_four/pulsar.yml.tmpl deleted file mode 100644 index e6caab2fb3..0000000000 --- a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_four/pulsar.yml.tmpl +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: messagebus -spec: - type: pubsub.pulsar - version: v1 - metadata: - - name: host - value: "localhost:6650" - - name: consumerID - value: certification4 - - name: redeliveryDelay - value: 200ms - - name: certification-pubsub-topic-active.jsonschema - value: "{\"type\":\"record\",\"name\":\"Example\",\"namespace\":\"test\",\"fields\":[{\"name\":\"ID\",\"type\":\"int\"},{\"name\":\"Name\",\"type\":\"string\"}]}" - - name: authType - value: oidc - - name: oidcTokenURL - value: https://localhost:8085/issuer1/token - - name: oidcClientID - value: foo - - name: oidcClientSecret - value: bar - - name: oidcScopes - value: openid - - name: oidcAudiences - value: pulsar - - name: oidcTokenCAPEM - value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_one/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_one/pulsar.yml.tmpl deleted file mode 100644 index 40e3ac6f33..0000000000 --- a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_one/pulsar.yml.tmpl +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: messagebus -spec: - type: pubsub.pulsar - version: v1 - metadata: - - name: host - value: "localhost:6650" - - name: consumerID - value: certification1 - - name: redeliveryDelay - value: 200ms - - name: authType - value: oidc - - name: oidcTokenURL - value: https://localhost:8085/issuer1/token - - name: oidcClientID - value: foo - - name: oidcClientSecret - value: bar - - name: oidcScopes - value: openid - - name: oidcAudiences - value: pulsar - - name: oidcTokenCAPEM - value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_six/pulsar.yaml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_six/pulsar.yaml.tmpl deleted file mode 100644 index 67f4ff56d1..0000000000 --- a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_six/pulsar.yaml.tmpl +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: messagebus -spec: - type: pubsub.pulsar - version: v1 - metadata: - - name: host - value: "localhost:6650" - - name: consumerID - value: certification5 - - name: redeliveryDelay - value: 200ms - - name: publicKey - value: "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1KDAM4L8RtJ+nLaXBrBh\nzVpvTemsKVZoAct8A+ShepOHT9lgHOCGLFGWNla6K6j+b3AV/P/fAAhwj82vwTDd\nruXSflvSdmYeFAw3Ypphc1A5oM53wSRWhg63potBNWqdDzj8ApYgqjpmjYSQdL5/\na3golb36GYFrY0MLFTv7wZ87pmMIPsOgGIcPbCHker2fRZ34WXYLb1hkeUpwx4eK\njpwcg35gccvR6o/UhbKAuc60V1J9Wof2sNgtlRaQej45wnpjWYzZrIyk5qUbn0Qi\nCdpIrXvYtANq0Id6gP8zJvUEdPIgNuYxEmVCl9jI+8eGI6peD0qIt8U80hf9axhJ\n3QIDAQAB\n-----END PUBLIC KEY-----\n" - - name: privateKey - value: "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA1KDAM4L8RtJ+nLaXBrBhzVpvTemsKVZoAct8A+ShepOHT9lg\nHOCGLFGWNla6K6j+b3AV/P/fAAhwj82vwTDdruXSflvSdmYeFAw3Ypphc1A5oM53\nwSRWhg63potBNWqdDzj8ApYgqjpmjYSQdL5/a3golb36GYFrY0MLFTv7wZ87pmMI\nPsOgGIcPbCHker2fRZ34WXYLb1hkeUpwx4eKjpwcg35gccvR6o/UhbKAuc60V1J9\nWof2sNgtlRaQej45wnpjWYzZrIyk5qUbn0QiCdpIrXvYtANq0Id6gP8zJvUEdPIg\nNuYxEmVCl9jI+8eGI6peD0qIt8U80hf9axhJ3QIDAQABAoIBAQCKuHnM4ac/eXM7\nQPDVX1vfgyHc3hgBPCtNCHnXfGFRvFBqavKGxIElBvGOcBS0CWQ+Rg1Ca5kMx3TQ\njSweSYhH5A7pe3Sa5FK5V6MGxJvRhMSkQi/lJZUBjzaIBJA9jln7pXzdHx8ekE16\nBMPONr6g2dr4nuI9o67xKrtfViwRDGaG6eh7jIMlEqMMc6WqyhvI67rlVDSTHFKX\njlMcozJ3IT8BtTzKg2Tpy7ReVuJEpehum8yn1ZVdAnotBDJxI07DC1cbOP4M2fHM\ngfgPYWmchauZuTeTFu4hrlY5jg0/WLs6by8r/81+vX3QTNvejX9UdTHMSIfQdX82\nAfkCKUVhAoGBAOvGv+YXeTlPRcYC642x5iOyLQm+BiSX4jKtnyJiTU2s/qvvKkIu\nxAOk3OtniT9NaUAHEZE9tI71dDN6IgTLQlAcPCzkVh6Sc5eG0MObqOO7WOMCWBkI\nlaAKKBbd6cGDJkwGCJKnx0pxC9f8R4dw3fmXWgWAr8ENiekMuvjSfjZ5AoGBAObd\ns2L5uiUPTtpyh8WZ7rEvrun3djBhzi+d7rgxEGdditeiLQGKyZbDPMSMBuus/5wH\nwfi0xUq50RtYDbzQQdC3T/C20oHmZbjWK5mDaLRVzWS89YG/NT2Q8eZLBstKqxkx\ngoT77zoUDfRy+CWs1xvXzgxagD5Yg8/OrCuXOqWFAoGAPIw3r6ELknoXEvihASxU\nS4pwInZYIYGXpygLG8teyrnIVOMAWSqlT8JAsXtPNaBtjPHDwyazfZrvEmEk51JD\nX0tA8M5ah1NYt+r5JaKNxp3P/8wUT6lyszyoeubWJsnFRfSusuq/NRC+1+KDg/aq\nKnSBu7QGbm9JoT2RrmBv5RECgYBRn8Lj1I1muvHTNDkiuRj2VniOSirkUkA2/6y+\nPMKi+SS0tqcY63v4rNCYYTW1L7Yz8V44U5mJoQb4lvpMbolGhPljjxAAU3hVkItb\nvGVRlSCIZHKczADD4rJUDOS7DYxO3P1bjUN4kkyYx+lKUMDBHFzCa2D6Kgt4dobS\n5qYajQKBgQC7u7MFPkkEMqNqNGu5erytQkBq1v1Ipmf9rCi3iIj4XJLopxMgw0fx\n6jwcwNInl72KzoUBLnGQ9PKGVeBcgEgdI+a+tq+1TJo6Ta+hZSx+4AYiKY18eRKG\neNuER9NOcSVJ7Eqkcw4viCGyYDm2vgNV9HJ0VlAo3RDh8x5spEN+mg==\n-----END RSA PRIVATE KEY-----\n" - - name: keys - value: myapp.key - - name: authType - value: oidc - - name: oidcTokenURL - value: https://localhost:8085/issuer1/token - - name: oidcClientID - value: foo - - name: oidcClientSecret - value: bar - - name: oidcScopes - value: openid - - name: oidcAudiences - value: pulsar - - name: oidcTokenCAPEM - value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_three/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_three/pulsar.yml.tmpl deleted file mode 100644 index d1dc3e1095..0000000000 --- a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_three/pulsar.yml.tmpl +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: messagebus -spec: - type: pubsub.pulsar - version: v1 - metadata: - - name: host - value: "localhost:6650" - - name: consumerID - value: certification3 - - name: redeliveryDelay - value: 200ms - - name: authType - value: oidc - - name: oidcTokenURL - value: https://localhost:8085/issuer1/token - - name: oidcClientID - value: foo - - name: oidcClientSecret - value: bar - - name: oidcScopes - value: openid - - name: oidcAudiences - value: pulsar - - name: oidcTokenCAPEM - value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_two/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_two/pulsar.yml.tmpl deleted file mode 100644 index 2b4834f17c..0000000000 --- a/tests/certification/pubsub/pulsar/components/auth-oidc/consumer_two/pulsar.yml.tmpl +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: messagebus -spec: - type: pubsub.pulsar - version: v1 - metadata: - - name: host - value: "localhost:6650" - - name: consumerID - value: certification2 - - name: redeliveryDelay - value: 200ms - - name: authType - value: oidc - - name: oidcTokenURL - value: https://localhost:8085/issuer1/token - - name: oidcClientID - value: foo - - name: oidcClientSecret - value: bar - - name: oidcScopes - value: openid - - name: oidcAudiences - value: pulsar - - name: oidcTokenCAPEM - value: "{{ .OIDCCAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-none/consumer_five/pulsar.yaml b/tests/certification/pubsub/pulsar/components/consumer_five/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/auth-none/consumer_five/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/consumer_five/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/auth-none/consumer_four/pulsar.yaml b/tests/certification/pubsub/pulsar/components/consumer_four/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/auth-none/consumer_four/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/consumer_four/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/auth-none/consumer_one/pulsar.yml b/tests/certification/pubsub/pulsar/components/consumer_one/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/auth-none/consumer_one/pulsar.yml rename to tests/certification/pubsub/pulsar/components/consumer_one/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/components/auth-none/consumer_six/pulsar.yaml b/tests/certification/pubsub/pulsar/components/consumer_six/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/auth-none/consumer_six/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/consumer_six/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/auth-none/consumer_three/pulsar.yml b/tests/certification/pubsub/pulsar/components/consumer_three/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/auth-none/consumer_three/pulsar.yml rename to tests/certification/pubsub/pulsar/components/consumer_three/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/components/auth-none/consumer_two/pulsar.yml b/tests/certification/pubsub/pulsar/components/consumer_two/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/auth-none/consumer_two/pulsar.yml rename to tests/certification/pubsub/pulsar/components/consumer_two/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/config/.gitignore b/tests/certification/pubsub/pulsar/config/.gitignore deleted file mode 100644 index 3af0ccb687..0000000000 --- a/tests/certification/pubsub/pulsar/config/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/data diff --git a/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oidc-server.yaml b/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oidc-server.yaml deleted file mode 100644 index 220011d949..0000000000 --- a/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oidc-server.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# We run in network_mode: "host" so `localhost` is the same for both the host -# and containers. This is required as the mock server uses the SNI hostname to -# build the issuer URL. -version: '3' -services: - mock-oauth2-server: - image: ghcr.io/navikt/mock-oauth2-server:1.0.0 - container_name: mock-oauth2-server - restart: on-failure - network_mode: "host" - environment: - - PORT=8085 - - LOG_LEVEL=DEBUG - - 'JSON_CONFIG={"interactiveLogin":false,"httpServer":{"type":"NettyWrapper","ssl":{}},"tokenCallbacks":[{"issuerId":"issuer1","tokenExpiry":120,"requestMappings":[{"requestParam":"scope","match":"openid","claims":{"sub":"foo","aud":["pulsar"]}}]}]}' diff --git a/tests/certification/pubsub/pulsar/config/docker-compose_auth-oidc.yaml.tmpl b/tests/certification/pubsub/pulsar/config/docker-compose_auth-oidc.yaml.tmpl deleted file mode 100644 index 256ba89677..0000000000 --- a/tests/certification/pubsub/pulsar/config/docker-compose_auth-oidc.yaml.tmpl +++ /dev/null @@ -1,96 +0,0 @@ -# We run the pulsar services individually as OIDC doesn't seem to work in -# standalone mode. OIDC is also only available from pulsar v3 onwards. We use -# host networking as the mock OAuth server uses the SNI host name to determine -# the host name of the OIDC issuer URL, so we need to have the mock server -# reachable by localhost from both the pulsar services and the host network. -version: '3' -services: - # Start zookeeper - zookeeper: - image: apachepulsar/pulsar:3.0.0 - container_name: zookeeper - restart: on-failure - network_mode: "host" - environment: - - metadataStoreUrl=zk:localhost:2181 - - metricsProvider.httpPort=7000 - - PULSAR_MEM=-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m - command: > - bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \ - bin/generate-zookeeper-config.sh conf/zookeeper.conf && \ - exec bin/pulsar zookeeper" - healthcheck: - test: ["CMD", "bin/pulsar-zookeeper-ruok.sh"] - interval: 1s - timeout: 5s - retries: 300 - - # Init cluster metadata - pulsar-init: - container_name: pulsar-init - image: apachepulsar/pulsar:3.0.0 - network_mode: "host" - env_file: - - ./pulsar_auth-oidc.conf - command: > - bin/pulsar initialize-cluster-metadata \ - --cluster cluster-a \ - --zookeeper localhost:2181 \ - --configuration-store localhost:2181 \ - --web-service-url http://localhost:8080 \ - --broker-service-url pulsar://localhost:6650 - depends_on: - zookeeper: - condition: service_healthy - - # Start bookie - bookie: - image: apachepulsar/pulsar:3.0.0 - container_name: bookie - restart: on-failure - network_mode: "host" - environment: - - clusterName=cluster-a - - zkServers=localhost:2181 - - metadataServiceUri=metadata-store:zk:localhost:2181 - # otherwise every time we run docker compose uo or down we fail to start due to Cookie - # See: https://github.com/apache/bookkeeper/blob/405e72acf42bb1104296447ea8840d805094c787/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Cookie.java#L57-68 - - advertisedAddress=localhost - - BOOKIE_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m - env_file: - - ./pulsar_auth-oidc.conf - volumes: - - "{{ .TmpDir }}:/pulsar/conf/dapr" - depends_on: - zookeeper: - condition: service_healthy - pulsar-init: - condition: service_completed_successfully - command: bash -c "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" - - # Start broker - broker: - image: apachepulsar/pulsar:3.0.0 - container_name: broker - restart: on-failure - network_mode: "host" - env_file: - - ./pulsar_auth-oidc.conf - volumes: - - "{{ .TmpDir }}:/pulsar/conf/dapr" - environment: - - metadataStoreUrl=zk:localhost:2181 - - zookeeperServers=localhost:2181 - - clusterName=cluster-a - - managedLedgerDefaultEnsembleSize=1 - - managedLedgerDefaultWriteQuorum=1 - - managedLedgerDefaultAckQuorum=1 - - advertisedAddress=localhost - - advertisedListeners=external:pulsar://127.0.0.1:6650 - - PULSAR_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m - depends_on: - zookeeper: - condition: service_healthy - bookie: - condition: service_started - command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" diff --git a/tests/certification/pubsub/pulsar/config/pulsar_auth-oidc.conf b/tests/certification/pubsub/pulsar/config/pulsar_auth-oidc.conf deleted file mode 100644 index 1bef86ea45..0000000000 --- a/tests/certification/pubsub/pulsar/config/pulsar_auth-oidc.conf +++ /dev/null @@ -1,39 +0,0 @@ -# Configuration to enable authentication -authenticationEnabled=true -authenticationProviders=org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID - -# Required settings for AuthenticationProviderOpenID -# A comma separated list of allowed, or trusted, token issuers. The token issuer is the URL of the token issuer. -PULSAR_PREFIX_openIDAllowedTokenIssuers=https://localhost:8085/issuer1 -# The list of allowed audiences for the token. The audience is the intended recipient of the token. A token with -# at least one of these audience claims will pass the audience validation check. -PULSAR_PREFIX_openIDAllowedAudiences=pulsar - -# Optional settings (values shown are the defaults) -# The path to the file containing the trusted certificate(s) of the token issuer(s). If not set, uses the default -# trust store of the JVM. Note: in version 3.0.0, the default only applies when this setting is not an environment -# variable and is not in the configuration file. -PULSAR_PREFIX_openIDTokenIssuerTrustCertsFilePath=/pulsar/conf/dapr/ca.pem -# The JWT's claim to use for the role/principal during authorization. -PULSAR_PREFIX_openIDRoleClaim=sub -# The leeway, in seconds, to use when validating the token's expiration time. -PULSAR_PREFIX_openIDAcceptedTimeLeewaySeconds=0 - -# Cache settings -PULSAR_PREFIX_openIDCacheSize=5 -PULSAR_PREFIX_openIDCacheRefreshAfterWriteSeconds=64800 -PULSAR_PREFIX_openIDCacheExpirationSeconds=86400 -PULSAR_PREFIX_openIDHttpConnectionTimeoutMillis=10000 -PULSAR_PREFIX_openIDHttpReadTimeoutMillis=10000 - -# The number of seconds to wait before refreshing the JWKS when a token presents a key ID (kid claim) that is not -# in the cache. This setting is available from Pulsar 3.0.1 and is documented below. -PULSAR_PREFIX_openIDKeyIdCacheMissRefreshSeconds=300 - -# Whether to require that issuers use HTTPS. It is part of the OIDC spec to use HTTPS, so the default is true. -# This setting is for testing purposes and is not recommended for any production environment. -#PULSAR_PREFIX_openIDRequireIssuersUseHttps=false - -# A setting describing how to handle discovery of the OpenID Connect configuration document when the issuer is not -# in the list of allowed issuers. This setting is documented below. -PULSAR_PREFIX_openIDFallbackDiscoveryMode=DISABLED diff --git a/tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml b/tests/certification/pubsub/pulsar/docker-compose.yml similarity index 97% rename from tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml rename to tests/certification/pubsub/pulsar/docker-compose.yml index 7857b77961..5ae0535596 100644 --- a/tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml +++ b/tests/certification/pubsub/pulsar/docker-compose.yml @@ -16,4 +16,4 @@ services: - pulsarconf:/pulsar/conf volumes: pulsardata: - pulsarconf: + pulsarconf: \ No newline at end of file diff --git a/tests/certification/pubsub/pulsar/pulsar_test.go b/tests/certification/pubsub/pulsar/pulsar_test.go index 8d6f1cd80c..9c7fd3e405 100644 --- a/tests/certification/pubsub/pulsar/pulsar_test.go +++ b/tests/certification/pubsub/pulsar/pulsar_test.go @@ -16,28 +16,17 @@ package pulsar_test import ( "bytes" "context" - "crypto/tls" "encoding/json" - "encoding/pem" "fmt" - "io" - "io/fs" "io/ioutil" "net/http" - "os" - "os/exec" - "path/filepath" - "strings" "testing" - "text/template" "time" "github.com/google/uuid" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" "go.uber.org/multierr" - "github.com/dapr/components-contrib/internal/authentication/oidc" pubsub_pulsar "github.com/dapr/components-contrib/pubsub/pulsar" pubsub_loader "github.com/dapr/dapr/pkg/components/pubsub" @@ -65,23 +54,21 @@ const ( appID1 = "app-1" appID2 = "app-2" - numMessages = 10 - appPort = 8001 - portOffset = 2 - messageKey = "partitionKey" - pubsubName = "messagebus" - topicActiveName = "certification-pubsub-topic-active" - topicPassiveName = "certification-pubsub-topic-passive" - topicToBeCreated = "certification-topic-per-test-run" - topicDefaultName = "certification-topic-default" - topicMultiPartitionName = "certification-topic-multi-partition8" - partition0 = "partition-0" - partition1 = "partition-1" - clusterName = "pulsarcertification" - dockerComposeAuthNoneYAML = "./config/docker-compose_auth-none.yaml" - dockerComposeAuthOIDCYAML = "./config/docker-compose_auth-oidc.yaml.tmpl" - dockerComposeMockOAuthYAML = "./config/docker-compose_auth-mock-oidc-server.yaml" - pulsarURL = "localhost:6650" + numMessages = 10 + appPort = 8000 + portOffset = 2 + messageKey = "partitionKey" + pubsubName = "messagebus" + topicActiveName = "certification-pubsub-topic-active" + topicPassiveName = "certification-pubsub-topic-passive" + topicToBeCreated = "certification-topic-per-test-run" + topicDefaultName = "certification-topic-default" + topicMultiPartitionName = "certification-topic-multi-partition8" + partition0 = "partition-0" + partition1 = "partition-1" + clusterName = "pulsarcertification" + dockerComposeYAML = "docker-compose.yml" + pulsarURL = "localhost:6650" subscribeTypeKey = "subscribeType" @@ -95,103 +82,6 @@ const ( processModeSync = "sync" ) -type pulsarSuite struct { - suite.Suite - - authType string - oidcCAPEM []byte - dockerComposeYAML string - componentsPath string - services []string -} - -func TestPulsar(t *testing.T) { - t.Run("Auth:None", func(t *testing.T) { - suite.Run(t, &pulsarSuite{ - authType: "none", - dockerComposeYAML: dockerComposeAuthNoneYAML, - componentsPath: "./components/auth-none", - services: []string{"standalone"}, - }) - }) - - t.Run("Auth:OIDC", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.Chmod(dir, 0o777)) - - t.Log("Starting OIDC server...") - out, err := exec.Command( - "docker-compose", - "-p", "oidc", - "-f", dockerComposeMockOAuthYAML, - "up", "-d").CombinedOutput() - require.NoError(t, err, string(out)) - t.Log(string(out)) - - t.Cleanup(func() { - t.Log("Stopping OIDC server...") - out, err = exec.Command( - "docker-compose", - "-p", "oidc", - "-f", dockerComposeMockOAuthYAML, - "down", "-v", - "--remove-orphans").CombinedOutput() - require.NoError(t, err, string(out)) - t.Log(string(out)) - }) - - t.Log("Waiting for OAuth server to be ready...") - oauthCA := peerCertificate(t, "localhost:8085") - t.Log("OAuth server is ready") - - require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.pem"), oauthCA, 0o644)) - outf, err := os.OpenFile("./config/pulsar_auth-oidc.conf", os.O_RDONLY, 0o644) - require.NoError(t, err) - inf, err := os.OpenFile(filepath.Join(dir, "pulsar_auth-oidc.conf"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) - require.NoError(t, err) - _, err = io.Copy(inf, outf) - require.NoError(t, err) - outf.Close() - inf.Close() - - td := struct { - TmpDir string - OIDCCAPEM string - }{ - TmpDir: dir, - OIDCCAPEM: strings.ReplaceAll(string(oauthCA), "\n", "\\n"), - } - - tmpl, err := template.New("").ParseFiles(dockerComposeAuthOIDCYAML) - require.NoError(t, err) - f, err := os.OpenFile(filepath.Join(dir, "docker-compose.yaml"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) - require.NoError(t, err) - require.NoError(t, tmpl.ExecuteTemplate(f, "docker-compose_auth-oidc.yaml.tmpl", td)) - - require.NoError(t, filepath.Walk("./components/auth-oidc", func(path string, info fs.FileInfo, err error) error { - if info.IsDir() { - return nil - } - tmpl, err := template.New("").ParseFiles(path) - require.NoError(t, err) - path = strings.TrimSuffix(path, ".tmpl") - require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, path)), 0o755)) - f, err := os.OpenFile(filepath.Join(dir, path), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) - require.NoError(t, err) - require.NoError(t, tmpl.ExecuteTemplate(f, filepath.Base(path)+".tmpl", td)) - return nil - })) - - suite.Run(t, &pulsarSuite{ - oidcCAPEM: oauthCA, - authType: "oidc", - dockerComposeYAML: filepath.Join(dir, "docker-compose.yaml"), - componentsPath: filepath.Join(dir, "components/auth-oidc"), - services: []string{"zookeeper", "pulsar-init", "bookie", "broker"}, - }) - }) -} - func subscriberApplication(appID string, topicName string, messagesWatcher *watcher.Watcher) app.SetupFn { return func(ctx flow.Context, s common.Service) error { // Simulate periodic errors. @@ -306,8 +196,7 @@ func assertMessages(timeout time.Duration, messageWatchers ...*watcher.Watcher) } } -func (p *pulsarSuite) TestPulsar() { - t := p.T() +func TestPulsar(t *testing.T) { consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -355,10 +244,10 @@ func (p *pulsarSuite) TestPulsar() { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -378,7 +267,7 @@ func (p *pulsarSuite) TestPulsar() { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), + embedded.WithComponentsPath("./components/consumer_one"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -391,7 +280,7 @@ func (p *pulsarSuite) TestPulsar() { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), + embedded.WithComponentsPath("./components/consumer_two"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -406,8 +295,7 @@ func (p *pulsarSuite) TestPulsar() { Run() } -func (p *pulsarSuite) TestPulsarMultipleSubsSameConsumerIDs() { - t := p.T() +func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -424,10 +312,10 @@ func (p *pulsarSuite) TestPulsarMultipleSubsSameConsumerIDs() { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -447,7 +335,7 @@ func (p *pulsarSuite) TestPulsarMultipleSubsSameConsumerIDs() { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), + embedded.WithComponentsPath("./components/consumer_one"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -460,7 +348,7 @@ func (p *pulsarSuite) TestPulsarMultipleSubsSameConsumerIDs() { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), + embedded.WithComponentsPath("./components/consumer_two"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -474,9 +362,7 @@ func (p *pulsarSuite) TestPulsarMultipleSubsSameConsumerIDs() { Run() } -func (p *pulsarSuite) TestPulsarMultipleSubsDifferentConsumerIDs() { - t := p.T() - +func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -490,10 +376,10 @@ func (p *pulsarSuite) TestPulsarMultipleSubsDifferentConsumerIDs() { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -515,7 +401,7 @@ func (p *pulsarSuite) TestPulsarMultipleSubsDifferentConsumerIDs() { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), + embedded.WithComponentsPath("./components/consumer_one"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -528,7 +414,7 @@ func (p *pulsarSuite) TestPulsarMultipleSubsDifferentConsumerIDs() { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), + embedded.WithComponentsPath("./components/consumer_two"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -541,8 +427,7 @@ func (p *pulsarSuite) TestPulsarMultipleSubsDifferentConsumerIDs() { Run() } -func (p *pulsarSuite) TestPulsarMultiplePubSubsDifferentConsumerIDs() { - t := p.T() +func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -560,10 +445,10 @@ func (p *pulsarSuite) TestPulsarMultiplePubSubsDifferentConsumerIDs() { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -585,7 +470,7 @@ func (p *pulsarSuite) TestPulsarMultiplePubSubsDifferentConsumerIDs() { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), + embedded.WithComponentsPath("./components/consumer_one"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -598,7 +483,7 @@ func (p *pulsarSuite) TestPulsarMultiplePubSubsDifferentConsumerIDs() { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), + embedded.WithComponentsPath("./components/consumer_two"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -613,8 +498,7 @@ func (p *pulsarSuite) TestPulsarMultiplePubSubsDifferentConsumerIDs() { Run() } -func (p *pulsarSuite) TestPulsarNonexistingTopic() { - t := p.T() +func TestPulsarNonexistingTopic(t *testing.T) { consumerGroup1 := watcher.NewUnordered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -627,10 +511,10 @@ func (p *pulsarSuite) TestPulsarNonexistingTopic() { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset*3), subscriberApplication(appID1, topicToBeCreated, consumerGroup1))). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -653,7 +537,7 @@ func (p *pulsarSuite) TestPulsarNonexistingTopic() { })). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), + embedded.WithComponentsPath("./components/consumer_one"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset*3), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*3), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*3), @@ -666,8 +550,7 @@ func (p *pulsarSuite) TestPulsarNonexistingTopic() { Run() } -func (p *pulsarSuite) TestPulsarNetworkInterruption() { - t := p.T() +func TestPulsarNetworkInterruption(t *testing.T) { consumerGroup1 := watcher.NewUnordered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -680,10 +563,10 @@ func (p *pulsarSuite) TestPulsarNetworkInterruption() { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -706,7 +589,7 @@ func (p *pulsarSuite) TestPulsarNetworkInterruption() { })). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), + embedded.WithComponentsPath("./components/consumer_one"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -720,8 +603,7 @@ func (p *pulsarSuite) TestPulsarNetworkInterruption() { Run() } -func (p *pulsarSuite) TestPulsarPersitant() { - t := p.T() +func TestPulsarPersitant(t *testing.T) { consumerGroup1 := watcher.NewUnordered() flow.New(t, "pulsar certification persistant test"). @@ -729,10 +611,10 @@ func (p *pulsarSuite) TestPulsarPersitant() { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -754,24 +636,23 @@ func (p *pulsarSuite) TestPulsarPersitant() { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), + embedded.WithComponentsPath("./components/consumer_one"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), componentRuntimeOptions(), )). Step("publish messages to topic1", publishMessages(nil, sidecarName1, topicActiveName, consumerGroup1)). - Step("stop pulsar server", dockercompose.Stop(clusterName, p.dockerComposeYAML, p.services...)). + Step("stop pulsar server", dockercompose.Stop(clusterName, dockerComposeYAML, "standalone")). Step("wait", flow.Sleep(5*time.Second)). - Step("start pulsar server", dockercompose.Start(clusterName, p.dockerComposeYAML, p.services...)). + Step("start pulsar server", dockercompose.Start(clusterName, dockerComposeYAML, "standalone")). Step("wait", flow.Sleep(10*time.Second)). Step("verify if app1 has received messages published to active topic", assertMessages(10*time.Second, consumerGroup1)). Step("reset", flow.Reset(consumerGroup1)). Run() } -func (p *pulsarSuite) TestPulsarDelay() { - t := p.T() +func TestPulsarDelay(t *testing.T) { consumerGroup1 := watcher.NewUnordered() date := time.Now() @@ -801,10 +682,10 @@ func (p *pulsarSuite) TestPulsarDelay() { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -826,7 +707,7 @@ func (p *pulsarSuite) TestPulsarDelay() { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_three")), + embedded.WithComponentsPath("./components/consumer_three"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -849,8 +730,7 @@ type schemaTest struct { Name string `json:"name"` } -func (p *pulsarSuite) TestPulsarSchema() { - t := p.T() +func TestPulsarSchema(t *testing.T) { consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -892,10 +772,10 @@ func (p *pulsarSuite) TestPulsarSchema() { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -915,7 +795,7 @@ func (p *pulsarSuite) TestPulsarSchema() { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_four")), + embedded.WithComponentsPath("./components/consumer_four"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -938,7 +818,7 @@ func componentRuntimeOptions() []runtime.Option { } } -func (p *pulsarSuite) createMultiPartitionTopic(tenant, namespace, topic string, partition int) flow.Runnable { +func createMultiPartitionTopic(tenant, namespace, topic string, partition int) flow.Runnable { return func(ctx flow.Context) error { reqURL := fmt.Sprintf("http://localhost:8080/admin/v2/persistent/%s/%s/%s/partitions", tenant, namespace, topic) @@ -958,19 +838,6 @@ func (p *pulsarSuite) createMultiPartitionTopic(tenant, namespace, topic string, req.Header.Set("Content-Type", "application/json") - if p.authType == "oidc" { - cc, err := p.oidcClientCredentials() - if err != nil { - return err - } - token, err := cc.Token() - if err != nil { - return err - } - - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - } - rsp, err := http.DefaultClient.Do(req) if err != nil { @@ -991,8 +858,7 @@ func (p *pulsarSuite) createMultiPartitionTopic(tenant, namespace, topic string, } } -func (p *pulsarSuite) TestPulsarPartitionedOrderingProcess() { - t := p.T() +func TestPulsarPartitionedOrderingProcess(t *testing.T) { consumerGroup1 := watcher.NewOrdered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -1001,14 +867,14 @@ func (p *pulsarSuite) TestPulsarPartitionedOrderingProcess() { } flow.New(t, "pulsar certification - process message in order with partitioned-topic"). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset), subscriberApplicationWithoutError(appID1, topicMultiPartitionName, consumerGroup1))). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -1031,10 +897,10 @@ func (p *pulsarSuite) TestPulsarPartitionedOrderingProcess() { return err })). Step("create multi-partition topic explicitly", retry.Do(10*time.Second, 30, - p.createMultiPartitionTopic("public", "default", topicMultiPartitionName, 4))). + createMultiPartitionTopic("public", "default", topicMultiPartitionName, 4))). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), + embedded.WithComponentsPath("./components/consumer_one"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -1047,7 +913,7 @@ func (p *pulsarSuite) TestPulsarPartitionedOrderingProcess() { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), + embedded.WithComponentsPath("./components/consumer_two"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset*3), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*3), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*3), @@ -1061,8 +927,7 @@ func (p *pulsarSuite) TestPulsarPartitionedOrderingProcess() { Run() } -func (p *pulsarSuite) TestPulsarEncryptionFromFile() { - t := p.T() +func TestPulsarEncryptionFromFile(t *testing.T) { consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -1104,10 +969,10 @@ func (p *pulsarSuite) TestPulsarEncryptionFromFile() { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -1127,7 +992,7 @@ func (p *pulsarSuite) TestPulsarEncryptionFromFile() { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_five")), + embedded.WithComponentsPath("./components/consumer_five"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -1139,8 +1004,7 @@ func (p *pulsarSuite) TestPulsarEncryptionFromFile() { Run() } -func (p *pulsarSuite) TestPulsarEncryptionFromData() { - t := p.T() +func TestPulsarEncryptionFromData(t *testing.T) { consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -1182,10 +1046,10 @@ func (p *pulsarSuite) TestPulsarEncryptionFromData() { // Run subscriberApplication app2 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). + Step(dockercompose.Run(clusterName, dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := p.client(t) + client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -1205,7 +1069,7 @@ func (p *pulsarSuite) TestPulsarEncryptionFromData() { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_six")), + embedded.WithComponentsPath("./components/consumer_six"), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -1216,57 +1080,3 @@ func (p *pulsarSuite) TestPulsarEncryptionFromData() { Step("reset", flow.Reset(consumerGroup1)). Run() } - -func (p *pulsarSuite) client(t *testing.T) (pulsar.Client, error) { - t.Helper() - - opts := pulsar.ClientOptions{ - URL: "pulsar://localhost:6650", - } - switch p.authType { - case "oidc": - cc, err := p.oidcClientCredentials() - require.NoError(t, err) - opts.Authentication = pulsar.NewAuthenticationTokenFromSupplier(cc.Token) - default: - } - - return pulsar.NewClient(opts) -} - -func (p *pulsarSuite) oidcClientCredentials() (*oidc.ClientCredentials, error) { - cc, err := oidc.NewClientCredentials(context.Background(), oidc.ClientCredentialsOptions{ - Logger: logger.NewLogger("dapr.test.readiness"), - TokenURL: "https://localhost:8085/issuer1/token", - ClientID: "foo", - ClientSecret: "bar", - Scopes: []string{"openid"}, - Audiences: []string{"pulsar"}, - CAPEM: p.oidcCAPEM, - }) - if err != nil { - return nil, err - } - - return cc, nil -} - -func peerCertificate(t *testing.T, hostport string) []byte { - conf := &tls.Config{InsecureSkipVerify: true} - - for { - time.Sleep(1 * time.Second) - - conn, err := tls.Dial("tcp", hostport, conf) - if err != nil { - t.Log(err) - continue - } - - defer conn.Close() - - certs := conn.ConnectionState().PeerCertificates - require.Len(t, certs, 1, "expected 1 peer certificate") - return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certs[0].Raw}) - } -} From 25656c175d4d568bbbe9a78b290129c5a990e9f9 Mon Sep 17 00:00:00 2001 From: Yash Nisar Date: Tue, 1 Aug 2023 19:28:19 -0500 Subject: [PATCH 09/22] Add metadata for dequeued message in bindings.azure.storagequeues (#3028) Signed-off-by: Yash Nisar --- bindings/azure/storagequeues/storagequeues.go | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/bindings/azure/storagequeues/storagequeues.go b/bindings/azure/storagequeues/storagequeues.go index babfb2feab..d637d16685 100644 --- a/bindings/azure/storagequeues/storagequeues.go +++ b/bindings/azure/storagequeues/storagequeues.go @@ -38,6 +38,12 @@ const ( defaultTTL = 10 * time.Minute defaultVisibilityTimeout = 30 * time.Second defaultPollingInterval = 10 * time.Second + dequeueCount = "dequeueCount" + insertionTime = "insertionTime" + expirationTime = "expirationTime" + nextVisibleTime = "nextVisibleTime" + popReceipt = "popReceipt" + messageID = "messageID" ) type consumer struct { @@ -177,9 +183,30 @@ func (d *AzureQueueHelper) Read(ctx context.Context, consumer *consumer) error { } } + metadata := make(map[string]string, 6) + + if res.Messages[0].MessageID != nil { + metadata[messageID] = *res.Messages[0].MessageID + } + if res.Messages[0].PopReceipt != nil { + metadata[popReceipt] = *res.Messages[0].PopReceipt + } + if res.Messages[0].InsertionTime != nil { + metadata[insertionTime] = res.Messages[0].InsertionTime.Format(time.RFC3339) + } + if res.Messages[0].ExpirationTime != nil { + metadata[expirationTime] = res.Messages[0].ExpirationTime.Format(time.RFC3339) + } + if res.Messages[0].TimeNextVisible != nil { + metadata[nextVisibleTime] = res.Messages[0].TimeNextVisible.Format(time.RFC3339) + } + if res.Messages[0].DequeueCount != nil { + metadata[dequeueCount] = strconv.FormatInt(*res.Messages[0].DequeueCount, 10) + } + _, err = consumer.callback(ctx, &bindings.ReadResponse{ Data: data, - Metadata: map[string]string{}, + Metadata: metadata, }) if err != nil { return err From c2dbb03069d92ee666011ed638c6fe72d6439753 Mon Sep 17 00:00:00 2001 From: Roberto Rojas Date: Wed, 2 Aug 2023 13:59:27 -0400 Subject: [PATCH 10/22] [AWS SSM Parameter SecretStore] Adds Component Metadata Schema (#2938) Signed-off-by: Roberto J Rojas Signed-off-by: Roberto Rojas --- secretstores/aws/parameterstore/metadata.yaml | 34 +++++++++++++++++++ .../aws/parameterstore/parameterstore.go | 4 +-- 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 secretstores/aws/parameterstore/metadata.yaml diff --git a/secretstores/aws/parameterstore/metadata.yaml b/secretstores/aws/parameterstore/metadata.yaml new file mode 100644 index 0000000000..e8b40d6441 --- /dev/null +++ b/secretstores/aws/parameterstore/metadata.yaml @@ -0,0 +1,34 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: secretstores +name: aws.parameterstore +version: v1 +status: alpha +title: "AWS SSM Parameter Store" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-secret-stores/aws-parameter-store/ +builtinAuthenticationProfiles: + - name: "aws" +metadata: + - name: region + required: true + description: | + The specific AWS region the AWS SSM Parameter Store instance is deployed in. + example: '"us-east-1"' + type: string + - name: sessionToken + required: false + sensitive: true + description: | + AWS session token to use. A session token is only required if you are using + temporary security credentials. + example: '"TOKEN"' + type: string + - name: prefix + required: false + description: | + The SSM Parameter Store prefix to be specified. If specified, it will be + used as the 'BeginsWith' as part of the 'ParameterStringFilter'. + example: '"myprefix"' + type: string \ No newline at end of file diff --git a/secretstores/aws/parameterstore/parameterstore.go b/secretstores/aws/parameterstore/parameterstore.go index fef4294f32..5a62804b01 100644 --- a/secretstores/aws/parameterstore/parameterstore.go +++ b/secretstores/aws/parameterstore/parameterstore.go @@ -42,8 +42,8 @@ func NewParameterStore(logger logger.Logger) secretstores.SecretStore { type ParameterStoreMetaData struct { Region string `json:"region"` - AccessKey string `json:"accessKey"` - SecretKey string `json:"secretKey"` + AccessKey string `json:"accessKey" mapstructure:"accessKey" mdignore:"true"` + SecretKey string `json:"secretKey" mapstructure:"secretKey" mdignore:"true"` SessionToken string `json:"sessionToken"` Prefix string `json:"prefix"` } From 60322a1f1c1c5cbadbd8f7b7e00baa9ce5c5681a Mon Sep 17 00:00:00 2001 From: Roberto Rojas Date: Wed, 2 Aug 2023 16:50:40 -0400 Subject: [PATCH 11/22] [AWS State DynamoDB] Adds Component Metadata Schema (#2906) Signed-off-by: Roberto J Rojas Signed-off-by: Roberto Rojas Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Artur Souza --- state/aws/dynamodb/dynamodb.go | 8 +++-- state/aws/dynamodb/metadata.yaml | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 state/aws/dynamodb/metadata.yaml diff --git a/state/aws/dynamodb/dynamodb.go b/state/aws/dynamodb/dynamodb.go index ba7c3ab96b..33592d5706 100644 --- a/state/aws/dynamodb/dynamodb.go +++ b/state/aws/dynamodb/dynamodb.go @@ -46,11 +46,13 @@ type StateStore struct { } type dynamoDBMetadata struct { + // Ignored by metadata parser because included in built-in authentication profile + AccessKey string `json:"accessKey" mapstructure:"accessKey" mdignore:"true"` + SecretKey string `json:"secretKey" mapstructure:"secretKey" mdignore:"true"` + SessionToken string `json:"sessionToken" mapstructure:"sessionToken" mdignore:"true"` + Region string `json:"region"` Endpoint string `json:"endpoint"` - AccessKey string `json:"accessKey"` - SecretKey string `json:"secretKey"` - SessionToken string `json:"sessionToken"` Table string `json:"table"` TTLAttributeName string `json:"ttlAttributeName"` PartitionKey string `json:"partitionKey"` diff --git a/state/aws/dynamodb/metadata.yaml b/state/aws/dynamodb/metadata.yaml new file mode 100644 index 0000000000..ed0c9a3fbc --- /dev/null +++ b/state/aws/dynamodb/metadata.yaml @@ -0,0 +1,59 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: aws.dynamodb +version: v1 +status: stable +title: "AWS DynamoDB" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-dynamodb/ +capabilities: + - crud + - transactional + - etag + - ttl + - actorStateStore +builtinAuthenticationProfiles: + - name: "aws" +metadata: + - name: table + required: true + description: | + The name of the DynamoDB table to use. + example: '"Contracts"' + type: string + - name: region + required: false + description: | + The AWS region to use. Ensure that DynamoDB is available in that region. + See the `Amazon DynamoDB endpoints and quotas` documentation. + url: + title: Amazon DynamoDB endpoints and quotas + url: https://docs.aws.amazon.com/general/latest/gr/ddb.html + example: '"us-east-1"' + type: string + - name: endpoint + required: false + description: | + AWS endpoint for the component to use. Only used for local development. + The endpoint is not necessary when running against production AWS. + example: '"http://localhost:4566"' + type: string + - name: ttlAttributeName + required: false + description: | + The table attribute name which should be used for TTL. + example: '"expiresAt"' + type: string + - name: partitionKey + required: false + description: | + The table primary key or partition key attribute name. + This field is used to replace the default primary key attribute name "key". + url: + title: More details + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-dynamodb/#partition-keys + example: '"ContractID"' + type: string + \ No newline at end of file From 566c7fd31adbd81740ba66995729a7eadeb3444a Mon Sep 17 00:00:00 2001 From: Robert Oh <36242112+robert-oh@users.noreply.github.com> Date: Wed, 2 Aug 2023 23:27:46 +0200 Subject: [PATCH 12/22] use rabbitmq message's header values as metadata values in the binding (#3031) Signed-off-by: Ohlicher Robert Signed-off-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Ohlicher Robert Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> --- bindings/rabbitmq/rabbitmq.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bindings/rabbitmq/rabbitmq.go b/bindings/rabbitmq/rabbitmq.go index e10f7422f3..4188851342 100644 --- a/bindings/rabbitmq/rabbitmq.go +++ b/bindings/rabbitmq/rabbitmq.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "math" + "net/url" "reflect" "strconv" "sync" @@ -464,8 +465,20 @@ func (r *RabbitMQ) handleMessage(ctx context.Context, handler bindings.Handler, r.logger.Info("Input binding channel closed") return } + + metadata := make(map[string]string, len(d.Headers)) + // Passthrough any custom metadata to the handler. + for k, v := range d.Headers { + if s, ok := v.(string); ok { + // Escape the key and value to ensure they are valid URL query parameters. + // This is necessary for them to be sent as HTTP Metadata. + metadata[url.QueryEscape(k)] = url.QueryEscape(s) + } + } + _, err := handler(ctx, &bindings.ReadResponse{ - Data: d.Body, + Data: d.Body, + Metadata: metadata, }) if err != nil { ch.Nack(d.DeliveryTag, false, true) From 7937d34bb7b43e1a069410cc12827afa4bb94340 Mon Sep 17 00:00:00 2001 From: Josh van Leeuwen Date: Thu, 3 Aug 2023 01:17:55 +0100 Subject: [PATCH 13/22] Pubsub pulsar authentication ~OIDC~ OAuth2 (#3026) Signed-off-by: joshvanl Signed-off-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Yaron Schneider Co-authored-by: Bernd Verst --- .github/workflows/certification.yml | 2 +- .../oauth2/clientcredentials.go | 165 +++++++++ .../oauth2/clientcredentials_test.go | 95 +++++ pubsub/pulsar/metadata.go | 10 +- pubsub/pulsar/pulsar.go | 27 +- .../{ => auth-none}/consumer_five/pulsar.yaml | 0 .../{ => auth-none}/consumer_four/pulsar.yaml | 0 .../{ => auth-none}/consumer_one/pulsar.yml | 0 .../{ => auth-none}/consumer_six/pulsar.yaml | 0 .../{ => auth-none}/consumer_three/pulsar.yml | 0 .../{ => auth-none}/consumer_two/pulsar.yml | 0 .../auth-oauth2/consumer_five/pulsar.yml.tmpl | 32 ++ .../auth-oauth2/consumer_four/pulsar.yml.tmpl | 28 ++ .../auth-oauth2/consumer_one/pulsar.yml.tmpl | 26 ++ .../auth-oauth2/consumer_six/pulsar.yaml.tmpl | 32 ++ .../consumer_three/pulsar.yml.tmpl | 26 ++ .../auth-oauth2/consumer_two/pulsar.yml.tmpl | 26 ++ ...ocker-compose_auth-mock-oauth2-server.yaml | 14 + .../docker-compose_auth-none.yaml} | 2 +- .../docker-compose_auth-oauth2.yaml.tmpl | 96 +++++ .../pulsar/config/pulsar_auth-oauth2.conf | 39 ++ .../pubsub/pulsar/pulsar_test.go | 334 ++++++++++++++---- 22 files changed, 874 insertions(+), 80 deletions(-) create mode 100644 internal/authentication/oauth2/clientcredentials.go create mode 100644 internal/authentication/oauth2/clientcredentials_test.go rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_five/pulsar.yaml (100%) rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_four/pulsar.yaml (100%) rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_one/pulsar.yml (100%) rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_six/pulsar.yaml (100%) rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_three/pulsar.yml (100%) rename tests/certification/pubsub/pulsar/components/{ => auth-none}/consumer_two/pulsar.yml (100%) create mode 100644 tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_five/pulsar.yml.tmpl create mode 100644 tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_four/pulsar.yml.tmpl create mode 100644 tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_one/pulsar.yml.tmpl create mode 100644 tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_six/pulsar.yaml.tmpl create mode 100644 tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_three/pulsar.yml.tmpl create mode 100644 tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_two/pulsar.yml.tmpl create mode 100644 tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oauth2-server.yaml rename tests/certification/pubsub/pulsar/{docker-compose.yml => config/docker-compose_auth-none.yaml} (97%) create mode 100644 tests/certification/pubsub/pulsar/config/docker-compose_auth-oauth2.yaml.tmpl create mode 100644 tests/certification/pubsub/pulsar/config/pulsar_auth-oauth2.conf diff --git a/.github/workflows/certification.yml b/.github/workflows/certification.yml index dfa2fe39ba..5482b986a1 100644 --- a/.github/workflows/certification.yml +++ b/.github/workflows/certification.yml @@ -258,7 +258,7 @@ jobs: set +e gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json \ --junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.xml --format standard-quiet -- \ - -coverprofile=cover.out -covermode=set -tags=certtests -coverpkg=${{ matrix.source-pkg }} + -coverprofile=cover.out -covermode=set -tags=certtests -timeout=30m -coverpkg=${{ matrix.source-pkg }} status=$? echo "Completed certification tests for ${{ matrix.component }} ... " if test $status -ne 0; then diff --git a/internal/authentication/oauth2/clientcredentials.go b/internal/authentication/oauth2/clientcredentials.go new file mode 100644 index 0000000000..8122984839 --- /dev/null +++ b/internal/authentication/oauth2/clientcredentials.go @@ -0,0 +1,165 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oauth2 + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net/http" + "net/url" + "sync" + "time" + + "golang.org/x/oauth2" + ccreds "golang.org/x/oauth2/clientcredentials" + + "github.com/dapr/kit/logger" +) + +// ClientCredentialsMetadata is the metadata fields which can be used by a +// component to configure an OIDC client_credentials token source. +type ClientCredentialsMetadata struct { + TokenCAPEM string `mapstructure:"oauth2TokenCAPEM"` + TokenURL string `mapstructure:"oauth2TokenURL"` + ClientID string `mapstructure:"oauth2ClientID"` + ClientSecret string `mapstructure:"oauth2ClientSecret"` + Audiences []string `mapstructure:"oauth2Audiences"` + Scopes []string `mapstructure:"oauth2Scopes"` +} + +type ClientCredentialsOptions struct { + Logger logger.Logger + TokenURL string + ClientID string + ClientSecret string + Scopes []string + Audiences []string + CAPEM []byte +} + +// ClientCredentials is an OAuth2 Token Source that uses the client_credentials +// grant type to fetch a token. +type ClientCredentials struct { + log logger.Logger + currentToken *oauth2.Token + httpClient *http.Client + fetchTokenFn func(context.Context) (*oauth2.Token, error) + + lock sync.RWMutex +} + +func NewClientCredentials(ctx context.Context, opts ClientCredentialsOptions) (*ClientCredentials, error) { + conf, httpClient, err := opts.toConfig() + if err != nil { + return nil, err + } + + token, err := conf.Token(context.WithValue(ctx, oauth2.HTTPClient, httpClient)) + if err != nil { + return nil, fmt.Errorf("error fetching initial oauth2 client_credentials token: %w", err) + } + + opts.Logger.Info("Fetched initial oauth2 client_credentials token") + + return &ClientCredentials{ + log: opts.Logger, + currentToken: token, + httpClient: httpClient, + fetchTokenFn: conf.Token, + }, nil +} + +func (c *ClientCredentialsOptions) toConfig() (*ccreds.Config, *http.Client, error) { + if len(c.Scopes) == 0 { + return nil, nil, errors.New("oauth2 client_credentials token source requires at least one scope") + } + + if len(c.Audiences) == 0 { + return nil, nil, errors.New("oauth2 client_credentials token source requires at least one audience") + } + + _, err := url.Parse(c.TokenURL) + if err != nil { + return nil, nil, fmt.Errorf("error parsing token URL: %w", err) + } + + conf := &ccreds.Config{ + ClientID: c.ClientID, + ClientSecret: c.ClientSecret, + TokenURL: c.TokenURL, + Scopes: c.Scopes, + EndpointParams: url.Values{"audience": c.Audiences}, + } + + // If caPool is nil, then the Go TLS library will use the system's root CA. + var caPool *x509.CertPool + if len(c.CAPEM) > 0 { + caPool = x509.NewCertPool() + if !caPool.AppendCertsFromPEM(c.CAPEM) { + return nil, nil, errors.New("failed to parse CA PEM") + } + } + + return conf, &http.Client{ + Timeout: time.Second * 30, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: caPool, + }, + }, + }, nil +} + +func (c *ClientCredentials) Token() (string, error) { + c.lock.RLock() + defer c.lock.RUnlock() + + if !c.currentToken.Valid() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + if err := c.renewToken(ctx); err != nil { + return "", err + } + } + + return c.currentToken.AccessToken, nil +} + +func (c *ClientCredentials) renewToken(ctx context.Context) error { + c.lock.Lock() + defer c.lock.Unlock() + + // We need to check if the current token is valid because we might have lost + // the mutex lock race from the caller and we don't want to double-fetch a + // token unnecessarily! + if c.currentToken.Valid() { + return nil + } + + token, err := c.fetchTokenFn(context.WithValue(ctx, oauth2.HTTPClient, c.httpClient)) + if err != nil { + return err + } + + if !c.currentToken.Valid() { + return errors.New("oauth2 client_credentials token source returned an invalid token") + } + + c.currentToken = token + return nil +} diff --git a/internal/authentication/oauth2/clientcredentials_test.go b/internal/authentication/oauth2/clientcredentials_test.go new file mode 100644 index 0000000000..ba643b2129 --- /dev/null +++ b/internal/authentication/oauth2/clientcredentials_test.go @@ -0,0 +1,95 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oauth2 + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + ccreds "golang.org/x/oauth2/clientcredentials" +) + +func Test_toConfig(t *testing.T) { + tests := map[string]struct { + opts ClientCredentialsOptions + expConfig *ccreds.Config + expErr bool + }{ + "no scopes should error": { + opts: ClientCredentialsOptions{ + TokenURL: "https://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Audiences: []string{"audience"}, + }, + expErr: true, + }, + "bad URL endpoint should error": { + opts: ClientCredentialsOptions{ + TokenURL: "&&htp:/f url", + ClientID: "client-id", + ClientSecret: "client-secret", + Audiences: []string{"audience"}, + Scopes: []string{"foo"}, + }, + expErr: true, + }, + "bad CA certificate should error": { + opts: ClientCredentialsOptions{ + TokenURL: "http://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Audiences: []string{"audience"}, + Scopes: []string{"foo"}, + CAPEM: []byte("ca-pem"), + }, + expErr: true, + }, + "no audiences should error": { + opts: ClientCredentialsOptions{ + TokenURL: "http://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Scopes: []string{"foo"}, + }, + expErr: true, + }, + "should default scope": { + opts: ClientCredentialsOptions{ + TokenURL: "http://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Audiences: []string{"audience"}, + Scopes: []string{"foo", "bar"}, + }, + expConfig: &ccreds.Config{ + ClientID: "client-id", + ClientSecret: "client-secret", + TokenURL: "http://localhost:8080", + Scopes: []string{"foo", "bar"}, + EndpointParams: url.Values{"audience": []string{"audience"}}, + }, + expErr: false, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + config, _, err := test.opts.toConfig() + assert.Equalf(t, test.expErr, err != nil, "%v", err) + assert.Equal(t, test.expConfig, config) + }) + } +} diff --git a/pubsub/pulsar/metadata.go b/pubsub/pulsar/metadata.go index 5bddedb6d6..719f75a69d 100644 --- a/pubsub/pulsar/metadata.go +++ b/pubsub/pulsar/metadata.go @@ -13,7 +13,11 @@ limitations under the License. package pulsar -import "time" +import ( + "time" + + "github.com/dapr/components-contrib/internal/authentication/oauth2" +) type pulsarMetadata struct { Host string `mapstructure:"host"` @@ -26,12 +30,14 @@ type pulsarMetadata struct { Tenant string `mapstructure:"tenant"` Namespace string `mapstructure:"namespace"` Persistent bool `mapstructure:"persistent"` - Token string `mapstructure:"token"` RedeliveryDelay time.Duration `mapstructure:"redeliveryDelay"` internalTopicSchemas map[string]schemaMetadata `mapstructure:"-"` PublicKey string `mapstructure:"publicKey"` PrivateKey string `mapstructure:"privateKey"` Keys string `mapstructure:"keys"` + + Token string `mapstructure:"token"` + oauth2.ClientCredentialsMetadata `mapstructure:",squash"` } type schemaMetadata struct { diff --git a/pubsub/pulsar/pulsar.go b/pubsub/pulsar/pulsar.go index 8846f2dc50..336b525fad 100644 --- a/pubsub/pulsar/pulsar.go +++ b/pubsub/pulsar/pulsar.go @@ -25,12 +25,12 @@ import ( "sync/atomic" "time" - "github.com/hamba/avro/v2" - "github.com/apache/pulsar-client-go/pulsar" "github.com/apache/pulsar-client-go/pulsar/crypto" + "github.com/hamba/avro/v2" lru "github.com/hashicorp/golang-lru/v2" + "github.com/dapr/components-contrib/internal/authentication/oauth2" "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/pubsub" "github.com/dapr/kit/logger" @@ -157,7 +157,7 @@ func parsePulsarMetadata(meta pubsub.Metadata) (*pulsarMetadata, error) { return &m, nil } -func (p *Pulsar) Init(_ context.Context, metadata pubsub.Metadata) error { +func (p *Pulsar) Init(ctx context.Context, metadata pubsub.Metadata) error { m, err := parsePulsarMetadata(metadata) if err != nil { return err @@ -173,9 +173,28 @@ func (p *Pulsar) Init(_ context.Context, metadata pubsub.Metadata) error { ConnectionTimeout: 30 * time.Second, TLSAllowInsecureConnection: !m.EnableTLS, } - if m.Token != "" { + + switch { + case len(m.Token) > 0: options.Authentication = pulsar.NewAuthenticationToken(m.Token) + case len(m.ClientCredentialsMetadata.TokenURL) > 0: + var cc *oauth2.ClientCredentials + cc, err = oauth2.NewClientCredentials(ctx, oauth2.ClientCredentialsOptions{ + Logger: p.logger, + TokenURL: m.ClientCredentialsMetadata.TokenURL, + CAPEM: []byte(m.ClientCredentialsMetadata.TokenCAPEM), + ClientID: m.ClientCredentialsMetadata.ClientID, + ClientSecret: m.ClientCredentialsMetadata.ClientSecret, + Scopes: m.ClientCredentialsMetadata.Scopes, + Audiences: m.ClientCredentialsMetadata.Audiences, + }) + if err != nil { + return fmt.Errorf("could not instantiate oauth2 token provider: %w", err) + } + + options.Authentication = pulsar.NewAuthenticationTokenFromSupplier(cc.Token) } + client, err := pulsar.NewClient(options) if err != nil { return fmt.Errorf("could not instantiate pulsar client: %v", err) diff --git a/tests/certification/pubsub/pulsar/components/consumer_five/pulsar.yaml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_five/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_five/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_five/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/consumer_four/pulsar.yaml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_four/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_four/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_four/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/consumer_one/pulsar.yml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_one/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_one/pulsar.yml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_one/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/components/consumer_six/pulsar.yaml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_six/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_six/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_six/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/consumer_three/pulsar.yml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_three/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_three/pulsar.yml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_three/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/components/consumer_two/pulsar.yml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_two/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_two/pulsar.yml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_two/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_five/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_five/pulsar.yml.tmpl new file mode 100644 index 0000000000..9251380ebb --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_five/pulsar.yml.tmpl @@ -0,0 +1,32 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification5 + - name: redeliveryDelay + value: 200ms + - name: publicKey + value: public.key + - name: privateKey + value: private.key + - name: keys + value: myapp.key + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_four/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_four/pulsar.yml.tmpl new file mode 100644 index 0000000000..3078529e2e --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_four/pulsar.yml.tmpl @@ -0,0 +1,28 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification4 + - name: redeliveryDelay + value: 200ms + - name: certification-pubsub-topic-active.jsonschema + value: "{\"type\":\"record\",\"name\":\"Example\",\"namespace\":\"test\",\"fields\":[{\"name\":\"ID\",\"type\":\"int\"},{\"name\":\"Name\",\"type\":\"string\"}]}" + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_one/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_one/pulsar.yml.tmpl new file mode 100644 index 0000000000..d5258c3a1c --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_one/pulsar.yml.tmpl @@ -0,0 +1,26 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification1 + - name: redeliveryDelay + value: 200ms + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_six/pulsar.yaml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_six/pulsar.yaml.tmpl new file mode 100644 index 0000000000..1f7bf98510 --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_six/pulsar.yaml.tmpl @@ -0,0 +1,32 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification5 + - name: redeliveryDelay + value: 200ms + - name: publicKey + value: "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1KDAM4L8RtJ+nLaXBrBh\nzVpvTemsKVZoAct8A+ShepOHT9lgHOCGLFGWNla6K6j+b3AV/P/fAAhwj82vwTDd\nruXSflvSdmYeFAw3Ypphc1A5oM53wSRWhg63potBNWqdDzj8ApYgqjpmjYSQdL5/\na3golb36GYFrY0MLFTv7wZ87pmMIPsOgGIcPbCHker2fRZ34WXYLb1hkeUpwx4eK\njpwcg35gccvR6o/UhbKAuc60V1J9Wof2sNgtlRaQej45wnpjWYzZrIyk5qUbn0Qi\nCdpIrXvYtANq0Id6gP8zJvUEdPIgNuYxEmVCl9jI+8eGI6peD0qIt8U80hf9axhJ\n3QIDAQAB\n-----END PUBLIC KEY-----\n" + - name: privateKey + value: "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA1KDAM4L8RtJ+nLaXBrBhzVpvTemsKVZoAct8A+ShepOHT9lg\nHOCGLFGWNla6K6j+b3AV/P/fAAhwj82vwTDdruXSflvSdmYeFAw3Ypphc1A5oM53\nwSRWhg63potBNWqdDzj8ApYgqjpmjYSQdL5/a3golb36GYFrY0MLFTv7wZ87pmMI\nPsOgGIcPbCHker2fRZ34WXYLb1hkeUpwx4eKjpwcg35gccvR6o/UhbKAuc60V1J9\nWof2sNgtlRaQej45wnpjWYzZrIyk5qUbn0QiCdpIrXvYtANq0Id6gP8zJvUEdPIg\nNuYxEmVCl9jI+8eGI6peD0qIt8U80hf9axhJ3QIDAQABAoIBAQCKuHnM4ac/eXM7\nQPDVX1vfgyHc3hgBPCtNCHnXfGFRvFBqavKGxIElBvGOcBS0CWQ+Rg1Ca5kMx3TQ\njSweSYhH5A7pe3Sa5FK5V6MGxJvRhMSkQi/lJZUBjzaIBJA9jln7pXzdHx8ekE16\nBMPONr6g2dr4nuI9o67xKrtfViwRDGaG6eh7jIMlEqMMc6WqyhvI67rlVDSTHFKX\njlMcozJ3IT8BtTzKg2Tpy7ReVuJEpehum8yn1ZVdAnotBDJxI07DC1cbOP4M2fHM\ngfgPYWmchauZuTeTFu4hrlY5jg0/WLs6by8r/81+vX3QTNvejX9UdTHMSIfQdX82\nAfkCKUVhAoGBAOvGv+YXeTlPRcYC642x5iOyLQm+BiSX4jKtnyJiTU2s/qvvKkIu\nxAOk3OtniT9NaUAHEZE9tI71dDN6IgTLQlAcPCzkVh6Sc5eG0MObqOO7WOMCWBkI\nlaAKKBbd6cGDJkwGCJKnx0pxC9f8R4dw3fmXWgWAr8ENiekMuvjSfjZ5AoGBAObd\ns2L5uiUPTtpyh8WZ7rEvrun3djBhzi+d7rgxEGdditeiLQGKyZbDPMSMBuus/5wH\nwfi0xUq50RtYDbzQQdC3T/C20oHmZbjWK5mDaLRVzWS89YG/NT2Q8eZLBstKqxkx\ngoT77zoUDfRy+CWs1xvXzgxagD5Yg8/OrCuXOqWFAoGAPIw3r6ELknoXEvihASxU\nS4pwInZYIYGXpygLG8teyrnIVOMAWSqlT8JAsXtPNaBtjPHDwyazfZrvEmEk51JD\nX0tA8M5ah1NYt+r5JaKNxp3P/8wUT6lyszyoeubWJsnFRfSusuq/NRC+1+KDg/aq\nKnSBu7QGbm9JoT2RrmBv5RECgYBRn8Lj1I1muvHTNDkiuRj2VniOSirkUkA2/6y+\nPMKi+SS0tqcY63v4rNCYYTW1L7Yz8V44U5mJoQb4lvpMbolGhPljjxAAU3hVkItb\nvGVRlSCIZHKczADD4rJUDOS7DYxO3P1bjUN4kkyYx+lKUMDBHFzCa2D6Kgt4dobS\n5qYajQKBgQC7u7MFPkkEMqNqNGu5erytQkBq1v1Ipmf9rCi3iIj4XJLopxMgw0fx\n6jwcwNInl72KzoUBLnGQ9PKGVeBcgEgdI+a+tq+1TJo6Ta+hZSx+4AYiKY18eRKG\neNuER9NOcSVJ7Eqkcw4viCGyYDm2vgNV9HJ0VlAo3RDh8x5spEN+mg==\n-----END RSA PRIVATE KEY-----\n" + - name: keys + value: myapp.key + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_three/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_three/pulsar.yml.tmpl new file mode 100644 index 0000000000..c61e7d0a37 --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_three/pulsar.yml.tmpl @@ -0,0 +1,26 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification3 + - name: redeliveryDelay + value: 200ms + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_two/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_two/pulsar.yml.tmpl new file mode 100644 index 0000000000..06262d2620 --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_two/pulsar.yml.tmpl @@ -0,0 +1,26 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification2 + - name: redeliveryDelay + value: 200ms + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oauth2-server.yaml b/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oauth2-server.yaml new file mode 100644 index 0000000000..220011d949 --- /dev/null +++ b/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oauth2-server.yaml @@ -0,0 +1,14 @@ +# We run in network_mode: "host" so `localhost` is the same for both the host +# and containers. This is required as the mock server uses the SNI hostname to +# build the issuer URL. +version: '3' +services: + mock-oauth2-server: + image: ghcr.io/navikt/mock-oauth2-server:1.0.0 + container_name: mock-oauth2-server + restart: on-failure + network_mode: "host" + environment: + - PORT=8085 + - LOG_LEVEL=DEBUG + - 'JSON_CONFIG={"interactiveLogin":false,"httpServer":{"type":"NettyWrapper","ssl":{}},"tokenCallbacks":[{"issuerId":"issuer1","tokenExpiry":120,"requestMappings":[{"requestParam":"scope","match":"openid","claims":{"sub":"foo","aud":["pulsar"]}}]}]}' diff --git a/tests/certification/pubsub/pulsar/docker-compose.yml b/tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml similarity index 97% rename from tests/certification/pubsub/pulsar/docker-compose.yml rename to tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml index 5ae0535596..7857b77961 100644 --- a/tests/certification/pubsub/pulsar/docker-compose.yml +++ b/tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml @@ -16,4 +16,4 @@ services: - pulsarconf:/pulsar/conf volumes: pulsardata: - pulsarconf: \ No newline at end of file + pulsarconf: diff --git a/tests/certification/pubsub/pulsar/config/docker-compose_auth-oauth2.yaml.tmpl b/tests/certification/pubsub/pulsar/config/docker-compose_auth-oauth2.yaml.tmpl new file mode 100644 index 0000000000..55ae595957 --- /dev/null +++ b/tests/certification/pubsub/pulsar/config/docker-compose_auth-oauth2.yaml.tmpl @@ -0,0 +1,96 @@ +# We run the pulsar services individually as OAuth2 doesn't seem to work in +# standalone mode. OAuth2 is also only available from pulsar v3 onwards. We use +# host networking as the mock OAuth server uses the SNI host name to determine +# the host name of the OAuth2 issuer URL, so we need to have the mock server +# reachable by localhost from both the pulsar services and the host network. +version: '3' +services: + # Start zookeeper + zookeeper: + image: apachepulsar/pulsar:3.0.0 + container_name: zookeeper + restart: on-failure + network_mode: "host" + environment: + - metadataStoreUrl=zk:localhost:2181 + - metricsProvider.httpPort=7000 + - PULSAR_MEM=-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m + command: > + bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \ + bin/generate-zookeeper-config.sh conf/zookeeper.conf && \ + exec bin/pulsar zookeeper" + healthcheck: + test: ["CMD", "bin/pulsar-zookeeper-ruok.sh"] + interval: 1s + timeout: 5s + retries: 300 + + # Init cluster metadata + pulsar-init: + container_name: pulsar-init + image: apachepulsar/pulsar:3.0.0 + network_mode: "host" + env_file: + - ./pulsar_auth-oauth2.conf + command: > + bin/pulsar initialize-cluster-metadata \ + --cluster cluster-a \ + --zookeeper localhost:2181 \ + --configuration-store localhost:2181 \ + --web-service-url http://localhost:8080 \ + --broker-service-url pulsar://localhost:6650 + depends_on: + zookeeper: + condition: service_healthy + + # Start bookie + bookie: + image: apachepulsar/pulsar:3.0.0 + container_name: bookie + restart: on-failure + network_mode: "host" + environment: + - clusterName=cluster-a + - zkServers=localhost:2181 + - metadataServiceUri=metadata-store:zk:localhost:2181 + # otherwise every time we run docker compose uo or down we fail to start due to Cookie + # See: https://github.com/apache/bookkeeper/blob/405e72acf42bb1104296447ea8840d805094c787/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Cookie.java#L57-68 + - advertisedAddress=localhost + - BOOKIE_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m + env_file: + - ./pulsar_auth-oauth2.conf + volumes: + - "{{ .TmpDir }}:/pulsar/conf/dapr" + depends_on: + zookeeper: + condition: service_healthy + pulsar-init: + condition: service_completed_successfully + command: bash -c "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + + # Start broker + broker: + image: apachepulsar/pulsar:3.0.0 + container_name: broker + restart: on-failure + network_mode: "host" + env_file: + - ./pulsar_auth-oauth2.conf + volumes: + - "{{ .TmpDir }}:/pulsar/conf/dapr" + environment: + - metadataStoreUrl=zk:localhost:2181 + - zookeeperServers=localhost:2181 + - clusterName=cluster-a + - managedLedgerDefaultEnsembleSize=1 + - managedLedgerDefaultWriteQuorum=1 + - managedLedgerDefaultAckQuorum=1 + - advertisedAddress=localhost + - advertisedListeners=external:pulsar://127.0.0.1:6650 + - PULSAR_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m + depends_on: + zookeeper: + condition: service_healthy + bookie: + condition: service_started + command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" diff --git a/tests/certification/pubsub/pulsar/config/pulsar_auth-oauth2.conf b/tests/certification/pubsub/pulsar/config/pulsar_auth-oauth2.conf new file mode 100644 index 0000000000..a9f35e4a7e --- /dev/null +++ b/tests/certification/pubsub/pulsar/config/pulsar_auth-oauth2.conf @@ -0,0 +1,39 @@ +# Configuration to enable authentication +authenticationEnabled=true +authenticationProviders=org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID + +# Required settings for AuthenticationProviderOpenID +# A comma separated list of allowed, or trusted, token issuers. The token issuer is the URL of the token issuer. +PULSAR_PREFIX_openIDAllowedTokenIssuers=https://localhost:8085/issuer1 +# The list of allowed audiences for the token. The audience is the intended recipient of the token. A token with +# at least one of these audience claims will pass the audience validation check. +PULSAR_PREFIX_openIDAllowedAudiences=pulsar + +# Optional settings (values shown are the defaults) +# The path to the file containing the trusted certificate(s) of the token issuer(s). If not set, uses the default +# trust store of the JVM. Note: in version 3.0.0, the default only applies when this setting is not an environment +# variable and is not in the configuration file. +PULSAR_PREFIX_openIDTokenIssuerTrustCertsFilePath=/pulsar/conf/dapr/ca.pem +# The JWT's claim to use for the role/principal during authorization. +PULSAR_PREFIX_openIDRoleClaim=sub +# The leeway, in seconds, to use when validating the token's expiration time. +PULSAR_PREFIX_openIDAcceptedTimeLeewaySeconds=0 + +# Cache settings +PULSAR_PREFIX_openIDCacheSize=5 +PULSAR_PREFIX_openIDCacheRefreshAfterWriteSeconds=64800 +PULSAR_PREFIX_openIDCacheExpirationSeconds=86400 +PULSAR_PREFIX_openIDHttpConnectionTimeoutMillis=10000 +PULSAR_PREFIX_openIDHttpReadTimeoutMillis=10000 + +# The number of seconds to wait before refreshing the JWKS when a token presents a key ID (kid claim) that is not +# in the cache. This setting is available from Pulsar 3.0.1 and is documented below. +PULSAR_PREFIX_openIDKeyIdCacheMissRefreshSeconds=300 + +# Whether to require that issuers use HTTPS. It is part of the OAuth2 spec to use HTTPS, so the default is true. +# This setting is for testing purposes and is not recommended for any production environment. +#PULSAR_PREFIX_openIDRequireIssuersUseHttps=false + +# A setting describing how to handle discovery of the OpenID Connect configuration document when the issuer is not +# in the list of allowed issuers. This setting is documented below. +PULSAR_PREFIX_openIDFallbackDiscoveryMode=DISABLED diff --git a/tests/certification/pubsub/pulsar/pulsar_test.go b/tests/certification/pubsub/pulsar/pulsar_test.go index 9c7fd3e405..df170e5678 100644 --- a/tests/certification/pubsub/pulsar/pulsar_test.go +++ b/tests/certification/pubsub/pulsar/pulsar_test.go @@ -16,17 +16,28 @@ package pulsar_test import ( "bytes" "context" + "crypto/tls" "encoding/json" + "encoding/pem" "fmt" + "io" + "io/fs" "io/ioutil" "net/http" + "os" + "os/exec" + "path/filepath" + "strings" "testing" + "text/template" "time" "github.com/google/uuid" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "go.uber.org/multierr" + "github.com/dapr/components-contrib/internal/authentication/oauth2" pubsub_pulsar "github.com/dapr/components-contrib/pubsub/pulsar" pubsub_loader "github.com/dapr/dapr/pkg/components/pubsub" @@ -54,21 +65,23 @@ const ( appID1 = "app-1" appID2 = "app-2" - numMessages = 10 - appPort = 8000 - portOffset = 2 - messageKey = "partitionKey" - pubsubName = "messagebus" - topicActiveName = "certification-pubsub-topic-active" - topicPassiveName = "certification-pubsub-topic-passive" - topicToBeCreated = "certification-topic-per-test-run" - topicDefaultName = "certification-topic-default" - topicMultiPartitionName = "certification-topic-multi-partition8" - partition0 = "partition-0" - partition1 = "partition-1" - clusterName = "pulsarcertification" - dockerComposeYAML = "docker-compose.yml" - pulsarURL = "localhost:6650" + numMessages = 10 + appPort = 8001 + portOffset = 2 + messageKey = "partitionKey" + pubsubName = "messagebus" + topicActiveName = "certification-pubsub-topic-active" + topicPassiveName = "certification-pubsub-topic-passive" + topicToBeCreated = "certification-topic-per-test-run" + topicDefaultName = "certification-topic-default" + topicMultiPartitionName = "certification-topic-multi-partition8" + partition0 = "partition-0" + partition1 = "partition-1" + clusterName = "pulsarcertification" + dockerComposeAuthNoneYAML = "./config/docker-compose_auth-none.yaml" + dockerComposeAuthOAuth2YAML = "./config/docker-compose_auth-oauth2.yaml.tmpl" + dockerComposeMockOAuth2YAML = "./config/docker-compose_auth-mock-oauth2-server.yaml" + pulsarURL = "localhost:6650" subscribeTypeKey = "subscribeType" @@ -82,6 +95,103 @@ const ( processModeSync = "sync" ) +type pulsarSuite struct { + suite.Suite + + authType string + oauth2CAPEM []byte + dockerComposeYAML string + componentsPath string + services []string +} + +func TestPulsar(t *testing.T) { + t.Run("Auth:None", func(t *testing.T) { + suite.Run(t, &pulsarSuite{ + authType: "none", + dockerComposeYAML: dockerComposeAuthNoneYAML, + componentsPath: "./components/auth-none", + services: []string{"standalone"}, + }) + }) + + t.Run("Auth:OAuth2", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.Chmod(dir, 0o777)) + + t.Log("Starting OAuth2 server...") + out, err := exec.Command( + "docker-compose", + "-p", "oauth2", + "-f", dockerComposeMockOAuth2YAML, + "up", "-d").CombinedOutput() + require.NoError(t, err, string(out)) + t.Log(string(out)) + + t.Cleanup(func() { + t.Log("Stopping OAuth2 server...") + out, err = exec.Command( + "docker-compose", + "-p", "oauth2", + "-f", dockerComposeMockOAuth2YAML, + "down", "-v", + "--remove-orphans").CombinedOutput() + require.NoError(t, err, string(out)) + t.Log(string(out)) + }) + + t.Log("Waiting for OAuth server to be ready...") + oauth2CA := peerCertificate(t, "localhost:8085") + t.Log("OAuth server is ready") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.pem"), oauth2CA, 0o644)) + outf, err := os.OpenFile("./config/pulsar_auth-oauth2.conf", os.O_RDONLY, 0o644) + require.NoError(t, err) + inf, err := os.OpenFile(filepath.Join(dir, "pulsar_auth-oauth2.conf"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + require.NoError(t, err) + _, err = io.Copy(inf, outf) + require.NoError(t, err) + outf.Close() + inf.Close() + + td := struct { + TmpDir string + OAuth2CAPEM string + }{ + TmpDir: dir, + OAuth2CAPEM: strings.ReplaceAll(string(oauth2CA), "\n", "\\n"), + } + + tmpl, err := template.New("").ParseFiles(dockerComposeAuthOAuth2YAML) + require.NoError(t, err) + f, err := os.OpenFile(filepath.Join(dir, "docker-compose.yaml"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + require.NoError(t, err) + require.NoError(t, tmpl.ExecuteTemplate(f, "docker-compose_auth-oauth2.yaml.tmpl", td)) + + require.NoError(t, filepath.Walk("./components/auth-oauth2", func(path string, info fs.FileInfo, err error) error { + if info.IsDir() { + return nil + } + tmpl, err := template.New("").ParseFiles(path) + require.NoError(t, err) + path = strings.TrimSuffix(path, ".tmpl") + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, path)), 0o755)) + f, err := os.OpenFile(filepath.Join(dir, path), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + require.NoError(t, err) + require.NoError(t, tmpl.ExecuteTemplate(f, filepath.Base(path)+".tmpl", td)) + return nil + })) + + suite.Run(t, &pulsarSuite{ + oauth2CAPEM: oauth2CA, + authType: "oauth2", + dockerComposeYAML: filepath.Join(dir, "docker-compose.yaml"), + componentsPath: filepath.Join(dir, "components/auth-oauth2"), + services: []string{"zookeeper", "pulsar-init", "bookie", "broker"}, + }) + }) +} + func subscriberApplication(appID string, topicName string, messagesWatcher *watcher.Watcher) app.SetupFn { return func(ctx flow.Context, s common.Service) error { // Simulate periodic errors. @@ -196,7 +306,8 @@ func assertMessages(timeout time.Duration, messageWatchers ...*watcher.Watcher) } } -func TestPulsar(t *testing.T) { +func (p *pulsarSuite) TestPulsar() { + t := p.T() consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -244,10 +355,10 @@ func TestPulsar(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -267,7 +378,7 @@ func TestPulsar(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -280,7 +391,7 @@ func TestPulsar(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -295,7 +406,8 @@ func TestPulsar(t *testing.T) { Run() } -func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { +func (p *pulsarSuite) TestPulsarMultipleSubsSameConsumerIDs() { + t := p.T() consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -312,10 +424,10 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -335,7 +447,7 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -348,7 +460,7 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -362,7 +474,9 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { Run() } -func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { +func (p *pulsarSuite) TestPulsarMultipleSubsDifferentConsumerIDs() { + t := p.T() + consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -376,10 +490,10 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -401,7 +515,7 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -414,7 +528,7 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -427,7 +541,8 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { Run() } -func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { +func (p *pulsarSuite) TestPulsarMultiplePubSubsDifferentConsumerIDs() { + t := p.T() consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -445,10 +560,10 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -470,7 +585,7 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -483,7 +598,7 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -498,7 +613,8 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { Run() } -func TestPulsarNonexistingTopic(t *testing.T) { +func (p *pulsarSuite) TestPulsarNonexistingTopic() { + t := p.T() consumerGroup1 := watcher.NewUnordered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -511,10 +627,10 @@ func TestPulsarNonexistingTopic(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset*3), subscriberApplication(appID1, topicToBeCreated, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -537,7 +653,7 @@ func TestPulsarNonexistingTopic(t *testing.T) { })). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset*3), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*3), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*3), @@ -550,7 +666,8 @@ func TestPulsarNonexistingTopic(t *testing.T) { Run() } -func TestPulsarNetworkInterruption(t *testing.T) { +func (p *pulsarSuite) TestPulsarNetworkInterruption() { + t := p.T() consumerGroup1 := watcher.NewUnordered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -563,10 +680,10 @@ func TestPulsarNetworkInterruption(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -589,7 +706,7 @@ func TestPulsarNetworkInterruption(t *testing.T) { })). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -603,7 +720,8 @@ func TestPulsarNetworkInterruption(t *testing.T) { Run() } -func TestPulsarPersitant(t *testing.T) { +func (p *pulsarSuite) TestPulsarPersitant() { + t := p.T() consumerGroup1 := watcher.NewUnordered() flow.New(t, "pulsar certification persistant test"). @@ -611,10 +729,10 @@ func TestPulsarPersitant(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -636,23 +754,24 @@ func TestPulsarPersitant(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), componentRuntimeOptions(), )). Step("publish messages to topic1", publishMessages(nil, sidecarName1, topicActiveName, consumerGroup1)). - Step("stop pulsar server", dockercompose.Stop(clusterName, dockerComposeYAML, "standalone")). + Step("stop pulsar server", dockercompose.Stop(clusterName, p.dockerComposeYAML, p.services...)). Step("wait", flow.Sleep(5*time.Second)). - Step("start pulsar server", dockercompose.Start(clusterName, dockerComposeYAML, "standalone")). + Step("start pulsar server", dockercompose.Start(clusterName, p.dockerComposeYAML, p.services...)). Step("wait", flow.Sleep(10*time.Second)). Step("verify if app1 has received messages published to active topic", assertMessages(10*time.Second, consumerGroup1)). Step("reset", flow.Reset(consumerGroup1)). Run() } -func TestPulsarDelay(t *testing.T) { +func (p *pulsarSuite) TestPulsarDelay() { + t := p.T() consumerGroup1 := watcher.NewUnordered() date := time.Now() @@ -682,10 +801,10 @@ func TestPulsarDelay(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -707,7 +826,7 @@ func TestPulsarDelay(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_three"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_three")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -730,7 +849,8 @@ type schemaTest struct { Name string `json:"name"` } -func TestPulsarSchema(t *testing.T) { +func (p *pulsarSuite) TestPulsarSchema() { + t := p.T() consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -772,10 +892,10 @@ func TestPulsarSchema(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -795,7 +915,7 @@ func TestPulsarSchema(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_four"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_four")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -818,7 +938,7 @@ func componentRuntimeOptions() []runtime.Option { } } -func createMultiPartitionTopic(tenant, namespace, topic string, partition int) flow.Runnable { +func (p *pulsarSuite) createMultiPartitionTopic(tenant, namespace, topic string, partition int) flow.Runnable { return func(ctx flow.Context) error { reqURL := fmt.Sprintf("http://localhost:8080/admin/v2/persistent/%s/%s/%s/partitions", tenant, namespace, topic) @@ -838,6 +958,19 @@ func createMultiPartitionTopic(tenant, namespace, topic string, partition int) f req.Header.Set("Content-Type", "application/json") + if p.authType == "oauth2" { + cc, err := p.oauth2ClientCredentials() + if err != nil { + return err + } + token, err := cc.Token() + if err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + } + rsp, err := http.DefaultClient.Do(req) if err != nil { @@ -858,7 +991,8 @@ func createMultiPartitionTopic(tenant, namespace, topic string, partition int) f } } -func TestPulsarPartitionedOrderingProcess(t *testing.T) { +func (p *pulsarSuite) TestPulsarPartitionedOrderingProcess() { + t := p.T() consumerGroup1 := watcher.NewOrdered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -867,14 +1001,14 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { } flow.New(t, "pulsar certification - process message in order with partitioned-topic"). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset), subscriberApplicationWithoutError(appID1, topicMultiPartitionName, consumerGroup1))). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -897,10 +1031,10 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { return err })). Step("create multi-partition topic explicitly", retry.Do(10*time.Second, 30, - createMultiPartitionTopic("public", "default", topicMultiPartitionName, 4))). + p.createMultiPartitionTopic("public", "default", topicMultiPartitionName, 4))). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -913,7 +1047,7 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset*3), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*3), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*3), @@ -927,7 +1061,8 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { Run() } -func TestPulsarEncryptionFromFile(t *testing.T) { +func (p *pulsarSuite) TestPulsarEncryptionFromFile() { + t := p.T() consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -969,10 +1104,10 @@ func TestPulsarEncryptionFromFile(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -992,7 +1127,7 @@ func TestPulsarEncryptionFromFile(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_five"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_five")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -1004,7 +1139,8 @@ func TestPulsarEncryptionFromFile(t *testing.T) { Run() } -func TestPulsarEncryptionFromData(t *testing.T) { +func (p *pulsarSuite) TestPulsarEncryptionFromData() { + t := p.T() consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -1046,10 +1182,10 @@ func TestPulsarEncryptionFromData(t *testing.T) { // Run subscriberApplication app2 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -1069,7 +1205,7 @@ func TestPulsarEncryptionFromData(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_six"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_six")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -1080,3 +1216,57 @@ func TestPulsarEncryptionFromData(t *testing.T) { Step("reset", flow.Reset(consumerGroup1)). Run() } + +func (p *pulsarSuite) client(t *testing.T) (pulsar.Client, error) { + t.Helper() + + opts := pulsar.ClientOptions{ + URL: "pulsar://localhost:6650", + } + switch p.authType { + case "oauth2": + cc, err := p.oauth2ClientCredentials() + require.NoError(t, err) + opts.Authentication = pulsar.NewAuthenticationTokenFromSupplier(cc.Token) + default: + } + + return pulsar.NewClient(opts) +} + +func (p *pulsarSuite) oauth2ClientCredentials() (*oauth2.ClientCredentials, error) { + cc, err := oauth2.NewClientCredentials(context.Background(), oauth2.ClientCredentialsOptions{ + Logger: logger.NewLogger("dapr.test.readiness"), + TokenURL: "https://localhost:8085/issuer1/token", + ClientID: "foo", + ClientSecret: "bar", + Scopes: []string{"openid"}, + Audiences: []string{"pulsar"}, + CAPEM: p.oauth2CAPEM, + }) + if err != nil { + return nil, err + } + + return cc, nil +} + +func peerCertificate(t *testing.T, hostport string) []byte { + conf := &tls.Config{InsecureSkipVerify: true} + + for { + time.Sleep(1 * time.Second) + + conn, err := tls.Dial("tcp", hostport, conf) + if err != nil { + t.Log(err) + continue + } + + defer conn.Close() + + certs := conn.ConnectionState().PeerCertificates + require.Len(t, certs, 1, "expected 1 peer certificate") + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certs[0].Raw}) + } +} From 8b6156f4f722631ab68ada17c926a294fe37526e Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Thu, 3 Aug 2023 07:37:22 -0700 Subject: [PATCH 14/22] Bearer middleware: add "infer algorithm from key" (#3033) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- middleware/http/bearer/bearer_middleware.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/middleware/http/bearer/bearer_middleware.go b/middleware/http/bearer/bearer_middleware.go index 030403716a..e3e7359e82 100644 --- a/middleware/http/bearer/bearer_middleware.go +++ b/middleware/http/bearer/bearer_middleware.go @@ -23,6 +23,7 @@ import ( "github.com/lestrrat-go/httprc" "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/lestrrat-go/jwx/v2/jws" "github.com/lestrrat-go/jwx/v2/jwt" "github.com/dapr/components-contrib/internal/httputils" @@ -112,7 +113,7 @@ func (m *Middleware) GetHandler(ctx context.Context, metadata middleware.Metadat _, err = jwt.Parse([]byte(rawToken), jwt.WithContext(r.Context()), jwt.WithAcceptableSkew(allowedClockSkew), - jwt.WithKeySet(keyset), + jwt.WithKeySet(keyset, jws.WithInferAlgorithmFromKey(true)), jwt.WithAudience(meta.Audience), jwt.WithIssuer(meta.Issuer), ) From 6f9ab04a339fcdfbe41f9338953d45ee34e185fe Mon Sep 17 00:00:00 2001 From: Filinto Duran Date: Thu, 3 Aug 2023 10:19:21 -0500 Subject: [PATCH 15/22] Add closer to couchbase (#3036) Signed-off-by: Filinto Duran --- state/couchbase/couchbase.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/state/couchbase/couchbase.go b/state/couchbase/couchbase.go index 3da43f1567..594f6386f5 100644 --- a/state/couchbase/couchbase.go +++ b/state/couchbase/couchbase.go @@ -271,3 +271,10 @@ func (cbs *Couchbase) GetComponentMetadata() (metadataInfo metadata.MetadataMap) metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) return } + +func (cbs *Couchbase) Close() error { + if cbs.bucket == nil { + return nil + } + return cbs.bucket.Close() +} From 7fcfd9a81599871cbb3cfc37f4dc1103b3c9d401 Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Thu, 3 Aug 2023 11:33:06 -0700 Subject: [PATCH 16/22] Update dependencies (#3037) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .build-tools/go.mod | 2 +- .build-tools/go.sum | 4 +- .golangci.yml | 10 ++++ bindings/azure/openai/openai.go | 2 +- go.mod | 30 ++++++------ go.sum | 53 +++++++++++---------- tests/certification/go.mod | 22 ++++----- tests/certification/go.sum | 45 ++++++++--------- tests/conformance/standalone_loader.go | 17 ++++--- tests/conformance/standalone_loader_test.go | 23 ++++----- tests/e2e/pubsub/jetstream/go.mod | 2 +- tests/e2e/pubsub/jetstream/go.sum | 4 +- 12 files changed, 110 insertions(+), 104 deletions(-) diff --git a/.build-tools/go.mod b/.build-tools/go.mod index 710651c41e..ad9f1a5798 100644 --- a/.build-tools/go.mod +++ b/.build-tools/go.mod @@ -7,7 +7,7 @@ require ( github.com/invopop/jsonschema v0.6.0 github.com/spf13/cobra v1.6.1 github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5 - golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb + golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b gopkg.in/yaml.v3 v3.0.1 sigs.k8s.io/yaml v1.3.0 ) diff --git a/.build-tools/go.sum b/.build-tools/go.sum index f03aa9a1bb..a86c9d6df6 100644 --- a/.build-tools/go.sum +++ b/.build-tools/go.sum @@ -37,8 +37,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5 h1:ImnGIsrcG8vwbovhYvvSY8fagVV6QhCWSWXfzwGDLVs= github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/.golangci.yml b/.golangci.yml index 73e3b28b38..c72308bc64 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -124,6 +124,8 @@ linters-settings: - "golang.org/x/net/context": "must use context" - "github.com/pkg/errors": "must use standard library (errors package and/or fmt.Errorf)" - "github.com/Sirupsen/logrus": "must use github.com/dapr/kit/logger" + - "github.com/labstack/gommon/log": "must use github.com/dapr/kit/logger" + - "github.com/gobuffalo/logger": "must use github.com/dapr/kit/logger" - "github.com/agrea/ptr": "must use github.com/dapr/kit/ptr" - "github.com/cenkalti/backoff$": "must use github.com/cenkalti/backoff/v4" - "github.com/cenkalti/backoff/v2": "must use github.com/cenkalti/backoff/v4" @@ -133,6 +135,14 @@ linters-settings: - "github.com/golang-jwt/jwt/v4": "must use github.com/lestrrat-go/jwx/v2" - "github.com/lestrrat-go/jwx/jwa": "must use github.com/lestrrat-go/jwx/v2" - "github.com/lestrrat-go/jwx/jwt": "must use github.com/lestrrat-go/jwx/v2" + - "github.com/lestrrat-go/jwx/jws": "must use github.com/lestrrat-go/jwx/v2" + - "github.com/gogo/status": "must use google.golang.org/grpc/status" + - "github.com/gogo/protobuf": "must use google.golang.org/protobuf" + - "k8s.io/utils/pointer": "must use github.com/dapr/kit/ptr" + - "k8s.io/utils/ptr": "must use github.com/dapr/kit/ptr" + - "github.com/ghodss/yaml": "must use sigs.k8s.io/yaml" + - "gopkg.in/yaml.v2": "must use gopkg.in/yaml.v3" + - "github.com/go-chi/chi$": "must use github.com/go-chi/chi/v5" misspell: # Correct spellings using locale preferences for US or UK. # Default is to use a neutral variety of English. diff --git a/bindings/azure/openai/openai.go b/bindings/azure/openai/openai.go index 263277fbbc..bb14293c6e 100644 --- a/bindings/azure/openai/openai.go +++ b/bindings/azure/openai/openai.go @@ -20,8 +20,8 @@ import ( "reflect" "time" + "github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai" "github.com/dapr/components-contrib/bindings" azauth "github.com/dapr/components-contrib/internal/authentication/azure" diff --git a/go.mod b/go.mod index 3ad6bd2b6d..50a6d8b617 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,24 @@ module github.com/dapr/components-contrib go 1.20 require ( - cloud.google.com/go/datastore v1.12.1 - cloud.google.com/go/pubsub v1.32.0 + cloud.google.com/go/datastore v1.13.0 + cloud.google.com/go/pubsub v1.33.0 cloud.google.com/go/secretmanager v1.11.1 cloud.google.com/go/storage v1.31.0 dubbo.apache.org/dubbo-go/v3 v3.0.3-0.20230118042253-4f159a2b38f3 + github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai v0.1.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 - github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai v0.1.0 github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v0.5.0 github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5 - github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 + github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2 github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.0.1 github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventgrid/armeventgrid/v2 v2.1.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1.1 - github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v0.12.0 - github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0 github.com/Azure/go-amqp v1.0.1 github.com/DATA-DOG/go-sqlmock v1.5.0 @@ -39,7 +39,7 @@ require ( github.com/apache/pulsar-client-go v0.11.0 github.com/apache/rocketmq-client-go/v2 v2.1.2-0.20230412142645-25003f6f083d github.com/apache/thrift v0.13.0 - github.com/aws/aws-sdk-go v1.44.299 + github.com/aws/aws-sdk-go v1.44.315 github.com/benbjohnson/clock v1.3.5 github.com/bradfitz/gomemcache v0.0.0-20230611145640-acc696258285 github.com/camunda/zeebe/clients/go/v8 v8.2.8 @@ -56,7 +56,6 @@ require ( github.com/didip/tollbooth/v7 v7.0.1 github.com/eclipse/paho.mqtt.golang v1.4.3 github.com/fasthttp-contrib/sessions v0.0.0-20160905201309-74f6ac73d5d5 - github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-redis/redis/v8 v8.11.5 github.com/go-sql-driver/mysql v1.7.1 github.com/go-zookeeper/zk v1.0.3 @@ -82,7 +81,7 @@ require ( github.com/lestrrat-go/jwx/v2 v2.0.11 github.com/machinebox/graphql v0.2.2 github.com/matoous/go-nanoid/v2 v2.0.0 - github.com/microsoft/go-mssqldb v1.3.0 + github.com/microsoft/go-mssqldb v1.5.0 github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 github.com/mrz1836/postmark v1.4.0 github.com/nacos-group/nacos-sdk-go/v2 v2.2.2 @@ -105,7 +104,7 @@ require ( github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.608 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/ssm v1.0.608 github.com/tetratelabs/wazero v1.3.0 - github.com/valyala/fasthttp v1.47.0 + github.com/valyala/fasthttp v1.48.0 github.com/vmware/vmware-go-kcl v1.5.0 github.com/xdg-go/scram v1.1.2 go.etcd.io/etcd/client/v3 v3.5.9 @@ -115,9 +114,9 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/ratelimit v0.3.0 golang.org/x/crypto v0.11.0 - golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb + golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b golang.org/x/mod v0.12.0 - golang.org/x/net v0.12.0 + golang.org/x/net v0.13.0 golang.org/x/oauth2 v0.10.0 google.golang.org/api v0.128.0 google.golang.org/grpc v1.56.1 @@ -129,8 +128,9 @@ require ( k8s.io/apiextensions-apiserver v0.26.3 k8s.io/apimachinery v0.26.3 k8s.io/client-go v0.26.3 - k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b modernc.org/sqlite v1.24.0 + sigs.k8s.io/yaml v1.3.0 ) require ( @@ -204,6 +204,7 @@ require ( github.com/fatih/color v1.15.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gavv/httpexpect v2.0.0+incompatible // indirect + github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect github.com/go-kit/kit v0.10.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect @@ -403,7 +404,6 @@ require ( modernc.org/token v1.1.0 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect stathat.com/c/consistent v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 6d8fabcffb..c17e675d8b 100644 --- a/go.sum +++ b/go.sum @@ -155,8 +155,8 @@ cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= -cloud.google.com/go/datastore v1.12.1 h1:i8HMKsqg/Sl3ZlOTGl471Z8j2uKtbRDT9VXJUIVlMik= -cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.13.0 h1:ktbC66bOQB3HJPQe8qNI1/aiQ77PMu7hD4mzE6uxe3w= +cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= @@ -286,8 +286,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= -cloud.google.com/go/pubsub v1.32.0 h1:JOEkgEYBuUTHSyHS4TcqOFuWr+vD6qO/imsFqShUCp4= -cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= @@ -417,19 +417,19 @@ github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai v0.1.1 h1:CZwHAPNp2pS80XfUr4xlC1n2M1xsGZ1UfnDW4EzHZGA= +github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai v0.1.1/go.mod h1:zPJgGMjMheJJrYgrQ4W8NrNCWtWXAkjI3KWYFnTtwdA= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= -github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai v0.1.0 h1:lkflJSWI6jicmEBImjpliUOWCr1PdJO/GcZj3bWx19Q= -github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai v0.1.0/go.mod h1:NwVkXm5Ty88Xd7cx6b53fGNeGG3W3ZDXgOXBNHLUy84= github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v0.5.0 h1:OrKZybbyagpgJiREiIVzH5mV/z9oS4rXqdX7i31DSF0= github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v0.5.0/go.mod h1:p74+tP95m8830ypJk53L93+BEsjTKY4SKQ75J2NmS5U= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5 h1:qS0Bp4do0cIvnuQgSGeO6ZCu/q/HlRKl4NPfv1eJ2p0= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5/go.mod h1:Beh5cHIXJ0oWEDWk9lNFtuklCojLLQ5hl+LqSNTTs0I= -github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 h1:bFa9IcjvrCber6gGgDAUZ+I2bO8J7s8JxXmu9fhi2ss= -github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1/go.mod h1:l3wvZkG9oW07GLBW5Cd0WwG5asOfJ8aqE8raUvNzLpk= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2 h1:iXFUCl7NK2DPVKfixcYDPGj3uLV7yf5eolBsoWD8Sc4= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2/go.mod h1:E1WPwLx0wZyus7NBHjhrHE1QgWwKJPE81fnUbT+FxqI= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.0.1 h1:7G4EhZbWFwfgkNfJkNoZmFL8FfWT6P96YVwG71uhNxY= @@ -442,14 +442,15 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1.1/go.mod h1:7fQVOnRA11ScLE8dOCWanXHQa2NMFOM2i0u/1VRICXA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v0.12.0 h1:4Kynh6Hn2ekyIsBgNQJb3dn1+/MyvzfUJebti2emB/A= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v0.12.0/go.mod h1:Q28U+75mpCaSCDowNEmhIo/rmgdkqmkmzI7N6TGR4UY= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0 h1:upXr9dsOnTJk3eHQ3ldyvIXAIGggHtkrfrgbcas6DXU= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0/go.mod h1:w2K61Z8eppIuGbQRx1SKYld2Lrr5vrGvnUwWAhF4nso= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.0 h1:yfJe15aSwEQ6Oo6J+gdfdulPNoZ3TEhmbhLIoxZcA+U= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.0/go.mod h1:Q28U+75mpCaSCDowNEmhIo/rmgdkqmkmzI7N6TGR4UY= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0 h1:qvCB+Za4z8dtU3R5CC7zhlxTLlT3eaEMugglVvjUWtk= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0/go.mod h1:w2K61Z8eppIuGbQRx1SKYld2Lrr5vrGvnUwWAhF4nso= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 h1:T028gtTPiYt/RMUfs8nVsAL7FDQrfLlrm/NnRG/zcC4= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0/go.mod h1:cw4zVQgBby0Z5f2v0itn6se2dDP17nTjbZFXW5uPyHA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0 h1:lJwNFV+xYjHREUTHJKx/ZF6CJSt9znxmLw9DqSTvyRU= github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0/go.mod h1:GfT0aGew8Qj5yiQVqOO5v7N8fanbJGyUoHqXg56qcVY= github.com/Azure/go-amqp v1.0.1 h1:Jf8OQCKzRDMZ3pCiH4onM7yrhl5curkRSGkRLTyP35o= @@ -592,8 +593,8 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ github.com/aws/aws-sdk-go v1.19.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.44.299 h1:HVD9lU4CAFHGxleMJp95FV/sRhtg7P4miHD1v88JAQk= -github.com/aws/aws-sdk-go v1.44.299/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.315 h1:kYTC+Y/bJ9M7QQRvkI/LN5OWvhkIOL/YuFFRhS5QAOo= +github.com/aws/aws-sdk-go v1.44.315/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= @@ -1503,8 +1504,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/microsoft/go-mssqldb v1.3.0 h1:JcPVl+acL8Z/cQcJc9zP0OkjQ+l20bco/cCDpMbmGJk= -github.com/microsoft/go-mssqldb v1.3.0/go.mod h1:lmWsjHD8XX/Txr0f8ZqgbEZSC+BZjmEQy/Ms+rLrvho= +github.com/microsoft/go-mssqldb v1.5.0 h1:CgENxkwtOBNj3Jg6T1X209y2blCfTTcwuOlznd2k9fk= +github.com/microsoft/go-mssqldb v1.5.0/go.mod h1:lmWsjHD8XX/Txr0f8ZqgbEZSC+BZjmEQy/Ms+rLrvho= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -1944,8 +1945,8 @@ github.com/urfave/cli/v2 v2.11.0/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhA github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.21.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= -github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c= -github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= +github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= @@ -2152,8 +2153,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2289,8 +2290,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2980,8 +2981,8 @@ k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 h1:xMMXJlJbsU8w3V5N2FLDQ8YgU8s1EoULdbQBcAeNJkY= -k8s.io/utils v0.0.0-20230313181309-38a27ef9d749/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= diff --git a/tests/certification/go.mod b/tests/certification/go.mod index 04177a6292..3fcac4c78c 100644 --- a/tests/certification/go.mod +++ b/tests/certification/go.mod @@ -3,15 +3,15 @@ module github.com/dapr/components-contrib/tests/certification go 1.20 require ( - cloud.google.com/go/pubsub v1.32.0 + cloud.google.com/go/pubsub v1.33.0 dubbo.apache.org/dubbo-go/v3 v3.0.3-0.20230118042253-4f159a2b38f3 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/Shopify/sarama v1.38.1 github.com/a8m/documentdb v1.3.0 github.com/apache/dubbo-go-hessian2 v1.11.5 github.com/apache/pulsar-client-go v0.11.0 github.com/apache/thrift v0.13.0 - github.com/aws/aws-sdk-go v1.44.299 + github.com/aws/aws-sdk-go v1.44.315 github.com/benbjohnson/clock v1.3.5 github.com/cenkalti/backoff/v4 v4.2.1 github.com/cloudwego/kitex v0.5.0 @@ -36,7 +36,7 @@ require ( go.mongodb.org/mongo-driver v1.12.0 go.uber.org/multierr v1.11.0 go.uber.org/ratelimit v0.3.0 - golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb + golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b modernc.org/sqlite v1.24.0 ) @@ -44,7 +44,7 @@ require ( cloud.google.com/go v0.110.2 // indirect cloud.google.com/go/compute v1.20.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/datastore v1.12.1 // indirect + cloud.google.com/go/datastore v1.13.0 // indirect cloud.google.com/go/iam v1.1.0 // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect @@ -55,12 +55,12 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5 // indirect - github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.0.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.4.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0 // indirect github.com/Azure/go-amqp v1.0.1 // indirect @@ -197,7 +197,7 @@ require ( github.com/mattn/go-isatty v0.0.18 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/microsoft/durabletask-go v0.2.4 // indirect - github.com/microsoft/go-mssqldb v1.3.0 // indirect + github.com/microsoft/go-mssqldb v1.5.0 // indirect github.com/miekg/dns v1.1.50 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -248,7 +248,7 @@ require ( github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.47.0 // indirect + github.com/valyala/fasthttp v1.48.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect @@ -269,7 +269,7 @@ require ( golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.11.0 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/net v0.13.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.10.0 // indirect @@ -298,7 +298,7 @@ require ( k8s.io/component-base v0.26.3 // indirect k8s.io/klog/v2 v2.80.1 // indirect k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect lukechampine.com/uint128 v1.3.0 // indirect modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect diff --git a/tests/certification/go.sum b/tests/certification/go.sum index 1075e827a0..488a687fb0 100644 --- a/tests/certification/go.sum +++ b/tests/certification/go.sum @@ -27,8 +27,8 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.12.1 h1:i8HMKsqg/Sl3ZlOTGl471Z8j2uKtbRDT9VXJUIVlMik= -cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.13.0 h1:ktbC66bOQB3HJPQe8qNI1/aiQ77PMu7hD4mzE6uxe3w= +cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94= cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= @@ -37,8 +37,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.32.0 h1:JOEkgEYBuUTHSyHS4TcqOFuWr+vD6qO/imsFqShUCp4= -cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -69,8 +69,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybI github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5 h1:qS0Bp4do0cIvnuQgSGeO6ZCu/q/HlRKl4NPfv1eJ2p0= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5/go.mod h1:Beh5cHIXJ0oWEDWk9lNFtuklCojLLQ5hl+LqSNTTs0I= -github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 h1:bFa9IcjvrCber6gGgDAUZ+I2bO8J7s8JxXmu9fhi2ss= -github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1/go.mod h1:l3wvZkG9oW07GLBW5Cd0WwG5asOfJ8aqE8raUvNzLpk= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2 h1:iXFUCl7NK2DPVKfixcYDPGj3uLV7yf5eolBsoWD8Sc4= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2/go.mod h1:E1WPwLx0wZyus7NBHjhrHE1QgWwKJPE81fnUbT+FxqI= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.0.1 h1:7G4EhZbWFwfgkNfJkNoZmFL8FfWT6P96YVwG71uhNxY= @@ -81,12 +81,13 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1.1/go.mod h1:7fQVOnRA11ScLE8dOCWanXHQa2NMFOM2i0u/1VRICXA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0 h1:upXr9dsOnTJk3eHQ3ldyvIXAIGggHtkrfrgbcas6DXU= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0/go.mod h1:w2K61Z8eppIuGbQRx1SKYld2Lrr5vrGvnUwWAhF4nso= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0 h1:qvCB+Za4z8dtU3R5CC7zhlxTLlT3eaEMugglVvjUWtk= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0/go.mod h1:w2K61Z8eppIuGbQRx1SKYld2Lrr5vrGvnUwWAhF4nso= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 h1:T028gtTPiYt/RMUfs8nVsAL7FDQrfLlrm/NnRG/zcC4= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0/go.mod h1:cw4zVQgBby0Z5f2v0itn6se2dDP17nTjbZFXW5uPyHA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0 h1:lJwNFV+xYjHREUTHJKx/ZF6CJSt9znxmLw9DqSTvyRU= github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0/go.mod h1:GfT0aGew8Qj5yiQVqOO5v7N8fanbJGyUoHqXg56qcVY= github.com/Azure/go-amqp v1.0.1 h1:Jf8OQCKzRDMZ3pCiH4onM7yrhl5curkRSGkRLTyP35o= @@ -171,8 +172,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.44.299 h1:HVD9lU4CAFHGxleMJp95FV/sRhtg7P4miHD1v88JAQk= -github.com/aws/aws-sdk-go v1.44.299/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.315 h1:kYTC+Y/bJ9M7QQRvkI/LN5OWvhkIOL/YuFFRhS5QAOo= +github.com/aws/aws-sdk-go v1.44.315/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= @@ -913,8 +914,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microsoft/durabletask-go v0.2.4 h1:jeTz559GSXHmOzp5iTbeIq35YYxKSaDHkJcnl8F9wX4= github.com/microsoft/durabletask-go v0.2.4/go.mod h1:UtJXHmKalksdccRiN9Y16cHJYYtZN0bqmqOSiy56V8g= -github.com/microsoft/go-mssqldb v1.3.0 h1:JcPVl+acL8Z/cQcJc9zP0OkjQ+l20bco/cCDpMbmGJk= -github.com/microsoft/go-mssqldb v1.3.0/go.mod h1:lmWsjHD8XX/Txr0f8ZqgbEZSC+BZjmEQy/Ms+rLrvho= +github.com/microsoft/go-mssqldb v1.5.0 h1:CgENxkwtOBNj3Jg6T1X209y2blCfTTcwuOlznd2k9fk= +github.com/microsoft/go-mssqldb v1.5.0/go.mod h1:lmWsjHD8XX/Txr0f8ZqgbEZSC+BZjmEQy/Ms+rLrvho= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -1244,8 +1245,8 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c= -github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= +github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -1412,8 +1413,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1515,8 +1516,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1964,8 +1965,8 @@ k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= diff --git a/tests/conformance/standalone_loader.go b/tests/conformance/standalone_loader.go index a32cde4edd..ef87dd9e4e 100644 --- a/tests/conformance/standalone_loader.go +++ b/tests/conformance/standalone_loader.go @@ -16,13 +16,14 @@ package conformance import ( "bufio" "bytes" + "errors" "io" "log" "os" "path/filepath" "strings" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) const ( @@ -63,7 +64,7 @@ func (s *StandaloneComponents) LoadComponents() ([]Component, error) { continue } - components, _ := s.decodeYaml(path, b) + components := s.decodeYaml(path, b) list = append(list, components...) } } @@ -82,17 +83,15 @@ func (s *StandaloneComponents) isYaml(fileName string) bool { } // decodeYaml decodes the yaml document. -func (s *StandaloneComponents) decodeYaml(filename string, b []byte) ([]Component, []error) { +func (s *StandaloneComponents) decodeYaml(filename string, b []byte) []Component { list := []Component{} - errors := []error{} scanner := bufio.NewScanner(bytes.NewReader(b)) scanner.Split(s.splitYamlDoc) for { var comp Component - comp.Spec = ComponentSpec{} err := s.decode(scanner, &comp) - if err == io.EOF { + if errors.Is(err, io.EOF) { break } @@ -103,13 +102,13 @@ func (s *StandaloneComponents) decodeYaml(filename string, b []byte) ([]Componen list = append(list, comp) } - return list, errors + return list } // decode reads the YAML resource in document. -func (s *StandaloneComponents) decode(scanner *bufio.Scanner, c interface{}) error { +func (s *StandaloneComponents) decode(scanner *bufio.Scanner, c any) error { if scanner.Scan() { - return yaml.Unmarshal(scanner.Bytes(), &c) + return yaml.Unmarshal(scanner.Bytes(), c) } err := scanner.Err() diff --git a/tests/conformance/standalone_loader_test.go b/tests/conformance/standalone_loader_test.go index 61c0e1f76a..18b3df8b66 100644 --- a/tests/conformance/standalone_loader_test.go +++ b/tests/conformance/standalone_loader_test.go @@ -17,6 +17,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestStandaloneIsYaml(t *testing.T) { @@ -48,12 +49,11 @@ spec: - name: prop2 value: value2 ` - components, errs := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) - assert.Len(t, components, 1) - assert.Empty(t, errs) + components := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) + require.Len(t, components, 1) assert.Equal(t, "statestore", components[0].Name) assert.Equal(t, "state.couchbase", components[0].Spec.Type) - assert.Len(t, components[0].Spec.Metadata, 2) + require.Len(t, components[0].Spec.Metadata, 2) assert.Equal(t, "prop1", components[0].Spec.Metadata[0].Name) assert.Equal(t, "value1", components[0].Spec.Metadata[0].Value.String()) } @@ -72,17 +72,15 @@ spec: - name: prop2 value: value2 ` - components, errs := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) + components := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) assert.Len(t, components, 0) - assert.Len(t, errs, 0) } func TestStandaloneDecodeUnsuspectingFile(t *testing.T) { request := NewStandaloneComponents("test_component_path") - components, errs := request.decodeYaml("components/messagebus.yaml", []byte("hey there")) + components := request.decodeYaml("components/messagebus.yaml", []byte("hey there")) assert.Len(t, components, 0) - assert.Len(t, errs, 0) } func TestStandaloneDecodeInvalidYaml(t *testing.T) { @@ -93,9 +91,8 @@ apiVersion: dapr.io/v1alpha1 kind: Component metadata: name: statestore` - components, errs := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) + components := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) assert.Len(t, components, 0) - assert.Len(t, errs, 0) } func TestStandaloneDecodeValidMultiYaml(t *testing.T) { @@ -123,9 +120,8 @@ spec: - name: prop3 value: value3 ` - components, errs := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) + components := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) assert.Len(t, components, 2) - assert.Empty(t, errs) assert.Equal(t, "statestore1", components[0].Name) assert.Equal(t, "state.couchbase", components[0].Spec.Type) assert.Len(t, components[0].Spec.Metadata, 2) @@ -172,9 +168,8 @@ spec: - name: prop3 value: value3 ` - components, errs := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) + components := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) assert.Len(t, components, 2) - assert.Len(t, errs, 0) assert.Equal(t, "statestore1", components[0].Name) assert.Equal(t, "state.couchbase", components[0].Spec.Type) diff --git a/tests/e2e/pubsub/jetstream/go.mod b/tests/e2e/pubsub/jetstream/go.mod index 90a6ffa8b7..9e1f8d9c97 100644 --- a/tests/e2e/pubsub/jetstream/go.mod +++ b/tests/e2e/pubsub/jetstream/go.mod @@ -24,7 +24,7 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cast v1.5.1 // indirect golang.org/x/crypto v0.11.0 // indirect - golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb // indirect + golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b // indirect golang.org/x/sys v0.10.0 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/tests/e2e/pubsub/jetstream/go.sum b/tests/e2e/pubsub/jetstream/go.sum index c760387b14..7c0a3cb5c4 100644 --- a/tests/e2e/pubsub/jetstream/go.sum +++ b/tests/e2e/pubsub/jetstream/go.sum @@ -56,8 +56,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 4a84a019a727a190e56645e245bf58a0e97ea230 Mon Sep 17 00:00:00 2001 From: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Date: Thu, 3 Aug 2023 19:11:37 +0000 Subject: [PATCH 17/22] Fixed flaky cert test for ASB Queue binding Also added prefix to all sent messages so tests don't fail if there are multiple concurrent runners (hopefully) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../servicebusqueues/servicebusqueue_test.go | 110 +++++++++++++++--- 1 file changed, 91 insertions(+), 19 deletions(-) diff --git a/tests/certification/bindings/azure/servicebusqueues/servicebusqueue_test.go b/tests/certification/bindings/azure/servicebusqueues/servicebusqueue_test.go index ad7d2af960..40a7e50021 100644 --- a/tests/certification/bindings/azure/servicebusqueues/servicebusqueue_test.go +++ b/tests/certification/bindings/azure/servicebusqueues/servicebusqueue_test.go @@ -14,8 +14,12 @@ limitations under the License. package servicebusqueue_test import ( + "bytes" "context" + "crypto/rand" + "encoding/base64" "fmt" + "io" "testing" "time" @@ -49,6 +53,15 @@ const ( numMessages = 100 ) +var testprefix string + +func init() { + // Generate a random test prefix + rnd := make([]byte, 7) + io.ReadFull(rand.Reader, rnd) + testprefix = base64.RawURLEncoding.EncodeToString(rnd) +} + func TestServiceBusQueue(t *testing.T) { messagesFor1 := watcher.NewOrdered() messagesFor2 := watcher.NewOrdered() @@ -67,11 +80,11 @@ func TestServiceBusQueue(t *testing.T) { msgsFor1 := make([]string, numMessages/2) msgsFor2 := make([]string, numMessages/2) for i := 0; i < numMessages/2; i++ { - msgsFor1[i] = fmt.Sprintf("sb-binding-1: Message %03d", i) + msgsFor1[i] = fmt.Sprintf("%s: sb-binding-1: Message %03d", testprefix, i) } for i := numMessages / 2; i < numMessages; i++ { - msgsFor2[i-(numMessages/2)] = fmt.Sprintf("sb-binding-2: Message %03d", i) + msgsFor2[i-(numMessages/2)] = fmt.Sprintf("%s: sb-binding-2: Message %03d", testprefix, i) } messagesFor1.ExpectStrings(msgsFor1...) @@ -108,11 +121,19 @@ func TestServiceBusQueue(t *testing.T) { // Setup the input binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("sb-binding-1", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + messagesFor1.Observe(string(in.Data)) ctx.Logf("Got message: %s", string(in.Data)) return []byte("{}"), nil }), s.AddBindingInvocationHandler("sb-binding-2", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + messagesFor2.Observe(string(in.Data)) ctx.Logf("Got message: %s", string(in.Data)) return []byte("{}"), nil @@ -128,7 +149,7 @@ func TestServiceBusQueue(t *testing.T) { embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(grpcPort), embedded.WithDaprHTTPPort(httpPort), - embedded.WithComponentsPath("./components/standard"), + embedded.WithResourcesPath("./components/standard"), componentRuntimeOptions(), )). // Block the standard AMPQ ports. @@ -151,23 +172,38 @@ func TestAzureServiceBusQueuesTTLs(t *testing.T) { ctx.Logf("Sending messages for expiration.") for i := 0; i < numMessages; i++ { - msg := fmt.Sprintf("Expiring message %d", i) + msg := fmt.Sprintf("%s: Expiring message %d", testprefix, i) metadata := make(map[string]string) // Send to the queue with TTL. - queueTTLReq := &daprClient.InvokeBindingRequest{Name: "queuettl", Operation: "create", Data: []byte(msg), Metadata: metadata} + queueTTLReq := &daprClient.InvokeBindingRequest{ + Name: "queuettl", + Operation: "create", + Data: []byte(msg), + Metadata: metadata, + } err := client.InvokeOutputBinding(ctx, queueTTLReq) require.NoError(ctx, err, "error publishing message") // Send message with TTL. - messageTTLReq := &daprClient.InvokeBindingRequest{Name: "messagettl", Operation: "create", Data: []byte(msg), Metadata: metadata} + messageTTLReq := &daprClient.InvokeBindingRequest{ + Name: "messagettl", + Operation: "create", + Data: []byte(msg), + Metadata: metadata, + } messageTTLReq.Metadata["ttlInSeconds"] = "10" err = client.InvokeOutputBinding(ctx, messageTTLReq) require.NoError(ctx, err, "error publishing message") // Send message with TTL to ensure it overwrites Queue TTL. - mixedTTLReq := &daprClient.InvokeBindingRequest{Name: "mixedttl", Operation: "create", Data: []byte(msg), Metadata: metadata} + mixedTTLReq := &daprClient.InvokeBindingRequest{ + Name: "mixedttl", + Operation: "create", + Data: []byte(msg), + Metadata: metadata, + } mixedTTLReq.Metadata["ttlInSeconds"] = "10" err = client.InvokeOutputBinding(ctx, mixedTTLReq) require.NoError(ctx, err, "error publishing message") @@ -182,16 +218,28 @@ func TestAzureServiceBusQueuesTTLs(t *testing.T) { // Setup the input binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("queuettl", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + ctx.Logf("Oh no! Got message: %s", string(in.Data)) ttlMessages.FailIfNotExpected(t, string(in.Data)) return []byte("{}"), nil }), s.AddBindingInvocationHandler("messagettl", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + ctx.Logf("Oh no! Got message: %s", string(in.Data)) ttlMessages.FailIfNotExpected(t, string(in.Data)) return []byte("{}"), nil }), s.AddBindingInvocationHandler("mixedttl", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + ctx.Logf("Oh no! Got message: %s", string(in.Data)) ttlMessages.FailIfNotExpected(t, string(in.Data)) return []byte("{}"), nil @@ -207,7 +255,7 @@ func TestAzureServiceBusQueuesTTLs(t *testing.T) { embedded.WithoutApp(), embedded.WithDaprGRPCPort(grpcPort), embedded.WithDaprHTTPPort(httpPort), - embedded.WithComponentsPath("./components/ttl"), + embedded.WithResourcesPath("./components/ttl"), componentRuntimeOptions(), )). Step("send ttl messages", sendTTLMessages). @@ -242,7 +290,7 @@ func TestAzureServiceBusQueueRetriesOnError(t *testing.T) { // that will satisfy the test. msgs := make([]string, numMessages/2) for i := 0; i < numMessages/2; i++ { - msgs[i] = fmt.Sprintf("Message %03d", i) + msgs[i] = fmt.Sprintf("%s: Message %03d", testprefix, i) } messages.ExpectStrings(msgs...) @@ -252,7 +300,11 @@ func TestAzureServiceBusQueueRetriesOnError(t *testing.T) { for _, msg := range msgs { ctx.Logf("Sending: %q", msg) - req := &daprClient.InvokeBindingRequest{Name: "retry-binding", Operation: "create", Data: []byte(msg)} + req := &daprClient.InvokeBindingRequest{ + Name: "retry-binding", + Operation: "create", + Data: []byte(msg), + } err := client.InvokeOutputBinding(ctx, req) require.NoError(ctx, err, "error publishing message") } @@ -271,6 +323,10 @@ func TestAzureServiceBusQueueRetriesOnError(t *testing.T) { // Setup the input binding endpoint err = multierr.Combine(err, s.AddBindingInvocationHandler("retry-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + if err := sim(); err != nil { ctx.Logf("Failing message: %s", string(in.Data)) return nil, err @@ -291,7 +347,7 @@ func TestAzureServiceBusQueueRetriesOnError(t *testing.T) { embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(grpcPort), embedded.WithDaprHTTPPort(httpPort), - embedded.WithComponentsPath("./components/retry"), + embedded.WithResourcesPath("./components/retry"), componentRuntimeOptions(), )). Step("send and wait", test). @@ -312,10 +368,17 @@ func TestServiceBusQueueMetadata(t *testing.T) { // Send events that the application above will observe. ctx.Log("Invoking binding!") - req := &daprClient.InvokeBindingRequest{Name: "sb-binding-1", Operation: "create", Data: []byte("test msg"), Metadata: map[string]string{"Testmetadata": "Some Metadata"}} + req := &daprClient.InvokeBindingRequest{ + Name: "sb-binding-1", + Operation: "create", + Data: []byte(testprefix + ": test msg"), + Metadata: map[string]string{"Testmetadata": "Some Metadata"}, + } err = client.InvokeOutputBinding(ctx, req) require.NoError(ctx, err, "error publishing message") + messages.ExpectStrings(string(req.Data)) + // Do the messages we observed match what we expect? messages.Assert(ctx, time.Minute) @@ -327,11 +390,15 @@ func TestServiceBusQueueMetadata(t *testing.T) { // Setup the input binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("sb-binding-1", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + messages.Observe(string(in.Data)) - ctx.Logf("Got message: %s - %+v", string(in.Data), in.Metadata) - require.NotEmpty(t, in.Metadata) - require.Contains(t, in.Metadata, "Testmetadata") - require.Equal(t, "Some Metadata", in.Metadata["Testmetadata"]) + ctx.Logf("Got message: %s - %#v", string(in.Data), in.Metadata) + require.NotEmptyf(t, in.Metadata, "Data: %s - Metadata: %#v", in.Data, in.Metadata) + require.Containsf(t, in.Metadata, "Testmetadata", "Data: %s - Metadata: %#v", in.Data, in.Metadata) + require.Equalf(t, "Some+Metadata", in.Metadata["Testmetadata"], "Data: %s - Metadata: %#v", in.Data, in.Metadata) // + because the message is encoded for HTTP headers return []byte("{}"), nil })) @@ -346,7 +413,7 @@ func TestServiceBusQueueMetadata(t *testing.T) { embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(grpcPort), embedded.WithDaprHTTPPort(httpPort), - embedded.WithComponentsPath("./components/standard"), + embedded.WithResourcesPath("./components/standard"), componentRuntimeOptions(), )). Step("send and wait", test). @@ -364,7 +431,12 @@ func TestServiceBusQueueDisableEntityManagement(t *testing.T) { // Send events that the application above will observe. ctx.Log("Invoking binding!") - req := &daprClient.InvokeBindingRequest{Name: "mgmt-binding", Operation: "create", Data: []byte("test msg"), Metadata: map[string]string{"TestMetadata": "Some Metadata"}} + req := &daprClient.InvokeBindingRequest{ + Name: "mgmt-binding", + Operation: "create", + Data: []byte(testprefix + ": test msg"), + Metadata: map[string]string{"TestMetadata": "Some Metadata"}, + } err = client.InvokeOutputBinding(ctx, req) require.Error(ctx, err, "error publishing message") return nil @@ -376,7 +448,7 @@ func TestServiceBusQueueDisableEntityManagement(t *testing.T) { embedded.WithoutApp(), embedded.WithDaprGRPCPort(grpcPort), embedded.WithDaprHTTPPort(httpPort), - embedded.WithComponentsPath("./components/disable_entity_mgmt"), + embedded.WithResourcesPath("./components/disable_entity_mgmt"), componentRuntimeOptions(), )). Step("send and wait", testWithExpectedFailure). From 46b7535f4a9b824defd1ea36c00ab1ba010c90ed Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Thu, 3 Aug 2023 12:40:23 -0700 Subject: [PATCH 18/22] DecodeMetadata: add support for aliases via "mapstructurealiases" struct tag (#3034) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- metadata/utils.go | 87 +++++++++++++- metadata/utils_test.go | 179 +++++++++++++++++++++++++++++ middleware/http/bearer/metadata.go | 20 +--- 3 files changed, 264 insertions(+), 22 deletions(-) diff --git a/metadata/utils.go b/metadata/utils.go index 1c9248c9ab..44c3165473 100644 --- a/metadata/utils.go +++ b/metadata/utils.go @@ -23,6 +23,7 @@ import ( "time" "github.com/mitchellh/mapstructure" + "github.com/spf13/cast" "github.com/dapr/components-contrib/internal/utils" "github.com/dapr/kit/ptr" @@ -142,16 +143,27 @@ func GetMetadataProperty(props map[string]string, keys ...string) (val string, o // This is an extension of mitchellh/mapstructure which also supports decoding durations func DecodeMetadata(input any, result any) error { // avoids a common mistake of passing the metadata struct, instead of the properties map - // if input is of type struct, case it to metadata.Base and access the Properties instead + // if input is of type struct, cast it to metadata.Base and access the Properties instead v := reflect.ValueOf(input) if v.Kind() == reflect.Struct { f := v.FieldByName("Properties") if f.IsValid() && f.Kind() == reflect.Map { - properties := f.Interface().(map[string]string) - input = properties + input = f.Interface().(map[string]string) } } + inputMap, err := cast.ToStringMapStringE(input) + if err != nil { + return fmt.Errorf("input object cannot be cast to map[string]string: %w", err) + } + + // Handle aliases + err = resolveAliases(inputMap, result) + if err != nil { + return fmt.Errorf("failed to resolve aliases: %w", err) + } + + // Finally, decode the metadata using mapstructure decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ DecodeHook: mapstructure.ComposeDecodeHookFunc( toTimeDurationArrayHookFunc(), @@ -166,10 +178,77 @@ func DecodeMetadata(input any, result any) error { if err != nil { return err } - err = decoder.Decode(input) + err = decoder.Decode(inputMap) return err } +func resolveAliases(md map[string]string, result any) error { + // Get the list of all keys in the map + keys := make(map[string]string, len(md)) + for k := range md { + lk := strings.ToLower(k) + + // Check if there are duplicate keys after lowercasing + _, ok := keys[lk] + if ok { + return fmt.Errorf("key %s is duplicate in the metadata", lk) + } + + keys[lk] = k + } + + // Error if result is not pointer to struct, or pointer to pointer to struct + t := reflect.TypeOf(result) + if t.Kind() != reflect.Pointer { + return fmt.Errorf("not a pointer: %s", t.Kind().String()) + } + t = t.Elem() + if t.Kind() == reflect.Pointer { + t = t.Elem() + } + if t.Kind() != reflect.Struct { + return fmt.Errorf("not a struct: %s", t.Kind().String()) + } + + // Iterate through all the properties of result to see if anyone has the "mapstructurealiases" property + for i := 0; i < t.NumField(); i++ { + currentField := t.Field(i) + + // Ignored fields that are not exported or that don't have a "mapstructure" tag + mapstructureTag := currentField.Tag.Get("mapstructure") + if !currentField.IsExported() || mapstructureTag == "" { + continue + } + + // If the current property has a value in the metadata, then we don't need to handle aliases + _, ok := keys[strings.ToLower(mapstructureTag)] + if ok { + continue + } + + // Check if there's a "mapstructurealiases" tag + aliasesTag := strings.ToLower(currentField.Tag.Get("mapstructurealiases")) + if aliasesTag == "" { + continue + } + + // Look for the first alias that has a value + var mdKey string + for _, alias := range strings.Split(aliasesTag, ",") { + mdKey, ok = keys[alias] + if !ok { + continue + } + + // We found an alias + md[mapstructureTag] = md[mdKey] + break + } + } + + return nil +} + func toTruthyBoolHookFunc() mapstructure.DecodeHookFunc { return func( f reflect.Type, diff --git a/metadata/utils_test.go b/metadata/utils_test.go index 33a4af23c5..13cfc559a6 100644 --- a/metadata/utils_test.go +++ b/metadata/utils_test.go @@ -19,6 +19,8 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/exp/maps" ) func TestIsRawPayload(t *testing.T) { @@ -111,6 +113,9 @@ func TestMetadataDecode(t *testing.T) { MyRegularDurationDefaultValueUnset time.Duration `mapstructure:"myregulardurationdefaultvalueunset"` MyRegularDurationDefaultValueEmpty time.Duration `mapstructure:"myregulardurationdefaultvalueempty"` + + AliasedFieldA string `mapstructure:"aliasA1" mapstructurealiases:"aliasA2"` + AliasedFieldB string `mapstructure:"aliasB1" mapstructurealiases:"aliasB2"` } var m testMetadata @@ -131,6 +136,9 @@ func TestMetadataDecode(t *testing.T) { "mydurationarray": "1s,2s,3s,10", "mydurationarraypointer": "1s,10,2s,20,3s,30", "mydurationarraypointerempty": ",", + "aliasA2": "hello", + "aliasB1": "ciao", + "aliasB2": "bonjour", } err := DecodeMetadata(testData, &m) @@ -149,6 +157,8 @@ func TestMetadataDecode(t *testing.T) { assert.Equal(t, []time.Duration{time.Second, time.Second * 2, time.Second * 3, time.Second * 10}, m.MyDurationArray) assert.Equal(t, []time.Duration{time.Second, time.Second * 10, time.Second * 2, time.Second * 20, time.Second * 3, time.Second * 30}, *m.MyDurationArrayPointer) assert.Equal(t, []time.Duration{}, *m.MyDurationArrayPointerEmpty) + assert.Equal(t, "hello", m.AliasedFieldA) + assert.Equal(t, "ciao", m.AliasedFieldB) }) t.Run("Test metadata decode hook for truthy values", func(t *testing.T) { @@ -303,3 +313,172 @@ func TestMetadataStructToStringMap(t *testing.T) { assert.Empty(t, metadatainfo["ignored"].Aliases) }) } + +func TestResolveAliases(t *testing.T) { + tests := []struct { + name string + md map[string]string + result any + wantErr bool + wantMd map[string]string + }{ + { + name: "no aliases", + md: map[string]string{ + "hello": "world", + "ciao": "mondo", + }, + result: &struct { + Hello string `mapstructure:"hello"` + Ciao string `mapstructure:"ciao"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "world", + "ciao": "mondo", + }, + }, + { + name: "set with aliased field", + md: map[string]string{ + "ciao": "mondo", + }, + result: &struct { + Hello string `mapstructure:"hello" mapstructurealiases:"ciao"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "mondo", + "ciao": "mondo", + }, + }, + { + name: "do not overwrite existing fields with aliases", + md: map[string]string{ + "hello": "world", + "ciao": "mondo", + }, + result: &struct { + Hello string `mapstructure:"hello" mapstructurealiases:"ciao"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "world", + "ciao": "mondo", + }, + }, + { + name: "no fields with aliased value", + md: map[string]string{ + "bonjour": "monde", + }, + result: &struct { + Hello string `mapstructure:"hello" mapstructurealiases:"ciao"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "bonjour": "monde", + }, + }, + { + name: "multiple aliases", + md: map[string]string{ + "bonjour": "monde", + }, + result: &struct { + Hello string `mapstructure:"hello" mapstructurealiases:"ciao,bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "monde", + "bonjour": "monde", + }, + }, + { + name: "first alias wins", + md: map[string]string{ + "ciao": "mondo", + "bonjour": "monde", + }, + result: &struct { + Hello string `mapstructure:"hello" mapstructurealiases:"ciao,bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "mondo", + "ciao": "mondo", + "bonjour": "monde", + }, + }, + { + name: "no aliases with mixed case", + md: map[string]string{ + "hello": "world", + "CIAO": "mondo", + }, + result: &struct { + Hello string `mapstructure:"Hello"` + Ciao string `mapstructure:"ciao"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "world", + "CIAO": "mondo", + }, + }, + { + name: "set with aliased field with mixed case", + md: map[string]string{ + "ciao": "mondo", + }, + result: &struct { + Hello string `mapstructure:"Hello" mapstructurealiases:"CIAO"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "Hello": "mondo", + "ciao": "mondo", + }, + }, + { + name: "do not overwrite existing fields with aliases with mixed cases", + md: map[string]string{ + "HELLO": "world", + "CIAO": "mondo", + }, + result: &struct { + Hello string `mapstructure:"hELLo" mapstructurealiases:"cIAo"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "HELLO": "world", + "CIAO": "mondo", + }, + }, + { + name: "multiple aliases with mixed cases", + md: map[string]string{ + "bonjour": "monde", + }, + result: &struct { + Hello string `mapstructure:"HELLO" mapstructurealiases:"CIAO,BONJOUR"` + }{}, + wantMd: map[string]string{ + "HELLO": "monde", + "bonjour": "monde", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + md := maps.Clone(tt.md) + err := resolveAliases(md, tt.result) + + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + require.Equal(t, tt.wantMd, md) + }) + } +} diff --git a/middleware/http/bearer/metadata.go b/middleware/http/bearer/metadata.go index 7bf831201f..54f45fa945 100644 --- a/middleware/http/bearer/metadata.go +++ b/middleware/http/bearer/metadata.go @@ -29,16 +29,12 @@ import ( type bearerMiddlewareMetadata struct { // Issuer authority. - Issuer string `json:"issuer" mapstructure:"issuer"` + Issuer string `json:"issuer" mapstructure:"issuer" mapstructurealiases:"issuerURL"` // Audience to expect in the token (usually, a client ID). - Audience string `json:"audience" mapstructure:"audience"` + Audience string `json:"audience" mapstructure:"audience" mapstructurealiases:"clientID"` // Optional address of the JKWS file. // If missing, will try to fetch the URL set in the OpenID Configuration document `/.well-known/openid-configuration`. JWKSURL string `json:"jwksURL" mapstructure:"jwksURL"` - // Deprecated - use "issuer" instead. - IssuerURL string `json:"issuerURL" mapstructure:"issuerURL"` - // Deprecated - use "audience" instead. - ClientID string `json:"clientID" mapstructure:"clientID"` // Internal properties logger logger.Logger `json:"-" mapstructure:"-"` @@ -52,18 +48,6 @@ func (md *bearerMiddlewareMetadata) fromMetadata(metadata middleware.Metadata) e return err } - // Support IssuerURL as deprecated alias for Issuer - if md.Issuer == "" && md.IssuerURL != "" { - md.Issuer = md.IssuerURL - md.logger.Warnf("Metadata property 'issuerURL' is deprecated and will be removed in the future. Please use 'issuer' instead.") - } - - // Support ClientID as deprecated alias for Audience - if md.Audience == "" && md.ClientID != "" { - md.Audience = md.ClientID - md.logger.Warnf("Metadata property 'clientID' is deprecated and will be removed in the future. Please use 'audience' instead.") - } - // Validate properties if md.Issuer == "" { return errors.New("metadata property 'issuer' is required") From 80fdafc59c47b93853d222936c7b4bee982ea9ac Mon Sep 17 00:00:00 2001 From: Crypt Keeper <64215+codefromthecrypt@users.noreply.github.com> Date: Fri, 4 Aug 2023 10:35:25 +0800 Subject: [PATCH 19/22] wasm: fixes test flake on Windows and strict mode (#3041) Signed-off-by: Adrian Cole --- internal/wasm/wasm_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/internal/wasm/wasm_test.go b/internal/wasm/wasm_test.go index 57379e7c72..be0276ab30 100644 --- a/internal/wasm/wasm_test.go +++ b/internal/wasm/wasm_test.go @@ -172,9 +172,14 @@ func TestNewModuleConfig(t *testing.T) { maxDuration: 50 * time.Millisecond * 5, }, { - name: "strictSandbox = true", - metadata: &InitMetadata{StrictSandbox: true, Guest: binStrict}, - minDuration: 10 * time.Microsecond, + name: "strictSandbox = true", + metadata: &InitMetadata{StrictSandbox: true, Guest: binStrict}, + // In strict mode, nanosleep is implemented by an incrementing + // number. The resolution of the real clock timing the wasm + // invocation is lower resolution in Windows, so we can't verify a + // lower bound. In any case, the important part is that we aren't + // actually sleeping 50ms, which is what wasm thinks is happening. + minDuration: 0, maxDuration: 1 * time.Millisecond, }, } @@ -211,7 +216,8 @@ func TestNewModuleConfig(t *testing.T) { } else { require.NotEqual(t, deterministicOut, out.String()) } - require.True(t, duration > tc.minDuration && duration < tc.maxDuration, duration) + require.GreaterOrEqual(t, duration, tc.minDuration) + require.LessOrEqual(t, duration, tc.maxDuration) }) } } From 5d988974f8f74b39efb2458c74301d47147c8335 Mon Sep 17 00:00:00 2001 From: Roberto Rojas Date: Fri, 4 Aug 2023 10:45:16 -0400 Subject: [PATCH 20/22] [GCP Bindings Bucket] Adds Component Metadata Schema (#2936) Signed-off-by: Roberto J Rojas Signed-off-by: Roberto Rojas Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> --- .../builtin-authentication-profiles.yaml | 69 +++++++++++++ bindings/gcp/bucket/bucket.go | 28 +++--- bindings/gcp/bucket/bucket_test.go | 96 +++++++++---------- bindings/gcp/bucket/metadata.yaml | 41 ++++++++ 4 files changed, 173 insertions(+), 61 deletions(-) create mode 100644 bindings/gcp/bucket/metadata.yaml diff --git a/.build-tools/builtin-authentication-profiles.yaml b/.build-tools/builtin-authentication-profiles.yaml index 548fbf5352..0f545d356f 100644 --- a/.build-tools/builtin-authentication-profiles.yaml +++ b/.build-tools/builtin-authentication-profiles.yaml @@ -103,3 +103,72 @@ azuread: - AzurePublicCloud - AzureChinaCloud - AzureUSGovernmentCloud + +gcp: + - title: "GCP API Authentication with Service Account Key" + description: | + Authenticate authenticates API calls with the given service account or refresh token JSON credentials. + metadata: + - name: privateKeyID + required: true + sensitive: true + description: | + The GCP private key id. Replace with the value of "private_key_id" field of the Service Account Key file. + example: '"privateKeyID"' + - name: privateKey + required: true + sensitive: true + description: | + The GCP credentials private key. Replace with the value of "private_key" field of the Service Account Key file. + example: '"-----BEGIN PRIVATE KEY-----\nMIIE...\\n-----END PRIVATE KEY-----\n"' + - name: type + type: string + required: false + description: | + The GCP credentials type. + example: '"service_account"' + allowedValues: + - service_account + - name: projectID + type: string + required: true + description: | + GCP project id. + example: '"projectID"' + - name: clientEmail + type: string + required: true + description: | + GCP client email. + example: '"client@email.com"' + - name: clientID + type: string + required: true + description: | + The GCP client ID. + example: '"0123456789-0123456789"' + - name: authURI + type: string + required: false + description: | + The GCP account OAuth2 authorization server endpoint URI. + example: '"https://accounts.google.com/o/oauth2/auth"' + - name: tokenURI + type: string + required: false + description: | + The GCP account token server endpoint URI. + example: '"https://oauth2.googleapis.com/token"' + - name: authProviderX509CertURL + type: string + required: false + description: | + The GCP URL of the public x509 certificate, used to verify the signature + on JWTs, such as ID tokens, signed by the authentication provider. + example: '"https://www.googleapis.com/oauth2/v1/certs"' + - name: clientX509CertURL + type: string + required: false + description: | + The GCP URL of the public x509 certificate, used to verify JWTs signed by the client. + example: '"https://www.googleapis.com/robot/v1/metadata/x509/.iam.gserviceaccount.com"' diff --git a/bindings/gcp/bucket/bucket.go b/bindings/gcp/bucket/bucket.go index 941b43ff4c..6507773c6c 100644 --- a/bindings/gcp/bucket/bucket.go +++ b/bindings/gcp/bucket/bucket.go @@ -54,19 +54,21 @@ type GCPStorage struct { } type gcpMetadata struct { - Bucket string `json:"bucket" mapstructure:"bucket"` - Type string `json:"type" mapstructure:"type"` - ProjectID string `json:"project_id" mapstructure:"project_id"` - PrivateKeyID string `json:"private_key_id" mapstructure:"private_key_id"` - PrivateKey string `json:"private_key" mapstructure:"private_key"` - ClientEmail string `json:"client_email " mapstructure:"client_email"` - ClientID string `json:"client_id" mapstructure:"client_id"` - AuthURI string `json:"auth_uri" mapstructure:"auth_uri"` - TokenURI string `json:"token_uri" mapstructure:"token_uri"` - AuthProviderCertURL string `json:"auth_provider_x509_cert_url" mapstructure:"auth_provider_x509_cert_url"` - ClientCertURL string `json:"client_x509_cert_url" mapstructure:"client_x509_cert_url"` - DecodeBase64 bool `json:"decodeBase64,string" mapstructure:"decodeBase64"` - EncodeBase64 bool `json:"encodeBase64,string" mapstructure:"encodeBase64"` + // Ignored by metadata parser because included in built-in authentication profile + Type string `json:"type" mapstructure:"type" mdignore:"true"` + ProjectID string `json:"project_id" mapstructure:"projectID" mdignore:"true" mapstructurealiases:"project_id"` + PrivateKeyID string `json:"private_key_id" mapstructure:"privateKeyID" mdignore:"true" mapstructurealiases:"private_key_id"` + PrivateKey string `json:"private_key" mapstructure:"privateKey" mdignore:"true" mapstructurealiases:"private_key"` + ClientEmail string `json:"client_email " mapstructure:"clientEmail" mdignore:"true" mapstructurealiases:"client_email"` + ClientID string `json:"client_id" mapstructure:"clientID" mdignore:"true" mapstructurealiases:"client_id"` + AuthURI string `json:"auth_uri" mapstructure:"authURI" mdignore:"true" mapstructurealiases:"auth_uri"` + TokenURI string `json:"token_uri" mapstructure:"tokenURI" mdignore:"true" mapstructurealiases:"token_uri"` + AuthProviderCertURL string `json:"auth_provider_x509_cert_url" mapstructure:"authProviderX509CertURL" mdignore:"true" mapstructurealiases:"auth_provider_x509_cert_url"` + ClientCertURL string `json:"client_x509_cert_url" mapstructure:"clientX509CertURL" mdignore:"true" mapstructurealiases:"client_x509_cert_url"` + + Bucket string `json:"bucket" mapstructure:"bucket"` + DecodeBase64 bool `json:"decodeBase64,string" mapstructure:"decodeBase64"` + EncodeBase64 bool `json:"encodeBase64,string" mapstructure:"encodeBase64"` } type listPayload struct { diff --git a/bindings/gcp/bucket/bucket_test.go b/bindings/gcp/bucket/bucket_test.go index 9bd249e87e..c1057c11bc 100644 --- a/bindings/gcp/bucket/bucket_test.go +++ b/bindings/gcp/bucket/bucket_test.go @@ -27,17 +27,17 @@ func TestParseMetadata(t *testing.T) { t.Run("Has correct metadata", func(t *testing.T) { m := bindings.Metadata{} m.Properties = map[string]string{ - "auth_provider_x509_cert_url": "my_auth_provider_x509", - "auth_uri": "my_auth_uri", - "Bucket": "my_bucket", - "client_x509_cert_url": "my_client_x509", - "client_email": "my_email@mail.dapr", - "client_id": "my_client_id", - "private_key": "my_private_key", - "private_key_id": "my_private_key_id", - "project_id": "my_project_id", - "token_uri": "my_token_uri", - "type": "my_type", + "authProviderX509CertURL": "my_auth_provider_x509", + "authURI": "my_auth_uri", + "Bucket": "my_bucket", + "clientX509CertURL": "my_client_x509", + "clientEmail": "my_email@mail.dapr", + "clientID": "my_client_id", + "privateKey": "my_private_key", + "privateKeyID": "my_private_key_id", + "projectID": "my_project_id", + "tokenURI": "my_token_uri", + "type": "my_type", } gs := GCPStorage{logger: logger.NewLogger("test")} meta, err := gs.parseMetadata(m) @@ -73,18 +73,18 @@ func TestMergeWithRequestMetadata(t *testing.T) { t.Run("Has merged metadata", func(t *testing.T) { m := bindings.Metadata{} m.Properties = map[string]string{ - "auth_provider_x509_cert_url": "my_auth_provider_x509", - "auth_uri": "my_auth_uri", - "Bucket": "my_bucket", - "client_x509_cert_url": "my_client_x509", - "client_email": "my_email@mail.dapr", - "client_id": "my_client_id", - "private_key": "my_private_key", - "private_key_id": "my_private_key_id", - "project_id": "my_project_id", - "token_uri": "my_token_uri", - "type": "my_type", - "decodeBase64": "false", + "authProviderX509CertURL": "my_auth_provider_x509", + "authURI": "my_auth_uri", + "Bucket": "my_bucket", + "clientX509CertURL": "my_client_x509", + "clientEmail": "my_email@mail.dapr", + "clientID": "my_client_id", + "privateKey": "my_private_key", + "privateKeyID": "my_private_key_id", + "projectID": "my_project_id", + "tokenURI": "my_token_uri", + "type": "my_type", + "decodeBase64": "false", } gs := GCPStorage{logger: logger.NewLogger("test")} meta, err := gs.parseMetadata(m) @@ -129,18 +129,18 @@ func TestMergeWithRequestMetadata(t *testing.T) { t.Run("Has invalid merged metadata decodeBase64", func(t *testing.T) { m := bindings.Metadata{} m.Properties = map[string]string{ - "auth_provider_x509_cert_url": "my_auth_provider_x509", - "auth_uri": "my_auth_uri", - "Bucket": "my_bucket", - "client_x509_cert_url": "my_client_x509", - "client_email": "my_email@mail.dapr", - "client_id": "my_client_id", - "private_key": "my_private_key", - "private_key_id": "my_private_key_id", - "project_id": "my_project_id", - "token_uri": "my_token_uri", - "type": "my_type", - "decodeBase64": "false", + "authProviderX509CertURL": "my_auth_provider_x509", + "authURI": "my_auth_uri", + "Bucket": "my_bucket", + "clientX509CertURL": "my_client_x509", + "clientEmail": "my_email@mail.dapr", + "clientID": "my_client_id", + "privateKey": "my_private_key", + "privateKeyID": "my_private_key_id", + "projectID": "my_project_id", + "tokenURI": "my_token_uri", + "type": "my_type", + "decodeBase64": "false", } gs := GCPStorage{logger: logger.NewLogger("test")} meta, err := gs.parseMetadata(m) @@ -173,19 +173,19 @@ func TestMergeWithRequestMetadata(t *testing.T) { t.Run("Has invalid merged metadata encodeBase64", func(t *testing.T) { m := bindings.Metadata{} m.Properties = map[string]string{ - "auth_provider_x509_cert_url": "my_auth_provider_x509", - "auth_uri": "my_auth_uri", - "Bucket": "my_bucket", - "client_x509_cert_url": "my_client_x509", - "client_email": "my_email@mail.dapr", - "client_id": "my_client_id", - "private_key": "my_private_key", - "private_key_id": "my_private_key_id", - "project_id": "my_project_id", - "token_uri": "my_token_uri", - "type": "my_type", - "decodeBase64": "false", - "encodeBase64": "true", + "authProviderX509CertURL": "my_auth_provider_x509", + "authURI": "my_auth_uri", + "Bucket": "my_bucket", + "clientX509CertURL": "my_client_x509", + "clientEmail": "my_email@mail.dapr", + "clientID": "my_client_id", + "privateKey": "my_private_key", + "privateKeyID": "my_private_key_id", + "projectID": "my_project_id", + "tokenURI": "my_token_uri", + "type": "my_type", + "decodeBase64": "false", + "encodeBase64": "true", } gs := GCPStorage{logger: logger.NewLogger("test")} meta, err := gs.parseMetadata(m) diff --git a/bindings/gcp/bucket/metadata.yaml b/bindings/gcp/bucket/metadata.yaml new file mode 100644 index 0000000000..e45a072a21 --- /dev/null +++ b/bindings/gcp/bucket/metadata.yaml @@ -0,0 +1,41 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: bindings +name: gcp.bucket +version: v1 +status: alpha +title: "GCP Storage Bucket" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-bindings/gcpbucket/ +binding: + output: true + operations: + - name: create + description: "Create an item." +capabilities: [] +builtinAuthenticationProfiles: + - name: "gcp" +metadata: + - name: bucket + required: true + description: | + The bucket name. + example: '"mybucket"' + type: string + - name: decodeBase64 + type: bool + required: false + default: 'false' + description: | + Configuration to decode base64 file content before saving to bucket storage. + (In case of opening a file with binary content). + example: '"true, false"' + - name: encodeBase64 + type: bool + required: false + default: 'false' + description: | + Configuration to encode base64 file content before return the content. + (In case of saving a file with binary content). + example: '"true, false"' \ No newline at end of file From b10ce96b49213b4543e9cd96dbbde3837c36cd76 Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Fri, 4 Aug 2023 09:38:22 -0700 Subject: [PATCH 21/22] State stores: expose TTL as a feature (#2987) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- .../conformance/state/aws/dynamodb/dynamodb.tf | 4 ++++ internal/component/postgresql/postgresql.go | 7 ++++++- state/aws/dynamodb/dynamodb.go | 14 +++++++++++++- state/azure/cosmosdb/cosmosdb.go | 1 + state/cassandra/cassandra.go | 4 +++- state/cloudflare/workerskv/workerskv.go | 4 +++- state/couchbase/couchbase.go | 8 +++++--- state/etcd/etcd.go | 10 +++++++--- state/feature.go | 4 +++- state/in-memory/in_memory.go | 1 + state/memcached/memcached.go | 4 +++- state/mongodb/metadata.yaml | 1 + state/mongodb/mongodb.go | 9 +++++++-- state/mysql/mysql.go | 6 +++++- state/oci/objectstorage/objectstorage.go | 11 +++++++---- state/oracledatabase/oracledatabase.go | 6 +++++- state/redis/redis.go | 4 ++-- state/rethinkdb/rethinkdb.go | 2 +- state/sqlite/sqlite.go | 1 + state/sqlserver/metadata.yaml | 1 + state/sqlserver/sqlserver.go | 12 +++++------- .../state/aws/dynamodb/terraform/statestore.yml | 4 +++- tests/config/state/tests.yml | 3 ++- tests/conformance/state/state.go | 15 +++++++++++++++ 24 files changed, 104 insertions(+), 32 deletions(-) diff --git a/.github/infrastructure/terraform/conformance/state/aws/dynamodb/dynamodb.tf b/.github/infrastructure/terraform/conformance/state/aws/dynamodb/dynamodb.tf index cfd0660139..350a0cd319 100644 --- a/.github/infrastructure/terraform/conformance/state/aws/dynamodb/dynamodb.tf +++ b/.github/infrastructure/terraform/conformance/state/aws/dynamodb/dynamodb.tf @@ -34,6 +34,10 @@ resource "aws_dynamodb_table" "conformance_test_basic_table" { billing_mode = "PROVISIONED" read_capacity = "10" write_capacity = "10" + ttl { + attribute_name = "expiresAt" + enabled = true + } attribute { name = "key" type = "S" diff --git a/internal/component/postgresql/postgresql.go b/internal/component/postgresql/postgresql.go index 76af6bb568..d7d0c3375f 100644 --- a/internal/component/postgresql/postgresql.go +++ b/internal/component/postgresql/postgresql.go @@ -72,7 +72,12 @@ func (p *PostgreSQL) Init(ctx context.Context, metadata state.Metadata) error { // Features returns the features available in this state store. func (p *PostgreSQL) Features() []state.Feature { - return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureQueryAPI} + return []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureQueryAPI, + state.FeatureTTL, + } } // Delete removes an entity from the store. diff --git a/state/aws/dynamodb/dynamodb.go b/state/aws/dynamodb/dynamodb.go index 33592d5706..86d6c79c48 100644 --- a/state/aws/dynamodb/dynamodb.go +++ b/state/aws/dynamodb/dynamodb.go @@ -94,7 +94,19 @@ func (d *StateStore) Init(_ context.Context, metadata state.Metadata) error { // Features returns the features available in this state store. func (d *StateStore) Features() []state.Feature { - return []state.Feature{state.FeatureETag, state.FeatureTransactional} + // TTLs are enabled only if ttlAttributeName is set + if d.ttlAttributeName == "" { + return []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + } + } + + return []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureTTL, + } } // Get retrieves a dynamoDB item. diff --git a/state/azure/cosmosdb/cosmosdb.go b/state/azure/cosmosdb/cosmosdb.go index 65a34e0fcb..c56352c38e 100644 --- a/state/azure/cosmosdb/cosmosdb.go +++ b/state/azure/cosmosdb/cosmosdb.go @@ -204,6 +204,7 @@ func (c *StateStore) Features() []state.Feature { state.FeatureETag, state.FeatureTransactional, state.FeatureQueryAPI, + state.FeatureTTL, } } diff --git a/state/cassandra/cassandra.go b/state/cassandra/cassandra.go index 3ac0cda60a..6bbc00b431 100644 --- a/state/cassandra/cassandra.go +++ b/state/cassandra/cassandra.go @@ -117,7 +117,9 @@ func (c *Cassandra) Init(_ context.Context, metadata state.Metadata) error { // Features returns the features available in this state store. func (c *Cassandra) Features() []state.Feature { - return nil + return []state.Feature{ + state.FeatureTTL, + } } func (c *Cassandra) tryCreateKeyspace(keyspace string, replicationFactor int) error { diff --git a/state/cloudflare/workerskv/workerskv.go b/state/cloudflare/workerskv/workerskv.go index c1a8609f6b..7a5ee03bb1 100644 --- a/state/cloudflare/workerskv/workerskv.go +++ b/state/cloudflare/workerskv/workerskv.go @@ -90,7 +90,9 @@ func (q *CFWorkersKV) GetComponentMetadata() (metadataInfo metadata.MetadataMap) // Features returns the features supported by this state store. func (q CFWorkersKV) Features() []state.Feature { - return []state.Feature{} + return []state.Feature{ + state.FeatureTTL, + } } func (q *CFWorkersKV) Delete(parentCtx context.Context, stateReq *state.DeleteRequest) error { diff --git a/state/couchbase/couchbase.go b/state/couchbase/couchbase.go index 594f6386f5..b1fe9ea7ac 100644 --- a/state/couchbase/couchbase.go +++ b/state/couchbase/couchbase.go @@ -66,9 +66,11 @@ type couchbaseMetadata struct { // NewCouchbaseStateStore returns a new couchbase state store. func NewCouchbaseStateStore(logger logger.Logger) state.Store { s := &Couchbase{ - json: jsoniter.ConfigFastest, - features: []state.Feature{state.FeatureETag}, - logger: logger, + json: jsoniter.ConfigFastest, + features: []state.Feature{ + state.FeatureETag, + }, + logger: logger, } s.BulkStore = state.NewDefaultBulkStore(s) return s diff --git a/state/etcd/etcd.go b/state/etcd/etcd.go index 4bac16a911..4ef6f1d6a7 100644 --- a/state/etcd/etcd.go +++ b/state/etcd/etcd.go @@ -67,9 +67,13 @@ func NewEtcdStateStoreV2(logger logger.Logger) state.Store { func newETCD(logger logger.Logger, schema schemaMarshaller) state.Store { s := &Etcd{ - schema: schema, - logger: logger, - features: []state.Feature{state.FeatureETag, state.FeatureTransactional}, + schema: schema, + logger: logger, + features: []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureTTL, + }, } s.BulkStore = state.NewDefaultBulkStore(s) return s diff --git a/state/feature.go b/state/feature.go index e42d3fdf5a..9a53370076 100644 --- a/state/feature.go +++ b/state/feature.go @@ -24,9 +24,11 @@ const ( FeatureTransactional Feature = "TRANSACTIONAL" // FeatureQueryAPI is the feature that performs query operations. FeatureQueryAPI Feature = "QUERY_API" + // FeatureTTL is the feature that supports TTLs. + FeatureTTL Feature = "TTL" ) -// Feature names a feature that can be implemented by PubSub components. +// Feature names a feature that can be implemented by state store components. type Feature string // IsPresent checks if a given feature is present in the list. diff --git a/state/in-memory/in_memory.go b/state/in-memory/in_memory.go index a05f2bd94e..f0bc73f3f7 100644 --- a/state/in-memory/in_memory.go +++ b/state/in-memory/in_memory.go @@ -91,6 +91,7 @@ func (store *inMemoryStore) Features() []state.Feature { return []state.Feature{ state.FeatureETag, state.FeatureTransactional, + state.FeatureTTL, } } diff --git a/state/memcached/memcached.go b/state/memcached/memcached.go index d4171e13ff..2bb0a99b2e 100644 --- a/state/memcached/memcached.go +++ b/state/memcached/memcached.go @@ -93,7 +93,9 @@ func (m *Memcached) Init(_ context.Context, metadata state.Metadata) error { // Features returns the features available in this state store. func (m *Memcached) Features() []state.Feature { - return nil + return []state.Feature{ + state.FeatureTTL, + } } func getMemcachedMetadata(meta state.Metadata) (*memcachedMetadata, error) { diff --git a/state/mongodb/metadata.yaml b/state/mongodb/metadata.yaml index 2b76b3d04f..7537dc9169 100644 --- a/state/mongodb/metadata.yaml +++ b/state/mongodb/metadata.yaml @@ -17,6 +17,7 @@ capabilities: - transactional - etag - query + - ttl authenticationProfiles: - title: "Connection string" description: | diff --git a/state/mongodb/mongodb.go b/state/mongodb/mongodb.go index 49075586db..f9a0f6fe36 100644 --- a/state/mongodb/mongodb.go +++ b/state/mongodb/mongodb.go @@ -112,8 +112,13 @@ type Item struct { // NewMongoDB returns a new MongoDB state store. func NewMongoDB(logger logger.Logger) state.Store { s := &MongoDB{ - features: []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureQueryAPI}, - logger: logger, + features: []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureQueryAPI, + state.FeatureTTL, + }, + logger: logger, } s.BulkStore = state.NewDefaultBulkStore(s) return s diff --git a/state/mysql/mysql.go b/state/mysql/mysql.go index 6e5f6414b9..4554a5415e 100644 --- a/state/mysql/mysql.go +++ b/state/mysql/mysql.go @@ -225,7 +225,11 @@ func (m *MySQL) parseMetadata(md map[string]string) error { // Features returns the features available in this state store. func (m *MySQL) Features() []state.Feature { - return []state.Feature{state.FeatureETag, state.FeatureTransactional} + return []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureTTL, + } } // Ping the database. diff --git a/state/oci/objectstorage/objectstorage.go b/state/oci/objectstorage/objectstorage.go index 7ed5748eca..471ee88145 100644 --- a/state/oci/objectstorage/objectstorage.go +++ b/state/oci/objectstorage/objectstorage.go @@ -168,10 +168,13 @@ func (r *StateStore) Ping(ctx context.Context) error { func NewOCIObjectStorageStore(logger logger.Logger) state.Store { s := &StateStore{ - json: jsoniter.ConfigFastest, - features: []state.Feature{state.FeatureETag}, - logger: logger, - client: nil, + json: jsoniter.ConfigFastest, + features: []state.Feature{ + state.FeatureETag, + state.FeatureTTL, + }, + logger: logger, + client: nil, } s.BulkStore = state.NewDefaultBulkStore(s) diff --git a/state/oracledatabase/oracledatabase.go b/state/oracledatabase/oracledatabase.go index 32e883287b..890e9e7606 100644 --- a/state/oracledatabase/oracledatabase.go +++ b/state/oracledatabase/oracledatabase.go @@ -44,7 +44,11 @@ func NewOracleDatabaseStateStore(logger logger.Logger) state.Store { // This unexported constructor allows injecting a dbAccess instance for unit testing. func newOracleDatabaseStateStore(logger logger.Logger, dba dbAccess) *OracleDatabase { return &OracleDatabase{ - features: []state.Feature{state.FeatureETag, state.FeatureTransactional}, + features: []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureTTL, + }, logger: logger, dbaccess: dba, } diff --git a/state/redis/redis.go b/state/redis/redis.go index d5e1c33ee4..a3f65af26b 100644 --- a/state/redis/redis.go +++ b/state/redis/redis.go @@ -161,9 +161,9 @@ func (r *StateStore) Init(ctx context.Context, metadata state.Metadata) error { // Features returns the features available in this state store. func (r *StateStore) Features() []state.Feature { if r.clientHasJSON { - return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureQueryAPI} + return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureTTL, state.FeatureQueryAPI} } else { - return []state.Feature{state.FeatureETag, state.FeatureTransactional} + return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureTTL} } } diff --git a/state/rethinkdb/rethinkdb.go b/state/rethinkdb/rethinkdb.go index 481ef740e7..dd33c5e09a 100644 --- a/state/rethinkdb/rethinkdb.go +++ b/state/rethinkdb/rethinkdb.go @@ -37,7 +37,7 @@ const ( stateArchiveTablePKName = "key" ) -// RethinkDB is a state store implementation with transactional support for RethinkDB. +// RethinkDB is a state store implementation for RethinkDB. type RethinkDB struct { session *r.Session config *stateConfig diff --git a/state/sqlite/sqlite.go b/state/sqlite/sqlite.go index 218c16508f..a9ff2ee41a 100644 --- a/state/sqlite/sqlite.go +++ b/state/sqlite/sqlite.go @@ -48,6 +48,7 @@ func newSQLiteStateStore(logger logger.Logger, dba DBAccess) *SQLiteStore { features: []state.Feature{ state.FeatureETag, state.FeatureTransactional, + state.FeatureTTL, }, dbaccess: dba, } diff --git a/state/sqlserver/metadata.yaml b/state/sqlserver/metadata.yaml index f5129bbb25..8b62a1ec26 100644 --- a/state/sqlserver/metadata.yaml +++ b/state/sqlserver/metadata.yaml @@ -15,6 +15,7 @@ capabilities: - "crud" - "transactional" - "etag" + - "ttl" authenticationProfiles: - title: "Connection string" description: | diff --git a/state/sqlserver/sqlserver.go b/state/sqlserver/sqlserver.go index c87cafa491..77623d6152 100644 --- a/state/sqlserver/sqlserver.go +++ b/state/sqlserver/sqlserver.go @@ -65,7 +65,11 @@ const ( // New creates a new instance of a SQL Server transaction store. func New(logger logger.Logger) state.Store { s := &SQLServer{ - features: []state.Feature{state.FeatureETag, state.FeatureTransactional}, + features: []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureTTL, + }, logger: logger, migratorFactory: newMigration, } @@ -238,12 +242,6 @@ func (s *SQLServer) executeDelete(ctx context.Context, db dbExecutor, req *state return nil } -// TvpDeleteTableStringKey defines a table type with string key. -type TvpDeleteTableStringKey struct { - ID string - RowVersion []byte -} - // Get returns an entity from store. func (s *SQLServer) Get(ctx context.Context, req *state.GetRequest) (*state.GetResponse, error) { rows, err := s.db.QueryContext(ctx, s.getCommand, sql.Named(keyColumnName, req.Key)) diff --git a/tests/config/state/aws/dynamodb/terraform/statestore.yml b/tests/config/state/aws/dynamodb/terraform/statestore.yml index 5f708ad74e..ea04cda14e 100644 --- a/tests/config/state/aws/dynamodb/terraform/statestore.yml +++ b/tests/config/state/aws/dynamodb/terraform/statestore.yml @@ -13,4 +13,6 @@ spec: - name: region value: "us-east-1" - name: table - value: ${{STATE_AWS_DYNAMODB_TABLE_1}} \ No newline at end of file + value: ${{STATE_AWS_DYNAMODB_TABLE_1}} + - name: ttlAttributeName + value: "expiresAt" \ No newline at end of file diff --git a/tests/config/state/tests.yml b/tests/config/state/tests.yml index 199a540c88..6228d34db4 100644 --- a/tests/config/state/tests.yml +++ b/tests/config/state/tests.yml @@ -75,9 +75,10 @@ components: - component: in-memory operations: [ "transaction", "etag", "first-write", "ttl" ] - component: aws.dynamodb.docker + # In the Docker variant, we do not set ttlAttributeName in the metadata, so TTLs are not enabled operations: [ "transaction", "etag", "first-write" ] - component: aws.dynamodb.terraform - operations: [ "transaction", "etag", "first-write" ] + operations: [ "transaction", "etag", "first-write", "ttl" ] - component: etcd.v1 operations: [ "transaction", "etag", "first-write", "ttl" ] - component: etcd.v2 diff --git a/tests/conformance/state/state.go b/tests/conformance/state/state.go index bbc2f78d0c..526d6f6d37 100644 --- a/tests/conformance/state/state.go +++ b/tests/conformance/state/state.go @@ -991,6 +991,10 @@ func ConformanceTests(t *testing.T, props map[string]string, statestore state.St if config.HasOperation("ttl") { t.Run("set and get with TTL", func(t *testing.T) { + // Check if ttl feature is listed + features := statestore.Features() + require.True(t, state.FeatureTTL.IsPresent(features)) + err := statestore.Set(context.Background(), &state.SetRequest{ Key: key + "-ttl", Value: "⏱️", @@ -1016,6 +1020,17 @@ func ConformanceTests(t *testing.T, props map[string]string, statestore state.St return res.Data == nil }, time.Second*3, 200*time.Millisecond, "expected object to have been deleted in time") }) + } else { + t.Run("ttl feature not present", func(t *testing.T) { + // We skip this check for Cloudflare Workers KV + // Even though the component supports TTLs, it's not tested in the conformance tests because the minimum TTL for the component is 1 minute, and the state store doesn't have strong consistency + if config.ComponentName == "cloudflare.workerskv" { + t.Skip() + } + + features := statestore.Features() + require.False(t, state.FeatureTTL.IsPresent(features)) + }) } } From 31ccb5f1690ffb3435168e5233ae5e0511068376 Mon Sep 17 00:00:00 2001 From: Josh van Leeuwen Date: Fri, 4 Aug 2023 20:11:52 +0100 Subject: [PATCH 22/22] Conformance Tests: State- ` ttlExpireTime` (#2863) Signed-off-by: joshvanl Signed-off-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> --- state/etcd/etcd.go | 43 +++---- tests/conformance/state/state.go | 187 +++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 26 deletions(-) diff --git a/state/etcd/etcd.go b/state/etcd/etcd.go index 4ef6f1d6a7..4d9360545a 100644 --- a/state/etcd/etcd.go +++ b/state/etcd/etcd.go @@ -192,38 +192,29 @@ func (e *Etcd) doSet(ctx context.Context, key string, val any, etag *string, ttl return err } + var leaseID clientv3.LeaseID if ttlInSeconds != nil { - resp, err := e.client.Grant(ctx, *ttlInSeconds) + var resp *clientv3.LeaseGrantResponse + resp, err = e.client.Grant(ctx, *ttlInSeconds) if err != nil { return fmt.Errorf("couldn't grant lease %s: %w", key, err) } - if etag != nil { - etag, _ := strconv.ParseInt(*etag, 10, 64) - _, err = e.client.Txn(ctx). - If(clientv3.Compare(clientv3.ModRevision(key), "=", etag)). - Then(clientv3.OpPut(key, reqVal, clientv3.WithLease(resp.ID))). - Commit() - } else { - _, err = e.client.Put(ctx, key, reqVal, clientv3.WithLease(resp.ID)) - } - if err != nil { - return fmt.Errorf("couldn't set key %s: %w", key, err) - } + leaseID = resp.ID + } + + if etag != nil { + etag, _ := strconv.ParseInt(*etag, 10, 64) + _, err = e.client.Txn(ctx). + If(clientv3.Compare(clientv3.ModRevision(key), "=", etag)). + Then(clientv3.OpPut(key, reqVal, clientv3.WithLease(leaseID))). + Commit() } else { - var err error - if etag != nil { - etag, _ := strconv.ParseInt(*etag, 10, 64) - _, err = e.client.Txn(ctx). - If(clientv3.Compare(clientv3.ModRevision(key), "=", etag)). - Then(clientv3.OpPut(key, reqVal)). - Commit() - } else { - _, err = e.client.Put(ctx, key, reqVal) - } - if err != nil { - return fmt.Errorf("couldn't set key %s: %w", key, err) - } + _, err = e.client.Put(ctx, key, reqVal, clientv3.WithLease(leaseID)) + } + if err != nil { + return fmt.Errorf("couldn't set key %s: %w", key, err) } + return nil } diff --git a/tests/conformance/state/state.go b/tests/conformance/state/state.go index 526d6f6d37..93c0a25729 100644 --- a/tests/conformance/state/state.go +++ b/tests/conformance/state/state.go @@ -990,6 +990,16 @@ func ConformanceTests(t *testing.T, props map[string]string, statestore state.St } if config.HasOperation("ttl") { + t.Run("set ttl with bad value should error", func(t *testing.T) { + require.Error(t, statestore.Set(context.Background(), &state.SetRequest{ + Key: key + "-ttl", + Value: "⏱️", + Metadata: map[string]string{ + "ttlInSeconds": "foo", + }, + })) + }) + t.Run("set and get with TTL", func(t *testing.T) { // Check if ttl feature is listed features := statestore.Features() @@ -1031,6 +1041,183 @@ func ConformanceTests(t *testing.T, props map[string]string, statestore state.St features := statestore.Features() require.False(t, state.FeatureTTL.IsPresent(features)) }) + + t.Run("no TTL should not return any expire time", func(t *testing.T) { + err := statestore.Set(context.Background(), &state.SetRequest{ + Key: key + "-no-ttl", + Value: "⏱️", + Metadata: map[string]string{}, + }) + require.NoError(t, err) + + // Request immediately + res, err := statestore.Get(context.Background(), &state.GetRequest{Key: key + "-no-ttl"}) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + + assert.NotContains(t, res.Metadata, "ttlExpireTime") + }) + + t.Run("ttlExpireTime", func(t *testing.T) { + if !config.HasOperation("transaction") { + // This test is only for state stores that support transactions + return + } + + unsupported := []string{ + "redis.v6", + "redis.v7", + "etcd.v1", + } + + for _, noSup := range unsupported { + if strings.Contains(config.ComponentName, noSup) { + t.Skipf("skipping test for unsupported state store %s", noSup) + } + } + + t.Run("set and get expire time", func(t *testing.T) { + now := time.Now() + err := statestore.Set(context.Background(), &state.SetRequest{ + Key: key + "-ttl-expire-time", + Value: "⏱️", + Metadata: map[string]string{ + // Expire in an hour. + "ttlInSeconds": "3600", + }, + }) + require.NoError(t, err) + + // Request immediately + res, err := statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + + require.Containsf(t, res.Metadata, "ttlExpireTime", "expected metadata to contain ttlExpireTime") + expireTime, err := time.Parse(time.RFC3339, res.Metadata["ttlExpireTime"]) + require.NoError(t, err) + assert.InDelta(t, now.Add(time.Hour).UnixMilli(), expireTime.UnixMilli(), float64(time.Minute*10)) + }) + + t.Run("ttl set to -1 should remove the TTL of a state store key", func(t *testing.T) { + req := func(meta map[string]string) *state.SetRequest { + return &state.SetRequest{ + Key: key + "-ttl-expire-time-minus-1", + Value: "⏱️", + Metadata: meta, + } + } + + require.NoError(t, statestore.Set(context.Background(), req(map[string]string{ + // Expire in 2 seconds. + "ttlInSeconds": "2", + }))) + + // Request immediately + res, err := statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + assert.Contains(t, res.Metadata, "ttlExpireTime") + + // Remove TTL by setting a value of -1. + require.NoError(t, statestore.Set(context.Background(), req(map[string]string{ + "ttlInSeconds": "-1", + }))) + res, err = statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + assert.NotContains(t, res.Metadata, "ttlExpireTime") + + // Ensure that the key is not expired after previous TTL. + time.Sleep(3 * time.Second) + + res, err = statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + + // Set a new TTL. + require.NoError(t, statestore.Set(context.Background(), req(map[string]string{ + "ttlInSeconds": "2", + }))) + res, err = statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + assert.Contains(t, res.Metadata, "ttlExpireTime") + + // Remove TTL by omitting the ttlInSeconds field. + require.NoError(t, statestore.Set(context.Background(), req(map[string]string{}))) + res, err = statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + assert.NotContains(t, res.Metadata, "ttlExpireTime") + + // Ensure key is not expired after previous TTL. + time.Sleep(3 * time.Second) + res, err = statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + assert.NotContains(t, res.Metadata, "ttlExpireTime") + }) + + t.Run("set and get expire time bulkGet", func(t *testing.T) { + now := time.Now() + require.NoError(t, statestore.Set(context.Background(), &state.SetRequest{ + Key: key + "-ttl-expire-time-bulk-1", + Value: "123", + Metadata: map[string]string{"ttlInSeconds": "3600"}, + })) + + require.NoError(t, statestore.Set(context.Background(), &state.SetRequest{ + Key: key + "-ttl-expire-time-bulk-2", + Value: "234", + Metadata: map[string]string{"ttlInSeconds": "3600"}, + })) + + // Request immediately + res, err := statestore.BulkGet(context.Background(), []state.GetRequest{ + {Key: key + "-ttl-expire-time-bulk-1"}, + {Key: key + "-ttl-expire-time-bulk-2"}, + }, state.BulkGetOpts{}) + require.NoError(t, err) + + require.Len(t, res, 2) + sort.Slice(res, func(i, j int) bool { + return res[i].Key < res[j].Key + }) + + assert.Equal(t, key+"-ttl-expire-time-bulk-1", res[0].Key) + assert.Equal(t, key+"-ttl-expire-time-bulk-2", res[1].Key) + assert.Equal(t, []byte(`"123"`), res[0].Data) + assert.Equal(t, []byte(`"234"`), res[1].Data) + + for i := range res { + if config.HasOperation("transaction") { + require.Containsf(t, res[i].Metadata, "ttlExpireTime", "expected metadata to contain ttlExpireTime") + expireTime, err := time.Parse(time.RFC3339, res[i].Metadata["ttlExpireTime"]) + require.NoError(t, err) + // Check the expire time is returned and is in a 10 minute window. This + // window should be _more_ than enough. + assert.InDelta(t, now.Add(time.Hour).UnixMilli(), expireTime.UnixMilli(), float64(time.Minute*10)) + } else { + assert.NotContains(t, res[i].Metadata, "ttlExpireTime") + } + } + }) + }) } }