From 760948a533106a8e8e4769781700dc67e9687016 Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:50:03 -0700 Subject: [PATCH 1/8] Local storage binding: disable access to other system folders for security reasons (#2947) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- bindings/localstorage/localstorage.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bindings/localstorage/localstorage.go b/bindings/localstorage/localstorage.go index a038fd4ce4..654bc54346 100644 --- a/bindings/localstorage/localstorage.go +++ b/bindings/localstorage/localstorage.go @@ -40,6 +40,9 @@ const ( // List of root paths that are disallowed var disallowedRootPaths = []string{ + filepath.Clean("/proc"), + filepath.Clean("/sys"), + filepath.Clean("/boot"), // See: https://github.com/dapr/components-contrib/issues/2444 filepath.Clean("/var/run/secrets"), } From fd8e3a208674713572dd94fa0282ed1fe27be064 Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:51:58 -0700 Subject: [PATCH 2/8] Fixes in HTTP binding (#2981) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Bernd Verst --- bindings/http/http.go | 47 +++++++++++++------------------------- bindings/http/http_test.go | 16 ------------- 2 files changed, 16 insertions(+), 47 deletions(-) diff --git a/bindings/http/http.go b/bindings/http/http.go index 3acc9dff2d..63fe4ee4f6 100644 --- a/bindings/http/http.go +++ b/bindings/http/http.go @@ -29,7 +29,6 @@ import ( "strconv" "strings" "time" - "unicode" "github.com/dapr/components-contrib/bindings" "github.com/dapr/components-contrib/internal/utils" @@ -102,11 +101,11 @@ func (h *HTTPSource) Init(_ context.Context, meta bindings.Metadata) error { // See guidance on proper HTTP client settings here: // https://medium.com/@nate510/don-t-use-go-s-default-http-client-4804cb19f779 dialer := &net.Dialer{ - Timeout: 5 * time.Second, + Timeout: 15 * time.Second, } netTransport := &http.Transport{ Dial: dialer.Dial, - TLSHandshakeTimeout: 5 * time.Second, + TLSHandshakeTimeout: 15 * time.Second, TLSClientConfig: tlsConfig, } @@ -150,17 +149,11 @@ func (h *HTTPSource) readMTLSClientCertificates(tlsConfig *tls.Config) error { func (h *HTTPSource) setTLSRenegotiation(tlsConfig *tls.Config) error { switch h.metadata.MTLSRenegotiation { case "RenegotiateNever": - { - tlsConfig.Renegotiation = tls.RenegotiateNever - } + tlsConfig.Renegotiation = tls.RenegotiateNever case "RenegotiateOnceAsClient": - { - tlsConfig.Renegotiation = tls.RenegotiateOnceAsClient - } + tlsConfig.Renegotiation = tls.RenegotiateOnceAsClient case "RenegotiateFreelyAsClient": - { - tlsConfig.Renegotiation = tls.RenegotiateFreelyAsClient - } + tlsConfig.Renegotiation = tls.RenegotiateFreelyAsClient default: return fmt.Errorf("invalid renegotiation value: %s", h.metadata.MTLSRenegotiation) } @@ -231,23 +224,18 @@ func (h *HTTPSource) Invoke(parentCtx context.Context, req *bindings.InvokeReque errorIfNot2XX := h.errorIfNot2XX // Default to the component config (default is true) - if req.Metadata != nil { - if path, ok := req.Metadata["path"]; ok { - // Simplicity and no "../../.." type exploits. - u = fmt.Sprintf("%s/%s", strings.TrimRight(u, "/"), strings.TrimLeft(path, "/")) - if strings.Contains(u, "..") { - return nil, fmt.Errorf("invalid path: %s", path) - } - } - - if _, ok := req.Metadata["errorIfNot2XX"]; ok { - errorIfNot2XX = utils.IsTruthy(req.Metadata["errorIfNot2XX"]) - } - } else { + if req.Metadata == nil { // Prevent things below from failing if req.Metadata is nil. req.Metadata = make(map[string]string) } + if req.Metadata["path"] != "" { + u = strings.TrimRight(u, "/") + "/" + strings.TrimLeft(req.Metadata["path"], "/") + } + if req.Metadata["errorIfNot2XX"] != "" { + errorIfNot2XX = utils.IsTruthy(req.Metadata["errorIfNot2XX"]) + } + var body io.Reader method := strings.ToUpper(string(req.Operation)) // For backward compatibility @@ -262,10 +250,8 @@ func (h *HTTPSource) Invoke(parentCtx context.Context, req *bindings.InvokeReque return nil, fmt.Errorf("invalid operation: %s", req.Operation) } - var ctx context.Context - if h.metadata.ResponseTimeout == nil { - ctx = parentCtx - } else { + ctx := parentCtx + if h.metadata.ResponseTimeout != nil { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(parentCtx, *h.metadata.ResponseTimeout) defer cancel() @@ -294,8 +280,7 @@ func (h *HTTPSource) Invoke(parentCtx context.Context, req *bindings.InvokeReque // Any metadata keys that start with a capital letter // are treated as request headers for mdKey, mdValue := range req.Metadata { - keyAsRunes := []rune(mdKey) - if len(keyAsRunes) > 0 && unicode.IsUpper(keyAsRunes[0]) { + if len(mdKey) > 0 && (mdKey[0] >= 'A' && mdKey[0] <= 'Z') { request.Header.Set(mdKey, mdValue) } } diff --git a/bindings/http/http_test.go b/bindings/http/http_test.go index 04e53cdfd0..164b1ef328 100644 --- a/bindings/http/http_test.go +++ b/bindings/http/http_test.go @@ -553,14 +553,6 @@ func verifyDefaultBehaviors(t *testing.T, hs bindings.OutputBinding, handler *HT err: "", statusCode: 200, }, - "invalid path": { - input: "expected", - operation: "POST", - metadata: map[string]string{"path": "/../test"}, - path: "", - err: "invalid path: /../test", - statusCode: 400, - }, "invalid operation": { input: "notvalid", operation: "notvalid", @@ -665,14 +657,6 @@ func verifyNon2XXErrorsSuppressed(t *testing.T, hs bindings.OutputBinding, handl err: "", statusCode: 200, }, - "invalid path": { - input: "expected", - operation: "POST", - metadata: map[string]string{"path": "/../test"}, - path: "", - err: "invalid path: /../test", - statusCode: 400, - }, } for name, tc := range tests { From 1349fca858369cc067a93576be0a19d0c05df58f Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Wed, 12 Jul 2023 14:16:35 -0700 Subject: [PATCH 3/8] MySQL binding: allow passing parameters for queries (#2975) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- bindings/mysql/metadata.yaml | 13 +- bindings/mysql/mysql.go | 129 +++++++++++------ bindings/mysql/mysql_integration_test.go | 176 +++++++++++++---------- bindings/mysql/mysql_test.go | 8 +- 4 files changed, 192 insertions(+), 134 deletions(-) diff --git a/bindings/mysql/metadata.yaml b/bindings/mysql/metadata.yaml index e305696d11..005b2e7637 100644 --- a/bindings/mysql/metadata.yaml +++ b/bindings/mysql/metadata.yaml @@ -17,17 +17,17 @@ binding: - name: query description: "The query operation is used for SELECT statements, which returns the metadata along with data in a form of an array of row values." - name: close - description: "The close operation can be used to explicitly close the DB connection and return it to the pool. This operation doesn’t have any response." + description: "The close operation can be used to explicitly close the DB connection and return it to the pool. This operation doesn't have any response." metadata: - name: url required: true - description: "Represent a DB connection in Data Source Name (DNS) format." - example: "user:password@tcp(localhost:3306)/dbname" + description: "Represent a DB connection in Data Source Name (DNS) format" + example: '"user:password@tcp(localhost:3306)/dbname"' type: string - name: pemPath required: false description: "Path to the PEM file. Used with SSL connection" - example: "path/to/pem/file" + example: '"path/to/pem/file"' type: string - name: maxIdleConns required: false @@ -49,8 +49,3 @@ metadata: description: "The max connection idel time." example: "12s" type: duration - - name: maxRetries - required: false - description: "MaxRetries is the maximum number of retries for a query." - example: "5" - type: number diff --git a/bindings/mysql/mysql.go b/bindings/mysql/mysql.go index 1c53dc8df4..2127caf0f3 100644 --- a/bindings/mysql/mysql.go +++ b/bindings/mysql/mysql.go @@ -25,6 +25,7 @@ import ( "os" "reflect" "strconv" + "sync/atomic" "time" "github.com/go-sql-driver/mysql" @@ -52,7 +53,8 @@ const ( // "%s:%s@tcp(%s:3306)/%s?allowNativePasswords=true&tls=custom",'myadmin@mydemoserver', 'yourpassword', 'mydemoserver.mysql.database.azure.com', 'targetdb'. // keys from request's metadata. - commandSQLKey = "sql" + commandSQLKey = "sql" + commandParamsKey = "params" // keys from response's metadata. respOpKey = "operation" @@ -67,6 +69,7 @@ const ( type Mysql struct { db *sql.DB logger logger.Logger + closed atomic.Bool } type mysqlMetadata struct { @@ -87,21 +90,22 @@ type mysqlMetadata struct { // ConnMaxIdleTime is the maximum amount of time a connection may be idle. ConnMaxIdleTime time.Duration `mapstructure:"connMaxIdleTime"` - - // MaxRetries is the maximum number of retries for a query. - MaxRetries int `mapstructure:"maxRetries"` } // NewMysql returns a new MySQL output binding. func NewMysql(logger logger.Logger) bindings.OutputBinding { - return &Mysql{logger: logger} + return &Mysql{ + logger: logger, + } } // Init initializes the MySQL binding. func (m *Mysql) Init(ctx context.Context, md bindings.Metadata) error { - m.logger.Debug("Initializing MySql binding") + if m.closed.Load() { + return errors.New("cannot initialize a previously-closed component") + } - // parse metadata + // Parse metadata meta := mysqlMetadata{} err := metadata.DecodeMetadata(md.Properties, &meta) if err != nil { @@ -112,23 +116,29 @@ func (m *Mysql) Init(ctx context.Context, md bindings.Metadata) error { return fmt.Errorf("missing MySql connection string") } - db, err := initDB(meta.URL, meta.PemPath) + m.db, err = initDB(meta.URL, meta.PemPath) if err != nil { return err } - db.SetMaxIdleConns(meta.MaxIdleConns) - db.SetMaxOpenConns(meta.MaxOpenConns) - db.SetConnMaxIdleTime(meta.ConnMaxIdleTime) - db.SetConnMaxLifetime(meta.ConnMaxLifetime) + if meta.MaxIdleConns > 0 { + m.db.SetMaxIdleConns(meta.MaxIdleConns) + } + if meta.MaxOpenConns > 0 { + m.db.SetMaxOpenConns(meta.MaxOpenConns) + } + if meta.ConnMaxIdleTime > 0 { + m.db.SetConnMaxIdleTime(meta.ConnMaxIdleTime) + } + if meta.ConnMaxLifetime > 0 { + m.db.SetConnMaxLifetime(meta.ConnMaxLifetime) + } - err = db.PingContext(ctx) + err = m.db.PingContext(ctx) if err != nil { return fmt.Errorf("unable to ping the DB: %w", err) } - m.db = db - return nil } @@ -138,22 +148,38 @@ func (m *Mysql) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*bindi return nil, errors.New("invoke request required") } + // We let the "close" operation here succeed even if the component has been closed already if req.Operation == closeOperation { - return nil, m.db.Close() + return nil, m.Close() + } + + if m.closed.Load() { + return nil, errors.New("component is closed") } if req.Metadata == nil { return nil, errors.New("metadata required") } - m.logger.Debugf("operation: %v", req.Operation) - s, ok := req.Metadata[commandSQLKey] - if !ok || s == "" { + s := req.Metadata[commandSQLKey] + if s == "" { return nil, fmt.Errorf("required metadata not set: %s", commandSQLKey) } - startTime := time.Now() + // Metadata property "params" contains JSON-encoded parameters, and it's optional + // If present, it must be unserializable into a []any object + var ( + params []any + err error + ) + if paramsStr := req.Metadata[commandParamsKey]; paramsStr != "" { + err = json.Unmarshal([]byte(paramsStr), ¶ms) + if err != nil { + return nil, fmt.Errorf("invalid metadata property %s: failed to unserialize into an array: %w", commandParamsKey, err) + } + } + startTime := time.Now().UTC() resp := &bindings.InvokeResponse{ Metadata: map[string]string{ respOpKey: string(req.Operation), @@ -162,16 +188,16 @@ func (m *Mysql) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*bindi }, } - switch req.Operation { //nolint:exhaustive + switch req.Operation { case execOperation: - r, err := m.exec(ctx, s) + r, err := m.exec(ctx, s, params...) if err != nil { return nil, err } resp.Metadata[respRowsAffectedKey] = strconv.FormatInt(r, 10) case queryOperation: - d, err := m.query(ctx, s) + d, err := m.query(ctx, s, params...) if err != nil { return nil, err } @@ -182,7 +208,7 @@ func (m *Mysql) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*bindi req.Operation, execOperation, queryOperation, closeOperation) } - endTime := time.Now() + endTime := time.Now().UTC() resp.Metadata[respEndTimeKey] = endTime.Format(time.RFC3339Nano) resp.Metadata[respDurationKey] = endTime.Sub(startTime).String() @@ -200,23 +226,26 @@ func (m *Mysql) Operations() []bindings.OperationKind { // Close will close the DB. func (m *Mysql) Close() error { + if !m.closed.CompareAndSwap(false, true) { + // If this failed, the component has already been closed + // We allow multiple calls to close + return nil + } + if m.db != nil { - return m.db.Close() + m.db.Close() + m.db = nil } return nil } -func (m *Mysql) query(ctx context.Context, sql string) ([]byte, error) { - rows, err := m.db.QueryContext(ctx, sql) +func (m *Mysql) query(ctx context.Context, sql string, params ...any) ([]byte, error) { + rows, err := m.db.QueryContext(ctx, sql, params...) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } - - defer func() { - _ = rows.Close() - _ = rows.Err() - }() + defer rows.Close() result, err := m.jsonify(rows) if err != nil { @@ -226,10 +255,8 @@ func (m *Mysql) query(ctx context.Context, sql string) ([]byte, error) { return result, nil } -func (m *Mysql) exec(ctx context.Context, sql string) (int64, error) { - m.logger.Debugf("exec: %s", sql) - - res, err := m.db.ExecContext(ctx, sql) +func (m *Mysql) exec(ctx context.Context, sql string, params ...any) (int64, error) { + res, err := m.db.ExecContext(ctx, sql, params...) if err != nil { return 0, fmt.Errorf("error executing query: %w", err) } @@ -238,13 +265,15 @@ func (m *Mysql) exec(ctx context.Context, sql string) (int64, error) { } func initDB(url, pemPath string) (*sql.DB, error) { - if _, err := mysql.ParseDSN(url); err != nil { + conf, err := mysql.ParseDSN(url) + if err != nil { return nil, fmt.Errorf("illegal Data Source Name (DSN) specified by %s", connectionURLKey) } if pemPath != "" { + var pem []byte rootCertPool := x509.NewCertPool() - pem, err := os.ReadFile(pemPath) + pem, err = os.ReadFile(pemPath) if err != nil { return nil, fmt.Errorf("error reading PEM file from %s: %w", pemPath, err) } @@ -254,17 +283,25 @@ func initDB(url, pemPath string) (*sql.DB, error) { return nil, fmt.Errorf("failed to append PEM") } - err = mysql.RegisterTLSConfig("custom", &tls.Config{RootCAs: rootCertPool, MinVersion: tls.VersionTLS12}) + err = mysql.RegisterTLSConfig("custom", &tls.Config{ + RootCAs: rootCertPool, + MinVersion: tls.VersionTLS12, + }) if err != nil { return nil, fmt.Errorf("error register TLS config: %w", err) } } - db, err := sql.Open("mysql", url) + // Required to correctly parse time columns + // See: https://stackoverflow.com/a/45040724 + conf.ParseTime = true + + connector, err := mysql.NewConnector(conf) if err != nil { return nil, fmt.Errorf("error opening DB connection: %w", err) } + db := sql.OpenDB(connector) return db, nil } @@ -274,7 +311,7 @@ func (m *Mysql) jsonify(rows *sql.Rows) ([]byte, error) { return nil, err } - var ret []interface{} + var ret []any for rows.Next() { values := prepareValues(columnTypes) err := rows.Scan(values...) @@ -289,13 +326,13 @@ func (m *Mysql) jsonify(rows *sql.Rows) ([]byte, error) { return json.Marshal(ret) } -func prepareValues(columnTypes []*sql.ColumnType) []interface{} { +func prepareValues(columnTypes []*sql.ColumnType) []any { types := make([]reflect.Type, len(columnTypes)) for i, tp := range columnTypes { types[i] = tp.ScanType() } - values := make([]interface{}, len(columnTypes)) + values := make([]any, len(columnTypes)) for i := range values { values[i] = reflect.New(types[i]).Interface() } @@ -303,8 +340,8 @@ func prepareValues(columnTypes []*sql.ColumnType) []interface{} { return values } -func (m *Mysql) convert(columnTypes []*sql.ColumnType, values []interface{}) map[string]interface{} { - r := map[string]interface{}{} +func (m *Mysql) convert(columnTypes []*sql.ColumnType, values []any) map[string]any { + r := map[string]any{} for i, ct := range columnTypes { value := values[i] @@ -312,7 +349,7 @@ func (m *Mysql) convert(columnTypes []*sql.ColumnType, values []interface{}) map switch v := values[i].(type) { case driver.Valuer: if vv, err := v.Value(); err == nil { - value = interface{}(vv) + value = any(vv) } else { m.logger.Warnf("error to convert value: %v", err) } diff --git a/bindings/mysql/mysql_integration_test.go b/bindings/mysql/mysql_integration_test.go index fcc173526f..d5df16dda3 100644 --- a/bindings/mysql/mysql_integration_test.go +++ b/bindings/mysql/mysql_integration_test.go @@ -22,36 +22,20 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/dapr/components-contrib/bindings" "github.com/dapr/components-contrib/metadata" "github.com/dapr/kit/logger" ) -const ( - // MySQL doesn't accept RFC3339 formatted time, rejects trailing 'Z' for UTC indicator. - mySQLDateTimeFormat = "2006-01-02 15:04:05" - - testCreateTable = `CREATE TABLE IF NOT EXISTS foo ( - id bigint NOT NULL, - v1 character varying(50) NOT NULL, - b BOOLEAN, - ts TIMESTAMP, - data LONGTEXT)` - testDropTable = `DROP TABLE foo` - testInsert = "INSERT INTO foo (id, v1, b, ts, data) VALUES (%d, 'test-%d', %t, '%v', '%s')" - testDelete = "DELETE FROM foo" - testUpdate = "UPDATE foo SET ts = '%v' WHERE id = %d" - testSelect = "SELECT * FROM foo WHERE id < 3" - testSelectJSONExtract = "SELECT JSON_EXTRACT(data, '$.key') AS `key` FROM foo WHERE id < 3" -) +// MySQL doesn't accept RFC3339 formatted time, rejects trailing 'Z' for UTC indicator. +const mySQLDateTimeFormat = "2006-01-02 15:04:05" func TestOperations(t *testing.T) { - t.Parallel() t.Run("Get operation list", func(t *testing.T) { - t.Parallel() - b := NewMysql(nil) - assert.NotNil(t, b) + b := NewMysql(logger.NewLogger("test")) + require.NotNil(t, b) l := b.Operations() assert.Equal(t, 3, len(l)) assert.Contains(t, l, execOperation) @@ -70,123 +54,165 @@ func TestOperations(t *testing.T) { func TestMysqlIntegration(t *testing.T) { url := os.Getenv("MYSQL_TEST_CONN_URL") if url == "" { - t.SkipNow() + t.Skip("Skipping because env var MYSQL_TEST_CONN_URL is empty") } b := NewMysql(logger.NewLogger("test")).(*Mysql) m := bindings.Metadata{Base: metadata.Base{Properties: map[string]string{connectionURLKey: url}}} - if err := b.Init(context.Background(), m); err != nil { - t.Fatal(err) - } - defer b.Close() + err := b.Init(context.Background(), m) + require.NoError(t, err) - req := &bindings.InvokeRequest{Metadata: map[string]string{}} + defer b.Close() t.Run("Invoke create table", func(t *testing.T) { - req.Operation = execOperation - req.Metadata[commandSQLKey] = testCreateTable - res, err := b.Invoke(context.TODO(), req) + res, err := b.Invoke(context.Background(), &bindings.InvokeRequest{ + Operation: execOperation, + Metadata: map[string]string{ + commandSQLKey: `CREATE TABLE IF NOT EXISTS foo ( + id bigint NOT NULL, + v1 character varying(50) NOT NULL, + b BOOLEAN, + ts TIMESTAMP, + data LONGTEXT)`, + }, + }) assertResponse(t, res, err) }) t.Run("Invoke delete", func(t *testing.T) { - req.Operation = execOperation - req.Metadata[commandSQLKey] = testDelete - res, err := b.Invoke(context.TODO(), req) + res, err := b.Invoke(context.Background(), &bindings.InvokeRequest{ + Operation: execOperation, + Metadata: map[string]string{ + commandSQLKey: "DELETE FROM foo", + }, + }) assertResponse(t, res, err) }) t.Run("Invoke insert", func(t *testing.T) { - req.Operation = execOperation for i := 0; i < 10; i++ { - req.Metadata[commandSQLKey] = fmt.Sprintf(testInsert, i, i, true, time.Now().Format(mySQLDateTimeFormat), "{\"key\":\"val\"}") - res, err := b.Invoke(context.TODO(), req) + res, err := b.Invoke(context.Background(), &bindings.InvokeRequest{ + Operation: execOperation, + Metadata: map[string]string{ + commandSQLKey: fmt.Sprintf( + "INSERT INTO foo (id, v1, b, ts, data) VALUES (%d, 'test-%d', %t, '%v', '%s')", + i, i, true, time.Now().Format(mySQLDateTimeFormat), `{"key":"val"}`), + }, + }) assertResponse(t, res, err) } }) t.Run("Invoke update", func(t *testing.T) { - req.Operation = execOperation + date := time.Now().Add(time.Hour) + for i := 0; i < 10; i++ { + res, err := b.Invoke(context.Background(), &bindings.InvokeRequest{ + Operation: execOperation, + Metadata: map[string]string{ + commandSQLKey: fmt.Sprintf( + "UPDATE foo SET ts = '%v' WHERE id = %d", + date.Add(10*time.Duration(i)*time.Second).Format(mySQLDateTimeFormat), i), + }, + }) + assertResponse(t, res, err) + assert.Equal(t, "1", res.Metadata[respRowsAffectedKey]) + } + }) + + t.Run("Invoke update with parameters", func(t *testing.T) { + date := time.Now().Add(2 * time.Hour) for i := 0; i < 10; i++ { - req.Metadata[commandSQLKey] = fmt.Sprintf(testUpdate, time.Now().Format(mySQLDateTimeFormat), i) - res, err := b.Invoke(context.TODO(), req) + res, err := b.Invoke(context.Background(), &bindings.InvokeRequest{ + Operation: execOperation, + Metadata: map[string]string{ + commandSQLKey: "UPDATE foo SET ts = ? WHERE id = ?", + commandParamsKey: fmt.Sprintf(`[%q,%d]`, date.Add(10*time.Duration(i)*time.Second).Format(mySQLDateTimeFormat), i), + }, + }) assertResponse(t, res, err) + assert.Equal(t, "1", res.Metadata[respRowsAffectedKey]) } }) t.Run("Invoke select", func(t *testing.T) { - req.Operation = queryOperation - req.Metadata[commandSQLKey] = testSelect - res, err := b.Invoke(context.TODO(), req) + res, err := b.Invoke(context.Background(), &bindings.InvokeRequest{ + Operation: queryOperation, + Metadata: map[string]string{ + commandSQLKey: "SELECT * FROM foo WHERE id < 3", + }, + }) assertResponse(t, res, err) t.Logf("received result: %s", res.Data) // verify number, boolean and string - assert.Contains(t, string(res.Data), "\"id\":1") - assert.Contains(t, string(res.Data), "\"b\":1") - assert.Contains(t, string(res.Data), "\"v1\":\"test-1\"") - assert.Contains(t, string(res.Data), "\"data\":\"{\\\"key\\\":\\\"val\\\"}\"") + assert.Contains(t, string(res.Data), `"id":1`) + assert.Contains(t, string(res.Data), `"b":1`) + assert.Contains(t, string(res.Data), `"v1":"test-1"`) + assert.Contains(t, string(res.Data), `"data":"{\"key\":\"val\"}"`) - result := make([]interface{}, 0) + result := make([]any, 0) err = json.Unmarshal(res.Data, &result) - assert.Nil(t, err) + require.NoError(t, err) assert.Equal(t, 3, len(result)) // verify timestamp - ts, ok := result[0].(map[string]interface{})["ts"].(string) + ts, ok := result[0].(map[string]any)["ts"].(string) assert.True(t, ok) // have to use custom layout to parse timestamp, see this: https://github.com/dapr/components-contrib/pull/615 var tt time.Time tt, err = time.Parse("2006-01-02T15:04:05Z", ts) - assert.Nil(t, err) + require.NoError(t, err) t.Logf("time stamp is: %v", tt) }) - t.Run("Invoke select JSON_EXTRACT", func(t *testing.T) { - req.Operation = queryOperation - req.Metadata[commandSQLKey] = testSelectJSONExtract - res, err := b.Invoke(context.TODO(), req) + t.Run("Invoke select with parameters", func(t *testing.T) { + res, err := b.Invoke(context.Background(), &bindings.InvokeRequest{ + Operation: queryOperation, + Metadata: map[string]string{ + commandSQLKey: "SELECT * FROM foo WHERE id = ?", + commandParamsKey: `[1]`, + }, + }) assertResponse(t, res, err) t.Logf("received result: %s", res.Data) - // verify json extract number - assert.Contains(t, string(res.Data), "{\"key\":\"\\\"val\\\"\"}") + // verify number, boolean and string + assert.Contains(t, string(res.Data), `"id":1`) + assert.Contains(t, string(res.Data), `"b":1`) + assert.Contains(t, string(res.Data), `"v1":"test-1"`) + assert.Contains(t, string(res.Data), `"data":"{\"key\":\"val\"}"`) - result := make([]interface{}, 0) + result := make([]any, 0) err = json.Unmarshal(res.Data, &result) - assert.Nil(t, err) - assert.Equal(t, 3, len(result)) - }) - - t.Run("Invoke delete", func(t *testing.T) { - req.Operation = execOperation - req.Metadata[commandSQLKey] = testDelete - req.Data = nil - res, err := b.Invoke(context.TODO(), req) - assertResponse(t, res, err) + require.NoError(t, err) + assert.Equal(t, 1, len(result)) }) t.Run("Invoke drop", func(t *testing.T) { - req.Operation = execOperation - req.Metadata[commandSQLKey] = testDropTable - res, err := b.Invoke(context.TODO(), req) + res, err := b.Invoke(context.Background(), &bindings.InvokeRequest{ + Operation: execOperation, + Metadata: map[string]string{ + commandSQLKey: "DROP TABLE foo", + }, + }) assertResponse(t, res, err) }) t.Run("Invoke close", func(t *testing.T) { - req.Operation = closeOperation - req.Metadata = nil - req.Data = nil - _, err := b.Invoke(context.TODO(), req) + _, err := b.Invoke(context.Background(), &bindings.InvokeRequest{ + Operation: closeOperation, + }) assert.NoError(t, err) }) } func assertResponse(t *testing.T, res *bindings.InvokeResponse, err error) { + t.Helper() + assert.NoError(t, err) assert.NotNil(t, res) if res != nil { - assert.NotNil(t, res.Metadata) + assert.NotEmpty(t, res.Metadata) } } diff --git a/bindings/mysql/mysql_test.go b/bindings/mysql/mysql_test.go index a17c151b12..37c8f8ce5e 100644 --- a/bindings/mysql/mysql_test.go +++ b/bindings/mysql/mysql_test.go @@ -42,7 +42,7 @@ func TestQuery(t *testing.T) { assert.Nil(t, err) t.Logf("query result: %s", ret) assert.Contains(t, string(ret), "\"id\":1") - var result []interface{} + var result []any err = json.Unmarshal(ret, &result) assert.Nil(t, err) assert.Equal(t, 3, len(result)) @@ -65,13 +65,13 @@ func TestQuery(t *testing.T) { assert.Contains(t, string(ret), "\"id\":1") assert.Contains(t, string(ret), "\"value\":2.2") - var result []interface{} + var result []any err = json.Unmarshal(ret, &result) assert.Nil(t, err) assert.Equal(t, 3, len(result)) // verify timestamp - ts, ok := result[0].(map[string]interface{})["timestamp"].(string) + ts, ok := result[0].(map[string]any)["timestamp"].(string) assert.True(t, ok) var tt time.Time tt, err = time.Parse(time.RFC3339, ts) @@ -134,7 +134,7 @@ func TestInvoke(t *testing.T) { } resp, err := m.Invoke(context.Background(), req) assert.Nil(t, err) - var data []interface{} + var data []any err = json.Unmarshal(resp.Data, &data) assert.Nil(t, err) assert.Equal(t, 1, len(data)) From a4012953ea92db109c8721203c095856f86c98af Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Wed, 12 Jul 2023 15:03:18 -0700 Subject: [PATCH 4/8] Add Azure AD support to Postgres configuration store and bindings (#2971) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Signed-off-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> --- .../azure/setup-azure-conf-test.sh | 12 ++ .../docker-compose-postgresql.yml | 2 +- .github/scripts/test-info.mjs | 56 ++++++++- bindings/postgres/metadata.go | 52 ++++++++ bindings/postgres/metadata.yaml | 54 ++++++++- bindings/postgres/postgres.go | 69 ++++++----- bindings/postgres/postgres_test.go | 4 +- configuration/postgres/metadata.go | 51 +++++++- configuration/postgres/metadata.yaml | 59 +++++++-- configuration/postgres/postgres.go | 57 ++------- configuration/postgres/postgres_test.go | 18 +-- .../authentication/postgresql/metadata.go | 113 ++++++++++++++++++ internal/component/postgresql/metadata.go | 113 +++--------------- .../component/postgresql/metadata_test.go | 28 ++--- .../component/postgresql/postgresdbaccess.go | 23 ++-- state/postgresql/metadata.yaml | 2 +- .../components/standard/postgres.yaml | 6 +- .../bindings/postgres/docker-compose.yml | 4 +- .../bindings/postgres/postgres_test.go | 38 +++--- .../configuration/postgres/docker-compose.yml | 2 +- tests/certification/go.mod | 1 - tests/certification/go.sum | 2 - .../state/postgresql/docker-compose.yml | 2 +- tests/config/bindings/postgres/bindings.yml | 10 -- .../bindings/postgresql/azure/bindings.yml | 18 +++ .../bindings/postgresql/docker/bindings.yml | 11 ++ tests/config/bindings/tests.yml | 4 +- .../postgresql/azure/configstore.yml | 20 ++++ .../docker}/configstore.yml | 3 +- tests/config/configuration/tests.yml | 4 +- .../state/postgresql/azure/statestore.yml | 6 + tests/conformance/common.go | 11 +- .../configuration/configuration.go | 17 ++- .../utils/configupdater/postgres/postgres.go | 32 +++-- 34 files changed, 599 insertions(+), 305 deletions(-) create mode 100644 bindings/postgres/metadata.go create mode 100644 internal/authentication/postgresql/metadata.go delete mode 100644 tests/config/bindings/postgres/bindings.yml create mode 100644 tests/config/bindings/postgresql/azure/bindings.yml create mode 100644 tests/config/bindings/postgresql/docker/bindings.yml create mode 100644 tests/config/configuration/postgresql/azure/configstore.yml rename tests/config/configuration/{postgres => postgresql/docker}/configstore.yml (85%) diff --git a/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh b/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh index 8220f57fcd..ca553026bd 100755 --- a/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh +++ b/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh @@ -230,6 +230,9 @@ SQL_SERVER_DB_NAME_VAR_NAME="AzureSqlServerDbName" SQL_SERVER_CONNECTION_STRING_VAR_NAME="AzureSqlServerConnectionString" AZURE_DB_POSTGRES_CONNSTRING_VAR_NAME="AzureDBPostgresConnectionString" +AZURE_DB_POSTGRES_CLIENT_ID_VAR_NAME="AzureDBPostgresClientId" +AZURE_DB_POSTGRES_CLIENT_SECRET_VAR_NAME="AzureDBPostgresClientSecret" +AZURE_DB_POSTGRES_TENANT_ID_VAR_NAME="AzureDBPostgresTenantId" STORAGE_ACCESS_KEY_VAR_NAME="AzureBlobStorageAccessKey" STORAGE_ACCOUNT_VAR_NAME="AzureBlobStorageAccount" @@ -693,6 +696,15 @@ AZURE_DB_POSTGRES_CONNSTRING="host=${PREFIX}-conf-test-pg.postgres.database.azur echo export ${AZURE_DB_POSTGRES_CONNSTRING_VAR_NAME}=\"${AZURE_DB_POSTGRES_CONNSTRING}\" >> "${ENV_CONFIG_FILENAME}" az keyvault secret set --name "${AZURE_DB_POSTGRES_CONNSTRING_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${AZURE_DB_POSTGRES_CONNSTRING}" +echo export ${AZURE_DB_POSTGRES_CLIENT_ID_VAR_NAME}=\"${SDK_AUTH_SP_APPID}\" >> "${ENV_CONFIG_FILENAME}" +az keyvault secret set --name "${AZURE_DB_POSTGRES_CLIENT_ID_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${SDK_AUTH_SP_APPID}" + +echo export ${AZURE_DB_POSTGRES_CLIENT_SECRET_VAR_NAME}=\"${SDK_AUTH_SP_CLIENT_SECRET}\" >> "${ENV_CONFIG_FILENAME}" +az keyvault secret set --name "${AZURE_DB_POSTGRES_CLIENT_SECRET_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${SDK_AUTH_SP_CLIENT_SECRET}" + +echo export ${AZURE_DB_POSTGRES_TENANT_ID_VAR_NAME}=\"${TENANT_ID}\" >> "${ENV_CONFIG_FILENAME}" +az keyvault secret set --name "${AZURE_DB_POSTGRES_TENANT_ID_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${TENANT_ID}" + # ---------------------------------- # Populate Event Hubs test settings # ---------------------------------- diff --git a/.github/infrastructure/docker-compose-postgresql.yml b/.github/infrastructure/docker-compose-postgresql.yml index 6819464d45..d11b2d537f 100644 --- a/.github/infrastructure/docker-compose-postgresql.yml +++ b/.github/infrastructure/docker-compose-postgresql.yml @@ -1,7 +1,7 @@ version: '2' services: db: - image: postgres:15 + image: postgres:15-alpine restart: always ports: - "5432:5432" diff --git a/.github/scripts/test-info.mjs b/.github/scripts/test-info.mjs index 8bfeb0187f..1c4468af26 100644 --- a/.github/scripts/test-info.mjs +++ b/.github/scripts/test-info.mjs @@ -167,9 +167,28 @@ const components = { sourcePkg: ['bindings/mqtt3'], }, 'bindings.postgres': { - conformance: true, certification: true, + }, + 'bindings.postgresql.docker': { + conformance: true, conformanceSetup: 'docker-compose.sh postgresql', + sourcePkg: [ + 'bindings/postgresql', + 'internal/authentication/postgresql', + ], + }, + 'bindings.postgresql.azure': { + conformance: true, + requiredSecrets: [ + 'AzureDBPostgresConnectionString', + 'AzureDBPostgresClientId', + 'AzureDBPostgresClientSecret', + 'AzureDBPostgresTenantId', + ], + sourcePkg: [ + 'bindings/postgresql', + 'internal/authentication/postgresql', + ], }, 'bindings.rabbitmq': { conformance: true, @@ -191,9 +210,32 @@ const components = { sourcePkg: ['bindings/redis', 'internal/component/redis'], }, 'configuration.postgres': { - conformance: true, certification: true, + sourcePkg: [ + 'configuration/postgresql', + 'internal/authentication/postgresql', + ], + }, + 'configuration.postgresql.docker': { + conformance: true, conformanceSetup: 'docker-compose.sh postgresql', + sourcePkg: [ + 'configuration/postgresql', + 'internal/authentication/postgresql', + ], + }, + 'configuration.postgresql.azure': { + conformance: true, + requiredSecrets: [ + 'AzureDBPostgresConnectionString', + 'AzureDBPostgresClientId', + 'AzureDBPostgresClientSecret', + 'AzureDBPostgresTenantId', + ], + sourcePkg: [ + 'configuration/postgresql', + 'internal/authentication/postgresql', + ], }, 'configuration.redis.v6': { conformance: true, @@ -585,6 +627,7 @@ const components = { certification: true, sourcePkg: [ 'state/postgresql', + 'internal/authentication/postgresql', 'internal/component/postgresql', 'internal/component/sql', ], @@ -594,15 +637,22 @@ const components = { conformanceSetup: 'docker-compose.sh postgresql', sourcePkg: [ 'state/postgresql', + 'internal/authentication/postgresql', 'internal/component/postgresql', 'internal/component/sql', ], }, 'state.postgresql.azure': { conformance: true, - requiredSecrets: ['AzureDBPostgresConnectionString'], + requiredSecrets: [ + 'AzureDBPostgresConnectionString', + 'AzureDBPostgresClientId', + 'AzureDBPostgresClientSecret', + 'AzureDBPostgresTenantId', + ], sourcePkg: [ 'state/postgresql', + 'internal/authentication/postgresql', 'internal/component/postgresql', 'internal/component/sql', ], diff --git a/bindings/postgres/metadata.go b/bindings/postgres/metadata.go new file mode 100644 index 0000000000..281b2c593d --- /dev/null +++ b/bindings/postgres/metadata.go @@ -0,0 +1,52 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package postgres + +import ( + pgauth "github.com/dapr/components-contrib/internal/authentication/postgresql" + contribMetadata "github.com/dapr/components-contrib/metadata" +) + +type psqlMetadata struct { + pgauth.PostgresAuthMetadata `mapstructure:",squash"` + + // URL is the connection string to connect to the database. + // Deprecated alias: use connectionString instead. + URL string `mapstructure:"url"` +} + +func (m *psqlMetadata) InitWithMetadata(meta map[string]string) error { + // Reset the object + m.PostgresAuthMetadata.Reset() + m.URL = "" + + err := contribMetadata.DecodeMetadata(meta, &m) + if err != nil { + return err + } + + // Legacy options + if m.ConnectionString == "" && m.URL != "" { + m.ConnectionString = m.URL + } + + // Validate and sanitize input + // Azure AD auth is supported for this component + err = m.PostgresAuthMetadata.InitWithMetadata(meta, true) + if err != nil { + return err + } + + return nil +} diff --git a/bindings/postgres/metadata.yaml b/bindings/postgres/metadata.yaml index 1d771523a3..01341e690b 100644 --- a/bindings/postgres/metadata.yaml +++ b/bindings/postgres/metadata.yaml @@ -19,17 +19,33 @@ binding: description: "The query operation is used for SELECT statements, which return both the metadata and the retrieved data in a form of an array of row values." - name: close description: "The close operation can be used to explicitly close the DB connection and return it to the pool. This operation doesn't have any response." +builtinAuthenticationProfiles: + - name: "azuread" + metadata: + - name: useAzureAD + required: true + type: bool + example: '"true"' + description: | + Must be set to `true` to enable the component to retrieve access tokens from Azure AD. + This authentication method only works with Azure Database for PostgreSQL databases. + - name: connectionString + required: true + sensitive: true + description: | + The connection string for the PostgreSQL database + This must contain the user, which corresponds to the name of the user created inside PostgreSQL that maps to the Azure AD identity; this is often the name of the corresponding principal (e.g. the name of the Azure AD application). This connection string should not contain any password. + example: | + "host=mydb.postgres.database.azure.com user=myapplication port=5432 database=dapr_test sslmode=require" + type: string authenticationProfiles: - title: "Connection string" - description: "Authenticate using a Connection String." + description: "Authenticate using a Connection String" metadata: - - name: url + - name: connectionString required: true sensitive: true - binding: - input: false - output: true - description: "Connection string for PostgreSQL." + description: "The connection string for the PostgreSQL database" url: title: More details url: https://docs.dapr.io/reference/components-reference/supported-bindings/postgres/#url-format @@ -37,3 +53,29 @@ authenticationProfiles: "user=dapr password=secret host=dapr.example.com port=5432 dbname=dapr sslmode=verify-ca" or "postgres://dapr:secret@dapr.example.com:5432/dapr?sslmode=verify-ca" type: string +metadata: + - name: maxConns + required: false + description: | + Maximum number of connections pooled by this component. + Set to 0 or lower to use the default value, which is the greater of 4 or the number of CPUs. + example: "4" + default: "0" + type: number + - name: connectionMaxIdleTime + required: false + description: | + Max idle time before unused connections are automatically closed in the + connection pool. By default, there's no value and this is left to the + database driver to choose. + example: "5m" + type: duration + - name: url + deprecated: true + required: false + description: | + Deprecated alias for "connectionString" + type: string + sensitive: true + example: | + "user=dapr password=secret host=dapr.example.com port=5432 dbname=dapr sslmode=verify-ca" diff --git a/bindings/postgres/postgres.go b/bindings/postgres/postgres.go index 7c03e3bb7a..9919f9b1d0 100644 --- a/bindings/postgres/postgres.go +++ b/bindings/postgres/postgres.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Dapr Authors +Copyright 2023 The Dapr Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -20,6 +20,7 @@ import ( "fmt" "reflect" "strconv" + "sync/atomic" "time" "github.com/jackc/pgx/v5/pgxpool" @@ -35,38 +36,36 @@ const ( queryOperation bindings.OperationKind = "query" closeOperation bindings.OperationKind = "close" - connectionURLKey = "url" - commandSQLKey = "sql" + commandSQLKey = "sql" ) // Postgres represents PostgreSQL output binding. type Postgres struct { logger logger.Logger db *pgxpool.Pool -} - -type psqlMetadata struct { - // ConnectionURL is the connection string to connect to the database. - ConnectionURL string `mapstructure:"url"` + closed atomic.Bool } // NewPostgres returns a new PostgreSQL output binding. func NewPostgres(logger logger.Logger) bindings.OutputBinding { - return &Postgres{logger: logger} + return &Postgres{ + logger: logger, + } } // Init initializes the PostgreSql binding. func (p *Postgres) Init(ctx context.Context, meta bindings.Metadata) error { + if p.closed.Load() { + return errors.New("cannot initialize a previously-closed component") + } + m := psqlMetadata{} - err := metadata.DecodeMetadata(meta.Properties, &m) + err := m.InitWithMetadata(meta.Properties) if err != nil { return err } - if m.ConnectionURL == "" { - return fmt.Errorf("required metadata not set: %s", connectionURLKey) - } - poolConfig, err := pgxpool.ParseConfig(m.ConnectionURL) + poolConfig, err := m.GetPgxPoolConfig() if err != nil { return fmt.Errorf("error opening DB connection: %w", err) } @@ -75,7 +74,7 @@ func (p *Postgres) Init(ctx context.Context, meta bindings.Metadata) error { // only scoped to postgres creating resources at init. p.db, err = pgxpool.NewWithConfig(ctx, poolConfig) if err != nil { - return fmt.Errorf("unable to ping the DB: %w", err) + return fmt.Errorf("unable to connect to the DB: %w", err) } return nil @@ -96,16 +95,19 @@ func (p *Postgres) Invoke(ctx context.Context, req *bindings.InvokeRequest) (res return nil, errors.New("invoke request required") } + // We let the "close" operation here succeed even if the component has been closed already if req.Operation == closeOperation { - p.db.Close() + err = p.Close() + return nil, err + } - return nil, nil + if p.closed.Load() { + return nil, errors.New("component is closed") } if req.Metadata == nil { return nil, errors.New("metadata required") } - p.logger.Debugf("operation: %v", req.Operation) sql, ok := req.Metadata[commandSQLKey] if !ok || sql == "" { @@ -125,14 +127,14 @@ func (p *Postgres) Invoke(ctx context.Context, req *bindings.InvokeRequest) (res case execOperation: r, err := p.exec(ctx, sql) if err != nil { - return nil, fmt.Errorf("error executing %s: %w", sql, err) + return nil, err } resp.Metadata["rows-affected"] = strconv.FormatInt(r, 10) // 0 if error case queryOperation: d, err := p.query(ctx, sql) if err != nil { - return nil, fmt.Errorf("error executing %s: %w", sql, err) + return nil, err } resp.Data = d @@ -152,17 +154,21 @@ func (p *Postgres) Invoke(ctx context.Context, req *bindings.InvokeRequest) (res // Close close PostgreSql instance. func (p *Postgres) Close() error { - if p.db == nil { + if !p.closed.CompareAndSwap(false, true) { + // If this failed, the component has already been closed + // We allow multiple calls to close return nil } - p.db.Close() + + if p.db != nil { + p.db.Close() + } + p.db = nil return nil } func (p *Postgres) query(ctx context.Context, sql string) (result []byte, err error) { - p.logger.Debugf("query: %s", sql) - rows, err := p.db.Query(ctx, sql) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) @@ -172,29 +178,26 @@ func (p *Postgres) query(ctx context.Context, sql string) (result []byte, err er for rows.Next() { val, rowErr := rows.Values() if rowErr != nil { - return nil, fmt.Errorf("error parsing result '%v': %w", rows.Err(), rowErr) + return nil, fmt.Errorf("error reading result '%v': %w", rows.Err(), rowErr) } rs = append(rs, val) //nolint:asasalint } - if result, err = json.Marshal(rs); err != nil { - err = fmt.Errorf("error serializing results: %w", err) + result, err = json.Marshal(rs) + if err != nil { + return nil, fmt.Errorf("error serializing results: %w", err) } - return + return result, nil } func (p *Postgres) exec(ctx context.Context, sql string) (result int64, err error) { - p.logger.Debugf("exec: %s", sql) - res, err := p.db.Exec(ctx, sql) if err != nil { return 0, fmt.Errorf("error executing query: %w", err) } - result = res.RowsAffected() - - return + return res.RowsAffected(), nil } // GetComponentMetadata returns the metadata of the component. diff --git a/bindings/postgres/postgres_test.go b/bindings/postgres/postgres_test.go index 78adb43ab0..392fe2a6ad 100644 --- a/bindings/postgres/postgres_test.go +++ b/bindings/postgres/postgres_test.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Dapr Authors +Copyright 2023 The Dapr Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -63,7 +63,7 @@ func TestPostgresIntegration(t *testing.T) { // live DB test b := NewPostgres(logger.NewLogger("test")).(*Postgres) - m := bindings.Metadata{Base: metadata.Base{Properties: map[string]string{connectionURLKey: url}}} + m := bindings.Metadata{Base: metadata.Base{Properties: map[string]string{"connectionString": url}}} if err := b.Init(context.Background(), m); err != nil { t.Fatal(err) } diff --git a/configuration/postgres/metadata.go b/configuration/postgres/metadata.go index 0bf10cb742..b53a5b2d3c 100644 --- a/configuration/postgres/metadata.go +++ b/configuration/postgres/metadata.go @@ -13,10 +13,53 @@ limitations under the License. package postgres -import "time" +import ( + "fmt" + "time" + + pgauth "github.com/dapr/components-contrib/internal/authentication/postgresql" + contribMetadata "github.com/dapr/components-contrib/metadata" +) type metadata struct { - MaxIdleTimeout time.Duration `mapstructure:"connMaxIdleTime"` - ConnectionString string `mapstructure:"connectionString"` - ConfigTable string `mapstructure:"table"` + pgauth.PostgresAuthMetadata `mapstructure:",squash"` + + ConfigTable string `mapstructure:"table"` + MaxIdleTimeoutOld time.Duration `mapstructure:"connMaxIdleTime"` // Deprecated alias for "connectionMaxIdleTime" +} + +func (m *metadata) InitWithMetadata(meta map[string]string) error { + // Reset the object + m.PostgresAuthMetadata.Reset() + m.ConfigTable = "" + m.MaxIdleTimeoutOld = 0 + + err := contribMetadata.DecodeMetadata(meta, &m) + if err != nil { + return err + } + + // Legacy options + if m.ConnectionMaxIdleTime == 0 && m.MaxIdleTimeoutOld > 0 { + m.ConnectionMaxIdleTime = m.MaxIdleTimeoutOld + } + + // Validate and sanitize input + if m.ConfigTable == "" { + return fmt.Errorf("missing postgreSQL configuration table name") + } + if len(m.ConfigTable) > maxIdentifierLength { + return fmt.Errorf("table name is too long - tableName : '%s'. max allowed field length is %d", m.ConfigTable, maxIdentifierLength) + } + if !allowedTableNameChars.MatchString(m.ConfigTable) { + return fmt.Errorf("invalid table name '%s'. non-alphanumerics or upper cased table names are not supported", m.ConfigTable) + } + + // Azure AD auth is supported for this component + err = m.PostgresAuthMetadata.InitWithMetadata(meta, true) + if err != nil { + return err + } + + return nil } diff --git a/configuration/postgres/metadata.yaml b/configuration/postgres/metadata.yaml index 16e411c8af..47fba4a0f4 100644 --- a/configuration/postgres/metadata.yaml +++ b/configuration/postgres/metadata.yaml @@ -1,14 +1,33 @@ # yaml-language-server: $schema=../../component-metadata-schema.json schemaVersion: v1 type: configuration -name: postgres +name: postgresql version: v1 -status: alpha -title: "Postgres" +status: stable +title: "PostgreSQL" urls: - title: Reference - url: https://docs.dapr.io/reference/components-reference/supported-configuration-stores/postgres-configuration-store/ + url: https://docs.dapr.io/reference/components-reference/supported-configuration-stores/postgresql-configuration-store/ capabilities: [] +builtinAuthenticationProfiles: + - name: "azuread" + metadata: + - name: useAzureAD + required: true + type: bool + example: '"true"' + description: | + Must be set to `true` to enable the component to retrieve access tokens from Azure AD. + This authentication method only works with Azure Database for PostgreSQL databases. + - name: connectionString + required: true + sensitive: true + description: | + The connection string for the PostgreSQL database + This must contain the user, which corresponds to the name of the user created inside PostgreSQL that maps to the Azure AD identity; this is often the name of the corresponding principal (e.g. the name of the Azure AD application). This connection string should not contain any password. + example: | + "host=mydb.postgres.database.azure.com user=myapplication port=5432 database=dapr_test sslmode=require" + type: string authenticationProfiles: - title: "Connection string" description: "Authenticate using a Connection String." @@ -16,10 +35,9 @@ authenticationProfiles: - name: connectionString required: true sensitive: true - description: | - The connection string for PostgreSQL, as a URL or DSN. - Note: the default value for `pool_max_conns` is 5. - example: "host=localhost user=postgres password=example port=5432 connect_timeout=10 database=dapr_test pool_max_conns=10" + description: The connection string for the PostgreSQL database + example: | + "host=localhost user=postgres password=example port=5432 connect_timeout=10 database=dapr_test" type: string metadata: - name: table @@ -27,9 +45,26 @@ metadata: description: The table name for configuration information. example: "configTable" type: string - - name: connMaxIdleTime + - name: connectionMaxIdleTime required: false - description: The maximum amount of time a connection may be idle. - example: "15s" - default: "30s" + description: | + Max idle time before unused connections are automatically closed in the + connection pool. By default, there's no value and this is left to the + database driver to choose. + example: "5m" type: duration + - name: maxConns + required: false + description: | + Maximum number of connections pooled by this component. + Set to 0 or lower to use the default value, which is the greater of 4 or the number of CPUs. + example: "4" + default: "0" + type: number + - name: connMaxIdleTime + deprecated: true + required: false + description: | + Deprecated alias for 'connectionMaxIdleTime'. + example: "5m" + type: duration \ No newline at end of file diff --git a/configuration/postgres/postgres.go b/configuration/postgres/postgres.go index f948ac0ed7..3a072da851 100644 --- a/configuration/postgres/postgres.go +++ b/configuration/postgres/postgres.go @@ -23,13 +23,12 @@ import ( "strconv" "strings" "sync" - "time" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" - "k8s.io/utils/strings/slices" + "golang.org/x/exp/slices" "github.com/dapr/components-contrib/configuration" contribMetadata "github.com/dapr/components-contrib/metadata" @@ -62,34 +61,29 @@ const ( ) var ( - allowedChars = regexp.MustCompile(`^[a-zA-Z0-9./_]*$`) - allowedTableNameChars = regexp.MustCompile(`^[a-z0-9./_]*$`) - defaultMaxConnIdleTime = time.Second * 30 + allowedChars = regexp.MustCompile(`^[a-zA-Z0-9./_]*$`) + allowedTableNameChars = regexp.MustCompile(`^[a-z0-9./_]*$`) ) func NewPostgresConfigurationStore(logger logger.Logger) configuration.Store { - logger.Debug("Instantiating PostgreSQL configuration store") return &ConfigurationStore{ logger: logger, subscribeStopChanMap: make(map[string]chan struct{}), } } -func (p *ConfigurationStore) Init(parentCtx context.Context, metadata configuration.Metadata) error { - if m, err := parseMetadata(metadata); err != nil { +func (p *ConfigurationStore) Init(ctx context.Context, metadata configuration.Metadata) error { + err := p.metadata.InitWithMetadata(metadata.Properties) + if err != nil { p.logger.Error(err) return err - } else { - p.metadata = m } + p.ActiveSubscriptions = make(map[string]*subscription) - ctx, cancel := context.WithTimeout(parentCtx, p.metadata.MaxIdleTimeout) - defer cancel() - client, err := Connect(ctx, p.metadata.ConnectionString, p.metadata.MaxIdleTimeout) + p.client, err = p.connectDB(ctx) if err != nil { return fmt.Errorf("error connecting to configuration store: '%w'", err) } - p.client = client err = p.client.Ping(ctx) if err != nil { return fmt.Errorf("unable to connect to configuration store: '%w'", err) @@ -180,7 +174,7 @@ func (p *ConfigurationStore) Subscribe(ctx context.Context, req *configuration.S } } if pgNotifyChannel == "" { - return "", fmt.Errorf("unable to subscribe to '%s'.pgNotifyChannel attribute cannot be empty", p.metadata.ConfigTable) + return "", fmt.Errorf("unable to subscribe to '%s'. pgNotifyChannel attribute cannot be empty", p.metadata.ConfigTable) } return p.subscribeToChannel(ctx, pgNotifyChannel, req, handler) } @@ -290,37 +284,8 @@ func (p *ConfigurationStore) handleSubscribedChange(ctx context.Context, handler } } -func parseMetadata(cmetadata configuration.Metadata) (metadata, error) { - m := metadata{ - MaxIdleTimeout: defaultMaxConnIdleTime, - } - decodeErr := contribMetadata.DecodeMetadata(cmetadata.Properties, &m) - if decodeErr != nil { - return m, decodeErr - } - - if m.ConnectionString == "" { - return m, fmt.Errorf("missing postgreSQL connection string") - } - - if m.ConfigTable != "" { - if !allowedTableNameChars.MatchString(m.ConfigTable) { - return m, fmt.Errorf("invalid table name '%s'. non-alphanumerics or upper cased table names are not supported", m.ConfigTable) - } - if len(m.ConfigTable) > maxIdentifierLength { - return m, fmt.Errorf("table name is too long - tableName : '%s'. max allowed field length is %d", m.ConfigTable, maxIdentifierLength) - } - } else { - return m, fmt.Errorf("missing postgreSQL configuration table name") - } - if m.MaxIdleTimeout <= 0 { - m.MaxIdleTimeout = defaultMaxConnIdleTime - } - return m, nil -} - -func Connect(ctx context.Context, conn string, maxTimeout time.Duration) (*pgxpool.Pool, error) { - config, err := pgxpool.ParseConfig(conn) +func (p *ConfigurationStore) connectDB(ctx context.Context) (*pgxpool.Pool, error) { + config, err := p.metadata.GetPgxPoolConfig() if err != nil { return nil, fmt.Errorf("postgres configuration store connection error : %w", err) } diff --git a/configuration/postgres/postgres_test.go b/configuration/postgres/postgres_test.go index 4a9666582e..74fb36a787 100644 --- a/configuration/postgres/postgres_test.go +++ b/configuration/postgres/postgres_test.go @@ -20,8 +20,10 @@ import ( "github.com/pashagolub/pgxmock/v2" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/dapr/components-contrib/configuration" + pgauth "github.com/dapr/components-contrib/internal/authentication/postgresql" ) func TestSelectAllQuery(t *testing.T) { @@ -43,7 +45,7 @@ func TestSelectAllQuery(t *testing.T) { if err != nil { t.Errorf("Error building query: %v ", err) } - assert.Nil(t, err, "Error building query: %v ", err) + assert.NoError(t, err, "Error building query: %v ", err) assert.Equal(t, expected, query, "did not get expected result. Got: '%v' , Expected: '%v'", query, expected) } @@ -57,7 +59,7 @@ func TestPostgresbuildQuery(t *testing.T) { query, params, err := buildQuery(g, "cfgtbl") _ = params - assert.Nil(t, err, "Error building query: %v ", err) + assert.NoError(t, err, "Error building query: %v ", err) expected := "SELECT * FROM cfgtbl WHERE KEY IN ($1) AND $2 = $3" assert.Equal(t, expected, query, "did not get expected result. Got: '%v' , Expected: '%v'", query, expected) i := 0 @@ -80,12 +82,14 @@ func TestPostgresbuildQuery(t *testing.T) { func TestConnectAndQuery(t *testing.T) { m := metadata{ - ConnectionString: "mockConnectionString", - ConfigTable: "mockConfigTable", + PostgresAuthMetadata: pgauth.PostgresAuthMetadata{ + ConnectionString: "mockConnectionString", + }, + ConfigTable: "mockConfigTable", } mock, err := pgxmock.NewPool() - assert.Nil(t, err) + require.NoError(t, err) defer mock.Close() query := "SELECT EXISTS (SELECT FROM pg_tables where tablename = '" + m.ConfigTable + "'" @@ -97,9 +101,9 @@ func TestConnectAndQuery(t *testing.T) { rows := mock.QueryRow(context.Background(), query) var id string err = rows.Scan(&id) - assert.Nil(t, err, "error in scan") + assert.NoError(t, err, "error in scan") err = mock.ExpectationsWereMet() - assert.Nil(t, err, "pgxmock error in expectations were met") + assert.NoError(t, err, "pgxmock error in expectations were met") } func TestValidateInput(t *testing.T) { diff --git a/internal/authentication/postgresql/metadata.go b/internal/authentication/postgresql/metadata.go new file mode 100644 index 0000000000..66f67fef1f --- /dev/null +++ b/internal/authentication/postgresql/metadata.go @@ -0,0 +1,113 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package postgresql + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/dapr/components-contrib/internal/authentication/azure" +) + +// PostgresAuthMetadata contains authentication metadata for PostgreSQL components. +type PostgresAuthMetadata struct { + ConnectionString string `mapstructure:"connectionString"` + ConnectionMaxIdleTime time.Duration `mapstructure:"connectionMaxIdleTime"` + MaxConns int `mapstructure:"maxConns"` + UseAzureAD bool `mapstructure:"useAzureAD"` + + azureEnv azure.EnvironmentSettings +} + +// Reset the object. +func (m *PostgresAuthMetadata) Reset() { + m.ConnectionString = "" + m.ConnectionMaxIdleTime = 0 + m.MaxConns = 0 + m.UseAzureAD = false +} + +// InitWithMetadata inits the object with metadata from the user. +// Set azureADEnabled to true if the component can support authentication with Azure AD. +// This is different from the "useAzureAD" property from the user, which is provided by the user and instructs the component to authenticate using Azure AD. +func (m *PostgresAuthMetadata) InitWithMetadata(meta map[string]string, azureADEnabled bool) (err error) { + // Validate input + if m.ConnectionString == "" { + return errors.New("missing connection string") + } + + // Populate the Azure environment if using Azure AD + if azureADEnabled && m.UseAzureAD { + m.azureEnv, err = azure.NewEnvironmentSettings(meta) + if err != nil { + return err + } + } else { + // Make sure this is false + m.UseAzureAD = false + } + + return nil +} + +// GetPgxPoolConfig returns the pgxpool.Config object that contains the credentials for connecting to PostgreSQL. +func (m *PostgresAuthMetadata) GetPgxPoolConfig() (*pgxpool.Config, error) { + // Get the config from the connection string + config, err := pgxpool.ParseConfig(m.ConnectionString) + if err != nil { + return nil, fmt.Errorf("failed to parse connection string: %w", err) + } + if m.ConnectionMaxIdleTime > 0 { + config.MaxConnIdleTime = m.ConnectionMaxIdleTime + } + if m.MaxConns > 1 { + config.MaxConns = int32(m.MaxConns) + } + + // Check if we should use Azure AD + if m.UseAzureAD { + tokenCred, errToken := m.azureEnv.GetTokenCredential() + if errToken != nil { + return nil, errToken + } + + // Reset the password + config.ConnConfig.Password = "" + + // We need to retrieve the token every time we attempt a new connection + // This is because tokens expire, and connections can drop and need to be re-established at any time + // Fortunately, we can do this with the "BeforeConnect" hook + config.BeforeConnect = func(ctx context.Context, cc *pgx.ConnConfig) error { + at, err := tokenCred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{ + m.azureEnv.Cloud.Services[azure.ServiceOSSRDBMS].Audience + "/.default", + }, + }) + if err != nil { + return err + } + + cc.Password = at.Token + return nil + } + } + + return config, nil +} diff --git a/internal/component/postgresql/metadata.go b/internal/component/postgresql/metadata.go index fb10bed6a0..a4dd06f076 100644 --- a/internal/component/postgresql/metadata.go +++ b/internal/component/postgresql/metadata.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Dapr Authors +Copyright 2023 The Dapr Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -14,15 +14,10 @@ limitations under the License. package postgresql import ( - "context" "fmt" "time" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgxpool" - - "github.com/dapr/components-contrib/internal/authentication/azure" + pgauth "github.com/dapr/components-contrib/internal/authentication/postgresql" "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/state" "github.com/dapr/kit/ptr" @@ -39,24 +34,17 @@ const ( ) type postgresMetadataStruct struct { - ConnectionString string `mapstructure:"connectionString"` - ConnectionMaxIdleTime time.Duration `mapstructure:"connectionMaxIdleTime"` - TableName string `mapstructure:"tableName"` // Could be in the format "schema.table" or just "table" - MetadataTableName string `mapstructure:"metadataTableName"` // Could be in the format "schema.table" or just "table" - Timeout time.Duration `mapstructure:"timeoutInSeconds"` - CleanupInterval *time.Duration `mapstructure:"cleanupIntervalInSeconds"` - MaxConns int `mapstructure:"maxConns"` - UseAzureAD bool `mapstructure:"useAzureAD"` + pgauth.PostgresAuthMetadata `mapstructure:",squash"` - // Set to true if the component can support authentication with Azure AD. - // This is different from the "useAzureAD" property above, which is provided by the user and instructs the component to authenticate using Azure AD. - azureADEnabled bool - azureEnv azure.EnvironmentSettings + TableName string `mapstructure:"tableName"` // Could be in the format "schema.table" or just "table" + MetadataTableName string `mapstructure:"metadataTableName"` // Could be in the format "schema.table" or just "table" + Timeout time.Duration `mapstructure:"timeoutInSeconds"` + CleanupInterval *time.Duration `mapstructure:"cleanupIntervalInSeconds"` } -func (m *postgresMetadataStruct) InitWithMetadata(meta state.Metadata) error { +func (m *postgresMetadataStruct) InitWithMetadata(meta state.Metadata, azureADEnabled bool) error { // Reset the object - m.ConnectionString = "" + m.PostgresAuthMetadata.Reset() m.TableName = defaultTableName m.MetadataTableName = defaultMetadataTableName m.CleanupInterval = ptr.Of(defaultCleanupInternal * time.Second) @@ -69,8 +57,9 @@ func (m *postgresMetadataStruct) InitWithMetadata(meta state.Metadata) error { } // Validate and sanitize input - if m.ConnectionString == "" { - return errMissingConnectionString + err = m.PostgresAuthMetadata.InitWithMetadata(meta.Properties, azureADEnabled) + if err != nil { + return err } // Timeout @@ -79,79 +68,15 @@ func (m *postgresMetadataStruct) InitWithMetadata(meta state.Metadata) error { } // Cleanup interval - if m.CleanupInterval != nil { - // Non-positive value from meta means disable auto cleanup. - if *m.CleanupInterval <= 0 { - if meta.Properties[cleanupIntervalKey] == "" { - // unfortunately the mapstructure decoder decodes an empty string to 0, a missing key would be nil however - m.CleanupInterval = ptr.Of(defaultCleanupInternal * time.Second) - } else { - m.CleanupInterval = nil - } - } - } - - // Populate the Azure environment if using Azure AD - if m.azureADEnabled && m.UseAzureAD { - m.azureEnv, err = azure.NewEnvironmentSettings(meta.Properties) - if err != nil { - return err + // Non-positive value from meta means disable auto cleanup. + if m.CleanupInterval != nil && *m.CleanupInterval <= 0 { + if meta.Properties[cleanupIntervalKey] == "" { + // Unfortunately the mapstructure decoder decodes an empty string to 0, a missing key would be nil however + m.CleanupInterval = ptr.Of(defaultCleanupInternal * time.Second) + } else { + m.CleanupInterval = nil } } return nil } - -// GetPgxPoolConfig returns the pgxpool.Config object that contains the credentials for connecting to Postgres. -func (m *postgresMetadataStruct) GetPgxPoolConfig() (*pgxpool.Config, error) { - // Get the config from the connection string - config, err := pgxpool.ParseConfig(m.ConnectionString) - if err != nil { - return nil, fmt.Errorf("failed to parse connection string: %w", err) - } - if m.ConnectionMaxIdleTime > 0 { - config.MaxConnIdleTime = m.ConnectionMaxIdleTime - } - if m.MaxConns > 1 { - config.MaxConns = int32(m.MaxConns) - } - - // Check if we should use Azure AD - if m.azureADEnabled && m.UseAzureAD { - tokenCred, errToken := m.azureEnv.GetTokenCredential() - if errToken != nil { - return nil, errToken - } - - // Reset the password - config.ConnConfig.Password = "" - - /*// For Azure AD, using SSL is required - // If not already enabled, configure TLS without certificate validation - if config.ConnConfig.TLSConfig == nil { - config.ConnConfig.TLSConfig = &tls.Config{ - //nolint:gosec - InsecureSkipVerify: true, - } - }*/ - - // We need to retrieve the token every time we attempt a new connection - // This is because tokens expire, and connections can drop and need to be re-established at any time - // Fortunately, we can do this with the "BeforeConnect" hook - config.BeforeConnect = func(ctx context.Context, cc *pgx.ConnConfig) error { - at, err := tokenCred.GetToken(ctx, policy.TokenRequestOptions{ - Scopes: []string{ - m.azureEnv.Cloud.Services[azure.ServiceOSSRDBMS].Audience + "/.default", - }, - }) - if err != nil { - return err - } - - cc.Password = at.Token - return nil - } - } - - return config, nil -} diff --git a/internal/component/postgresql/metadata_test.go b/internal/component/postgresql/metadata_test.go index 9d56f0ab34..bb4b612f18 100644 --- a/internal/component/postgresql/metadata_test.go +++ b/internal/component/postgresql/metadata_test.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Dapr Authors +Copyright 2023 The Dapr Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -28,9 +28,9 @@ func TestMetadata(t *testing.T) { m := postgresMetadataStruct{} props := map[string]string{} - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.Error(t, err) - assert.ErrorIs(t, err, errMissingConnectionString) + assert.ErrorContains(t, err, "connection string") }) t.Run("has connection string", func(t *testing.T) { @@ -39,7 +39,7 @@ func TestMetadata(t *testing.T) { "connectionString": "foo", } - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.NoError(t, err) }) @@ -49,7 +49,7 @@ func TestMetadata(t *testing.T) { "connectionString": "foo", } - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.NoError(t, err) assert.Equal(t, m.TableName, defaultTableName) }) @@ -61,7 +61,7 @@ func TestMetadata(t *testing.T) { "tableName": "mytable", } - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.NoError(t, err) assert.Equal(t, m.TableName, "mytable") }) @@ -72,7 +72,7 @@ func TestMetadata(t *testing.T) { "connectionString": "foo", } - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.NoError(t, err) assert.Equal(t, defaultTimeout*time.Second, m.Timeout) }) @@ -84,7 +84,7 @@ func TestMetadata(t *testing.T) { "timeoutInSeconds": "NaN", } - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.Error(t, err) }) @@ -95,7 +95,7 @@ func TestMetadata(t *testing.T) { "timeoutInSeconds": "42", } - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.NoError(t, err) assert.Equal(t, 42*time.Second, m.Timeout) }) @@ -107,7 +107,7 @@ func TestMetadata(t *testing.T) { "timeoutInSeconds": "0", } - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.Error(t, err) }) @@ -117,7 +117,7 @@ func TestMetadata(t *testing.T) { "connectionString": "foo", } - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.NoError(t, err) _ = assert.NotNil(t, m.CleanupInterval) && assert.Equal(t, defaultCleanupInternal*time.Second, *m.CleanupInterval) @@ -130,7 +130,7 @@ func TestMetadata(t *testing.T) { "cleanupIntervalInSeconds": "NaN", } - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.Error(t, err) }) @@ -141,7 +141,7 @@ func TestMetadata(t *testing.T) { "cleanupIntervalInSeconds": "42", } - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.NoError(t, err) _ = assert.NotNil(t, m.CleanupInterval) && assert.Equal(t, 42*time.Second, *m.CleanupInterval) @@ -154,7 +154,7 @@ func TestMetadata(t *testing.T) { "cleanupIntervalInSeconds": "0", } - err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}) + err := m.InitWithMetadata(state.Metadata{Base: metadata.Base{Properties: props}}, false) assert.NoError(t, err) assert.Nil(t, m.CleanupInterval) }) diff --git a/internal/component/postgresql/postgresdbaccess.go b/internal/component/postgresql/postgresdbaccess.go index 5a5ec16e1b..431279569c 100644 --- a/internal/component/postgresql/postgresdbaccess.go +++ b/internal/component/postgresql/postgresdbaccess.go @@ -35,8 +35,6 @@ import ( "github.com/dapr/kit/ptr" ) -var errMissingConnectionString = errors.New("missing connection string") - // Interface that applies to *pgxpool.Pool. // We need this to be able to mock the connection in tests. type PGXPoolConn interface { @@ -57,9 +55,10 @@ type PostgresDBAccess struct { gc internalsql.GarbageCollector - migrateFn func(context.Context, PGXPoolConn, MigrateOptions) error - setQueryFn func(*state.SetRequest, SetQueryOptions) string - etagColumn string + migrateFn func(context.Context, PGXPoolConn, MigrateOptions) error + setQueryFn func(*state.SetRequest, SetQueryOptions) string + etagColumn string + enableAzureAD bool } // newPostgresDBAccess creates a new instance of postgresAccess. @@ -67,13 +66,11 @@ func newPostgresDBAccess(logger logger.Logger, opts Options) *PostgresDBAccess { logger.Debug("Instantiating new Postgres state store") return &PostgresDBAccess{ - logger: logger, - metadata: postgresMetadataStruct{ - azureADEnabled: opts.EnableAzureAD, - }, - migrateFn: opts.MigrateFn, - setQueryFn: opts.SetQueryFn, - etagColumn: opts.ETagColumn, + logger: logger, + migrateFn: opts.MigrateFn, + setQueryFn: opts.SetQueryFn, + etagColumn: opts.ETagColumn, + enableAzureAD: opts.EnableAzureAD, } } @@ -81,7 +78,7 @@ func newPostgresDBAccess(logger logger.Logger, opts Options) *PostgresDBAccess { func (p *PostgresDBAccess) Init(ctx context.Context, meta state.Metadata) error { p.logger.Debug("Initializing Postgres state store") - err := p.metadata.InitWithMetadata(meta) + err := p.metadata.InitWithMetadata(meta, p.enableAzureAD) if err != nil { p.logger.Errorf("Failed to parse metadata: %v", err) return err diff --git a/state/postgresql/metadata.yaml b/state/postgresql/metadata.yaml index 2d88cf6b6b..e8eee6fbb1 100644 --- a/state/postgresql/metadata.yaml +++ b/state/postgresql/metadata.yaml @@ -37,7 +37,7 @@ builtinAuthenticationProfiles: type: string authenticationProfiles: - title: "Connection string" - description: "Authenticate using a Connection String." + description: "Authenticate using a Connection String" metadata: - name: connectionString required: true diff --git a/tests/certification/bindings/postgres/components/standard/postgres.yaml b/tests/certification/bindings/postgres/components/standard/postgres.yaml index 0c9f368f74..ebfc129b32 100644 --- a/tests/certification/bindings/postgres/components/standard/postgres.yaml +++ b/tests/certification/bindings/postgres/components/standard/postgres.yaml @@ -3,8 +3,8 @@ kind: Component metadata: name: standard-binding spec: - type: bindings.postgres + type: bindings.postgresql version: v1 metadata: - - name: url - value: "host=localhost user=postgres password=example port=5432 connect_timeout=10 database=dapr_test" \ No newline at end of file + - name: connectionString + value: "host=localhost user=postgres password=example port=5432 connect_timeout=10 database=dapr_test" diff --git a/tests/certification/bindings/postgres/docker-compose.yml b/tests/certification/bindings/postgres/docker-compose.yml index 48a388d3f9..2d62bd1b27 100644 --- a/tests/certification/bindings/postgres/docker-compose.yml +++ b/tests/certification/bindings/postgres/docker-compose.yml @@ -1,11 +1,11 @@ version: '2' services: db: - image: postgres + image: postgres:15-alpine restart: always ports: - "5432:5432" environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: example - POSTGRES_DB: dapr_test \ No newline at end of file + POSTGRES_DB: dapr_test diff --git a/tests/certification/bindings/postgres/postgres_test.go b/tests/certification/bindings/postgres/postgres_test.go index ac825be467..6cc0052f98 100644 --- a/tests/certification/bindings/postgres/postgres_test.go +++ b/tests/certification/bindings/postgres/postgres_test.go @@ -19,7 +19,8 @@ import ( "testing" "time" - _ "github.com/lib/pq" + // PGX driver for database/sql + _ "github.com/jackc/pgx/v5/stdlib" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -46,6 +47,7 @@ const ( ) func TestPostgres(t *testing.T) { + const tableName = "dapr_test_table" ports, _ := dapr_testing.GetFreePorts(3) grpcPort := ports[0] @@ -59,7 +61,7 @@ func TestPostgres(t *testing.T) { ctx.Log("Invoking output binding for exec operation!") req := &daprClient.InvokeBindingRequest{Name: "standard-binding", Operation: "exec", Metadata: metadata} - req.Metadata["sql"] = "INSERT INTO dapr_test_table (id, c1, ts) VALUES (1, 'demo', '2020-09-24T11:45:05Z07:00');" + req.Metadata["sql"] = "INSERT INTO " + tableName + " (id, c1, ts) VALUES (1, 'demo', '2020-09-24T11:45:05Z07:00');" errBinding := client.InvokeOutputBinding(ctx, req) require.NoError(ctx, errBinding, "error in output binding - exec") @@ -74,7 +76,7 @@ func TestPostgres(t *testing.T) { ctx.Log("Invoking output binding for query operation!") req := &daprClient.InvokeBindingRequest{Name: "standard-binding", Operation: "query", Metadata: metadata} - req.Metadata["sql"] = "SELECT * FROM dapr_test_table WHERE id = 1;" + req.Metadata["sql"] = "SELECT * FROM " + tableName + " WHERE id = 1;" resp, errBinding := client.InvokeBinding(ctx, req) assert.Contains(t, string(resp.Data), "1,\"demo\",\"2020-09-24T11:45:05Z07:00\"") require.NoError(ctx, errBinding, "error in output binding - query") @@ -84,17 +86,18 @@ func TestPostgres(t *testing.T) { testClose := func(ctx flow.Context) error { client, err := daprClient.NewClientWithPort(fmt.Sprintf("%d", grpcPort)) - require.NoError(t, err, "Could not initialize dapr client.") + require.NoError(ctx, err, "Could not initialize dapr client.") metadata := make(map[string]string) - ctx.Log("Invoking output binding for query operation!") + ctx.Log("Invoking output binding for close operation!") req := &daprClient.InvokeBindingRequest{Name: "standard-binding", Operation: "close", Metadata: metadata} errBinding := client.InvokeOutputBinding(ctx, req) require.NoError(ctx, errBinding, "error in output binding - close") + ctx.Log("Invoking output binding for query operation!") req = &daprClient.InvokeBindingRequest{Name: "standard-binding", Operation: "query", Metadata: metadata} - req.Metadata["sql"] = "SELECT * FROM dapr_test_table WHERE id = 1;" + req.Metadata["sql"] = "SELECT * FROM " + tableName + " WHERE id = 1;" errBinding = client.InvokeOutputBinding(ctx, req) require.Error(ctx, errBinding, "error in output binding - query") @@ -102,9 +105,9 @@ func TestPostgres(t *testing.T) { } createTable := func(ctx flow.Context) error { - db, err := sql.Open("postgres", dockerConnectionString) + db, err := sql.Open("pgx", dockerConnectionString) assert.NoError(t, err) - _, err = db.Exec("CREATE TABLE dapr_test_table(id INT, c1 TEXT, ts TEXT);") + _, err = db.Exec("CREATE TABLE " + tableName + " (id INT, c1 TEXT, ts TEXT);") assert.NoError(t, err) db.Close() return nil @@ -114,7 +117,6 @@ func TestPostgres(t *testing.T) { Step(dockercompose.Run("db", dockerComposeYAML)). Step("wait for component to start", flow.Sleep(10*time.Second)). Step("Creating table", createTable). - Step("wait for component to start", flow.Sleep(10*time.Second)). Step(sidecar.Run("standardSidecar", embedded.WithoutApp(), embedded.WithDaprGRPCPort(grpcPort), @@ -124,14 +126,13 @@ func TestPostgres(t *testing.T) { )). Step("Run exec test", testExec). Step("Run query test", testQuery). - Step("wait for DB operations to complete", flow.Sleep(10*time.Second)). Step("Run close test", testClose). Step("stop postgresql", dockercompose.Stop("db", dockerComposeYAML, "db")). - Step("wait for component to start", flow.Sleep(10*time.Second)). Run() } func TestPostgresNetworkError(t *testing.T) { + const tableName = "dapr_test_table_network" ports, _ := dapr_testing.GetFreePorts(3) grpcPort := ports[0] @@ -145,7 +146,7 @@ func TestPostgresNetworkError(t *testing.T) { ctx.Log("Invoking output binding for exec operation!") req := &daprClient.InvokeBindingRequest{Name: "standard-binding", Operation: "exec", Metadata: metadata} - req.Metadata["sql"] = "INSERT INTO dapr_test_table (id, c1, ts) VALUES (1, 'demo', '2020-09-24T11:45:05Z07:00');" + req.Metadata["sql"] = "INSERT INTO " + tableName + " (id, c1, ts) VALUES (1, 'demo', '2020-09-24T11:45:05Z07:00');" errBinding := client.InvokeOutputBinding(ctx, req) require.NoError(ctx, errBinding, "error in output binding - exec") @@ -160,7 +161,7 @@ func TestPostgresNetworkError(t *testing.T) { ctx.Log("Invoking output binding for query operation!") req := &daprClient.InvokeBindingRequest{Name: "standard-binding", Operation: "query", Metadata: metadata} - req.Metadata["sql"] = "SELECT * FROM dapr_test_table WHERE id = 1;" + req.Metadata["sql"] = "SELECT * FROM " + tableName + " WHERE id = 1;" errBinding := client.InvokeOutputBinding(ctx, req) require.NoError(ctx, errBinding, "error in output binding - query") @@ -168,9 +169,9 @@ func TestPostgresNetworkError(t *testing.T) { } createTable := func(ctx flow.Context) error { - db, err := sql.Open("postgres", dockerConnectionString) + db, err := sql.Open("pgx", dockerConnectionString) assert.NoError(t, err) - _, err = db.Exec("CREATE TABLE dapr_test_table(id INT, c1 TEXT, ts TEXT);") + _, err = db.Exec("CREATE TABLE " + tableName + "(id INT, c1 TEXT, ts TEXT);") assert.NoError(t, err) db.Close() return nil @@ -180,7 +181,6 @@ func TestPostgresNetworkError(t *testing.T) { Step(dockercompose.Run("db", dockerComposeYAML)). Step("wait for component to start", flow.Sleep(10*time.Second)). Step("Creating table", createTable). - Step("wait for component to start", flow.Sleep(10*time.Second)). Step(sidecar.Run("standardSidecar", embedded.WithoutApp(), embedded.WithDaprGRPCPort(grpcPort), @@ -190,8 +190,8 @@ func TestPostgresNetworkError(t *testing.T) { )). Step("Run exec test", testExec). Step("Run query test", testQuery). - Step("wait for DB operations to complete", flow.Sleep(10*time.Second)). - Step("interrupt network", network.InterruptNetwork(30*time.Second, nil, nil, "5432")). + Step("wait for DB operations to complete", flow.Sleep(5*time.Second)). + Step("interrupt network", network.InterruptNetwork(20*time.Second, nil, nil, "5432")). Step("wait for component to recover", flow.Sleep(10*time.Second)). Step("Run query test", testQuery). Run() @@ -204,7 +204,7 @@ func componentRuntimeOptions() []runtime.Option { bindingsRegistry.Logger = log bindingsRegistry.RegisterOutputBinding(func(l logger.Logger) bindings.OutputBinding { return binding_postgres.NewPostgres(l) - }, "postgres") + }, "postgresql") return []runtime.Option{ runtime.WithBindings(bindingsRegistry), diff --git a/tests/certification/configuration/postgres/docker-compose.yml b/tests/certification/configuration/postgres/docker-compose.yml index 560a5d111c..dd46ec9693 100644 --- a/tests/certification/configuration/postgres/docker-compose.yml +++ b/tests/certification/configuration/postgres/docker-compose.yml @@ -1,7 +1,7 @@ version: '2' services: db: - image: postgres:15 + image: postgres:15-alpine restart: always ports: - "5432:5432" diff --git a/tests/certification/go.mod b/tests/certification/go.mod index 8896bf8d2f..6c16649bd7 100644 --- a/tests/certification/go.mod +++ b/tests/certification/go.mod @@ -28,7 +28,6 @@ require ( github.com/google/uuid v1.3.0 github.com/jackc/pgx/v5 v5.3.1 github.com/lestrrat-go/jwx/v2 v2.0.11 - github.com/lib/pq v1.10.7 github.com/nacos-group/nacos-sdk-go/v2 v2.1.3 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/rabbitmq/amqp091-go v1.8.1 diff --git a/tests/certification/go.sum b/tests/certification/go.sum index 3bdfe28575..9f6d9eb0f8 100644 --- a/tests/certification/go.sum +++ b/tests/certification/go.sum @@ -893,8 +893,6 @@ github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmt github.com/lestrrat/go-envload v0.0.0-20180220120943-6ed08b54a570/go.mod h1:BLt8L9ld7wVsvEWQbuLrUZnCMnUmLZ+CGDzKtclrTlE= github.com/lestrrat/go-file-rotatelogs v0.0.0-20180223000712-d3151e2a480f/go.mod h1:UGmTpUd3rjbtfIpwAPrcfmGf/Z1HS95TATB+m57TPB8= github.com/lestrrat/go-strftime v0.0.0-20180220042222-ba3bf9c1d042/go.mod h1:TPpsiPUEh0zFL1Snz4crhMlBe60PYxRHr5oFF3rRYg0= -github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= -github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg= diff --git a/tests/certification/state/postgresql/docker-compose.yml b/tests/certification/state/postgresql/docker-compose.yml index 48a388d3f9..dd46ec9693 100644 --- a/tests/certification/state/postgresql/docker-compose.yml +++ b/tests/certification/state/postgresql/docker-compose.yml @@ -1,7 +1,7 @@ version: '2' services: db: - image: postgres + image: postgres:15-alpine restart: always ports: - "5432:5432" diff --git a/tests/config/bindings/postgres/bindings.yml b/tests/config/bindings/postgres/bindings.yml deleted file mode 100644 index be68503baa..0000000000 --- a/tests/config/bindings/postgres/bindings.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: postgres-binding -spec: - type: bindings.postgres - version: v1 - metadata: - - name: url # Required - value: "host=localhost user=postgres password=example port=5432 connect_timeout=10 database=dapr_test" diff --git a/tests/config/bindings/postgresql/azure/bindings.yml b/tests/config/bindings/postgresql/azure/bindings.yml new file mode 100644 index 0000000000..7d66ed5021 --- /dev/null +++ b/tests/config/bindings/postgresql/azure/bindings.yml @@ -0,0 +1,18 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: postgres-binding +spec: + type: bindings.postgresql + version: v1 + metadata: + - name: connectionString + value: "${{AzureDBPostgresConnectionString}}" + - name: azureClientId + value: "${{AzureDBPostgresClientId}}" + - name: azureClientSecret + value: "${{AzureDBPostgresClientSecret}}" + - name: azureTenantId + value: "${{AzureDBPostgresTenantId}}" + - name: useAzureAD + value: "true" \ No newline at end of file diff --git a/tests/config/bindings/postgresql/docker/bindings.yml b/tests/config/bindings/postgresql/docker/bindings.yml new file mode 100644 index 0000000000..f6b9679791 --- /dev/null +++ b/tests/config/bindings/postgresql/docker/bindings.yml @@ -0,0 +1,11 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: postgres-binding +spec: + type: bindings.postgresql + version: v1 + metadata: + # "url" is the old name for "connectionString" and is kept here to test for backwards-compatibility + - name: url + value: "host=localhost user=postgres password=example port=5432 connect_timeout=10 database=dapr_test" diff --git a/tests/config/bindings/tests.yml b/tests/config/bindings/tests.yml index b8efdcf7a8..62c83c0c81 100644 --- a/tests/config/bindings/tests.yml +++ b/tests/config/bindings/tests.yml @@ -73,7 +73,9 @@ components: checkInOrderProcessing: false - component: kubemq operations: [ "create", "operations", "read" ] - - component: postgres + - component: postgresql.docker + operations: [ "exec", "query", "close", "operations" ] + - component: postgresql.azure operations: [ "exec", "query", "close", "operations" ] - component: aws.s3.docker operations: ["create", "operations", "get", "list"] diff --git a/tests/config/configuration/postgresql/azure/configstore.yml b/tests/config/configuration/postgresql/azure/configstore.yml new file mode 100644 index 0000000000..375f4f5391 --- /dev/null +++ b/tests/config/configuration/postgresql/azure/configstore.yml @@ -0,0 +1,20 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: configstore +spec: + type: configuration.postgresql + version: v1 + metadata: + - name: connectionString + value: "${{AzureDBPostgresConnectionString}}" + - name: azureClientId + value: "${{AzureDBPostgresClientId}}" + - name: azureClientSecret + value: "${{AzureDBPostgresClientSecret}}" + - name: azureTenantId + value: "${{AzureDBPostgresTenantId}}" + - name: useAzureAD + value: "true" + - name: table + value: configtable \ No newline at end of file diff --git a/tests/config/configuration/postgres/configstore.yml b/tests/config/configuration/postgresql/docker/configstore.yml similarity index 85% rename from tests/config/configuration/postgres/configstore.yml rename to tests/config/configuration/postgresql/docker/configstore.yml index b2d313707e..5f408c6f4d 100644 --- a/tests/config/configuration/postgres/configstore.yml +++ b/tests/config/configuration/postgresql/docker/configstore.yml @@ -3,7 +3,8 @@ kind: Component metadata: name: configstore spec: - type: configuration.postgres + type: configuration.postgresql + version: v1 metadata: - name: connectionString value: "host=localhost user=postgres password=example port=5432 connect_timeout=10 database=dapr_test" diff --git a/tests/config/configuration/tests.yml b/tests/config/configuration/tests.yml index 17dd6354a0..6713453afa 100644 --- a/tests/config/configuration/tests.yml +++ b/tests/config/configuration/tests.yml @@ -5,5 +5,7 @@ components: operations: [] - component: redis.v7 operations: [] - - component: postgres + - component: postgresql.azure + operations: [] + - component: postgresql.docker operations: [] diff --git a/tests/config/state/postgresql/azure/statestore.yml b/tests/config/state/postgresql/azure/statestore.yml index 1788537e8e..b58a92c49c 100644 --- a/tests/config/state/postgresql/azure/statestore.yml +++ b/tests/config/state/postgresql/azure/statestore.yml @@ -8,5 +8,11 @@ spec: metadata: - name: connectionString value: "${{AzureDBPostgresConnectionString}}" + - name: azureClientId + value: "${{AzureDBPostgresClientId}}" + - name: azureClientSecret + value: "${{AzureDBPostgresClientSecret}}" + - name: azureTenantId + value: "${{AzureDBPostgresTenantId}}" - name: useAzureAD value: "true" \ No newline at end of file diff --git a/tests/conformance/common.go b/tests/conformance/common.go index 7ba46cd7e5..bfd4850992 100644 --- a/tests/conformance/common.go +++ b/tests/conformance/common.go @@ -431,13 +431,10 @@ func loadConfigurationStore(tc TestComponent) (configuration.Store, configupdate var store configuration.Store var updater configupdater.Updater switch tc.Component { - case redisv6: - store = c_redis.NewRedisConfigurationStore(testLogger) - updater = cu_redis.NewRedisConfigUpdater(testLogger) - case redisv7: + case redisv6, redisv7: store = c_redis.NewRedisConfigurationStore(testLogger) updater = cu_redis.NewRedisConfigUpdater(testLogger) - case "postgres": + case "postgresql.docker", "postgresql.azure": store = c_postgres.NewPostgresConfigurationStore(testLogger) updater = cu_postgres.NewPostgresConfigUpdater(testLogger) default: @@ -624,7 +621,9 @@ func loadOutputBindings(tc TestComponent) bindings.OutputBinding { binding = b_rabbitmq.NewRabbitMQ(testLogger) case "kubemq": binding = b_kubemq.NewKubeMQ(testLogger) - case "postgres": + case "postgresql.docker": + binding = b_postgres.NewPostgres(testLogger) + case "postgresql.azure": binding = b_postgres.NewPostgres(testLogger) case "aws.s3.docker": binding = b_aws_s3.NewAWSS3(testLogger) diff --git a/tests/conformance/configuration/configuration.go b/tests/conformance/configuration/configuration.go index 4dd76bac70..988e6192c5 100644 --- a/tests/conformance/configuration/configuration.go +++ b/tests/conformance/configuration/configuration.go @@ -37,7 +37,7 @@ const ( v1 = "1.0.0" defaultMaxReadDuration = 30 * time.Second defaultWaitDuration = 5 * time.Second - postgresComponent = "postgres" + postgresComponent = "postgresql" pgNotifyChannelKey = "pgNotifyChannel" pgNotifyChannel = "config" ) @@ -152,7 +152,7 @@ func ConformanceTests(t *testing.T, props map[string]string, store configuration require.NoError(t, err) // Creating trigger for postgres config updater - if component == postgresComponent { + if strings.HasPrefix(component, postgresComponent) { err = updater.(*postgres_updater.ConfigUpdater).CreateTrigger(pgNotifyChannel) require.NoError(t, err) } @@ -223,7 +223,7 @@ func ConformanceTests(t *testing.T, props map[string]string, store configuration t.Run("subscribe", func(t *testing.T) { subscribeMetadata := make(map[string]string) - if component == postgresComponent { + if strings.HasPrefix(component, postgresComponent) { subscribeMetadata[pgNotifyChannelKey] = pgNotifyChannel } t.Run("subscriber 1 with non-empty key list", func(t *testing.T) { @@ -307,7 +307,7 @@ func ConformanceTests(t *testing.T, props map[string]string, store configuration // Delete initValues2 errDelete := updater.DeleteKey(getKeys(initValues2)) assert.NoError(t, errDelete, "expected no error on updating keys") - if component != postgresComponent { + if !strings.HasPrefix(component, postgresComponent) { for k := range initValues2 { initValues2[k] = &configuration.Item{} } @@ -323,10 +323,9 @@ func ConformanceTests(t *testing.T, props map[string]string, store configuration t.Run("unsubscribe", func(t *testing.T) { t.Run("unsubscribe subscriber 1", func(t *testing.T) { - ID1 := subscribeIDs[0] err := store.Unsubscribe(context.Background(), &configuration.UnsubscribeRequest{ - ID: ID1, + ID: subscribeIDs[0], }, ) assert.NoError(t, err, "expected no error in unsubscribe") @@ -346,10 +345,9 @@ func ConformanceTests(t *testing.T, props map[string]string, store configuration }) t.Run("unsubscribe subscriber 2", func(t *testing.T) { - ID2 := subscribeIDs[1] err := store.Unsubscribe(context.Background(), &configuration.UnsubscribeRequest{ - ID: ID2, + ID: subscribeIDs[1], }, ) assert.NoError(t, err, "expected no error in unsubscribe") @@ -367,10 +365,9 @@ func ConformanceTests(t *testing.T, props map[string]string, store configuration }) t.Run("unsubscribe subscriber 3", func(t *testing.T) { - ID3 := subscribeIDs[2] err := store.Unsubscribe(context.Background(), &configuration.UnsubscribeRequest{ - ID: ID3, + ID: subscribeIDs[2], }, ) assert.NoError(t, err, "expected no error in unsubscribe") diff --git a/tests/utils/configupdater/postgres/postgres.go b/tests/utils/configupdater/postgres/postgres.go index 609bdd469b..ce5b02bb2b 100644 --- a/tests/utils/configupdater/postgres/postgres.go +++ b/tests/utils/configupdater/postgres/postgres.go @@ -5,10 +5,13 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/jackc/pgx/v5/pgxpool" "github.com/dapr/components-contrib/configuration" + pgauth "github.com/dapr/components-contrib/internal/authentication/postgresql" + "github.com/dapr/components-contrib/internal/utils" "github.com/dapr/components-contrib/tests/utils/configupdater" "github.com/dapr/kit/logger" ) @@ -72,7 +75,7 @@ func (r *ConfigUpdater) CreateTrigger(channel string) error { return fmt.Errorf("error creating config event function : %w", err) } triggerName := "configTrigger_" + channel - createTriggerSQL := "CREATE TRIGGER " + triggerName + " AFTER INSERT OR UPDATE OR DELETE ON " + r.configTable + " FOR EACH ROW EXECUTE PROCEDURE " + procedureName + "();" + createTriggerSQL := "CREATE OR REPLACE TRIGGER " + triggerName + " AFTER INSERT OR UPDATE OR DELETE ON " + r.configTable + " FOR EACH ROW EXECUTE PROCEDURE " + procedureName + "();" _, err = r.client.Exec(ctx, createTriggerSQL) if err != nil { return fmt.Errorf("error creating config trigger : %w", err) @@ -81,31 +84,38 @@ func (r *ConfigUpdater) CreateTrigger(channel string) error { } func (r *ConfigUpdater) Init(props map[string]string) error { - var conn string - ctx := context.Background() - if val, ok := props["connectionString"]; ok && val != "" { - conn = val - } else { - return fmt.Errorf("missing postgreSQL connection string") + md := pgauth.PostgresAuthMetadata{ + ConnectionString: props["connectionString"], + UseAzureAD: utils.IsTruthy(props["useAzureAD"]), + } + err := md.InitWithMetadata(props, true) + if err != nil { + return err } + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if tbl, ok := props["table"]; ok && tbl != "" { r.configTable = tbl } else { return fmt.Errorf("missing postgreSQL configuration table name") } - config, err := pgxpool.ParseConfig(conn) + + config, err := md.GetPgxPoolConfig() if err != nil { return fmt.Errorf("postgres configuration store connection error : %w", err) } - pool, err := pgxpool.NewWithConfig(ctx, config) + + r.client, err = pgxpool.NewWithConfig(ctx, config) if err != nil { return fmt.Errorf("postgres configuration store connection error : %w", err) } - err = pool.Ping(ctx) + err = r.client.Ping(ctx) if err != nil { return fmt.Errorf("postgres configuration store ping error : %w", err) } - r.client = pool + err = createAndSetTable(ctx, r.client, r.configTable) if err != nil { return fmt.Errorf("postgres configuration store table creation error : %w", err) From 1ab15ef04ba571ff7c7c51b299d77afaa4565071 Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Wed, 12 Jul 2023 15:34:14 -0700 Subject: [PATCH 5/8] Postgres binding: support parametrized queries (#2972) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> --- bindings/postgres/postgres.go | 30 ++++-- .../certification/bindings/postgres/README.md | 1 + .../bindings/postgres/postgres_test.go | 99 ++++++++++++------- 3 files changed, 87 insertions(+), 43 deletions(-) diff --git a/bindings/postgres/postgres.go b/bindings/postgres/postgres.go index 9919f9b1d0..637b88d344 100644 --- a/bindings/postgres/postgres.go +++ b/bindings/postgres/postgres.go @@ -36,7 +36,8 @@ const ( queryOperation bindings.OperationKind = "query" closeOperation bindings.OperationKind = "close" - commandSQLKey = "sql" + commandSQLKey = "sql" + commandArgsKey = "params" ) // Postgres represents PostgreSQL output binding. @@ -109,11 +110,22 @@ func (p *Postgres) Invoke(ctx context.Context, req *bindings.InvokeRequest) (res return nil, errors.New("metadata required") } - sql, ok := req.Metadata[commandSQLKey] - if !ok || sql == "" { + // Metadata property "sql" contains the query to execute + sql := req.Metadata[commandSQLKey] + if sql == "" { return nil, fmt.Errorf("required metadata not set: %s", commandSQLKey) } + // Metadata property "params" contains JSON-encoded parameters, and it's optional + // If present, it must be unserializable into a []any object + var args []any + if argsStr := req.Metadata[commandArgsKey]; argsStr != "" { + err = json.Unmarshal([]byte(argsStr), &args) + if err != nil { + return nil, fmt.Errorf("invalid metadata property %s: failed to unserialize into an array: %w", commandArgsKey, err) + } + } + startTime := time.Now().UTC() resp = &bindings.InvokeResponse{ Metadata: map[string]string{ @@ -125,14 +137,14 @@ func (p *Postgres) Invoke(ctx context.Context, req *bindings.InvokeRequest) (res switch req.Operation { //nolint:exhaustive case execOperation: - r, err := p.exec(ctx, sql) + r, err := p.exec(ctx, sql, args...) if err != nil { return nil, err } resp.Metadata["rows-affected"] = strconv.FormatInt(r, 10) // 0 if error case queryOperation: - d, err := p.query(ctx, sql) + d, err := p.query(ctx, sql, args...) if err != nil { return nil, err } @@ -168,8 +180,8 @@ func (p *Postgres) Close() error { return nil } -func (p *Postgres) query(ctx context.Context, sql string) (result []byte, err error) { - rows, err := p.db.Query(ctx, sql) +func (p *Postgres) query(ctx context.Context, sql string, args ...any) (result []byte, err error) { + rows, err := p.db.Query(ctx, sql, args...) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } @@ -191,8 +203,8 @@ func (p *Postgres) query(ctx context.Context, sql string) (result []byte, err er return result, nil } -func (p *Postgres) exec(ctx context.Context, sql string) (result int64, err error) { - res, err := p.db.Exec(ctx, sql) +func (p *Postgres) exec(ctx context.Context, sql string, args ...any) (result int64, err error) { + res, err := p.db.Exec(ctx, sql, args...) if err != nil { return 0, fmt.Errorf("error executing query: %w", err) } diff --git a/tests/certification/bindings/postgres/README.md b/tests/certification/bindings/postgres/README.md index 1759d74259..f30909c863 100644 --- a/tests/certification/bindings/postgres/README.md +++ b/tests/certification/bindings/postgres/README.md @@ -17,6 +17,7 @@ The purpose of this module is to provide tests that certify the PostgreSQL Outpu * Run dapr application with component to store data in postgres as output binding. * Read stored data from postgres. * Ensure that read data is same as the data that was stored. + * Verify the ability to use named paramters in queries. * Verify reconnection to postgres for output binding. * Simulate a network error before sending any messages. * Run dapr application with the component. diff --git a/tests/certification/bindings/postgres/postgres_test.go b/tests/certification/bindings/postgres/postgres_test.go index 6cc0052f98..deddaaccb2 100644 --- a/tests/certification/bindings/postgres/postgres_test.go +++ b/tests/certification/bindings/postgres/postgres_test.go @@ -55,31 +55,58 @@ func TestPostgres(t *testing.T) { testExec := func(ctx flow.Context) error { client, err := daprClient.NewClientWithPort(fmt.Sprintf("%d", grpcPort)) - require.NoError(t, err, "Could not initialize dapr client.") - - metadata := make(map[string]string) - - ctx.Log("Invoking output binding for exec operation!") - req := &daprClient.InvokeBindingRequest{Name: "standard-binding", Operation: "exec", Metadata: metadata} - req.Metadata["sql"] = "INSERT INTO " + tableName + " (id, c1, ts) VALUES (1, 'demo', '2020-09-24T11:45:05Z07:00');" - errBinding := client.InvokeOutputBinding(ctx, req) - require.NoError(ctx, errBinding, "error in output binding - exec") + require.NoError(t, err, "Could not initialize dapr client") + + ctx.Log("Invoking output binding for exec operation") + err = client.InvokeOutputBinding(ctx, &daprClient.InvokeBindingRequest{ + Name: "standard-binding", + Operation: "exec", + Metadata: map[string]string{ + "sql": "INSERT INTO " + tableName + " (id, c1, ts) VALUES (1, 'demo', '2020-09-24T11:45:05+07:00');", + }, + }) + require.NoError(ctx, err, "error in output binding - exec") + + ctx.Log("Invoking output binding for exec operation with parameters") + err = client.InvokeOutputBinding(ctx, &daprClient.InvokeBindingRequest{ + Name: "standard-binding", + Operation: "exec", + Metadata: map[string]string{ + "sql": "INSERT INTO " + tableName + " (id, c1, ts) VALUES ($1, $2, $3);", + "params": `[2, "demo2", "2021-03-19T11:45:05+07:00"]`, + }, + }) + require.NoError(ctx, err, "error in output binding - exec") return nil } testQuery := func(ctx flow.Context) error { client, err := daprClient.NewClientWithPort(fmt.Sprintf("%d", grpcPort)) - require.NoError(t, err, "Could not initialize dapr client.") - - metadata := make(map[string]string) - - ctx.Log("Invoking output binding for query operation!") - req := &daprClient.InvokeBindingRequest{Name: "standard-binding", Operation: "query", Metadata: metadata} - req.Metadata["sql"] = "SELECT * FROM " + tableName + " WHERE id = 1;" - resp, errBinding := client.InvokeBinding(ctx, req) - assert.Contains(t, string(resp.Data), "1,\"demo\",\"2020-09-24T11:45:05Z07:00\"") - require.NoError(ctx, errBinding, "error in output binding - query") + require.NoError(t, err, "Could not initialize dapr client") + + ctx.Log("Invoking output binding for query operation") + resp, err := client.InvokeBinding(ctx, &daprClient.InvokeBindingRequest{ + Name: "standard-binding", + Operation: "query", + Metadata: map[string]string{ + "sql": "SELECT * FROM " + tableName + " WHERE id = 1;", + }, + }) + assert.Equal(t, `[[1,"demo","2020-09-24T11:45:05Z"]]`, string(resp.Data)) + require.NoError(ctx, err, "error in output binding - query") + + ctx.Log("Invoking output binding for query operation with parameters") + resp, err = client.InvokeBinding(ctx, &daprClient.InvokeBindingRequest{ + Name: "standard-binding", + Operation: "query", + Metadata: map[string]string{ + "sql": "SELECT * FROM " + tableName + " WHERE id = ANY($1);", + "params": `[[1, 2]]`, + }, + }) + assert.Equal(t, `[[1,"demo","2020-09-24T11:45:05Z"],[2,"demo2","2021-03-19T11:45:05Z"]]`, string(resp.Data)) + require.NoError(ctx, err, "error in output binding - query") return nil } @@ -107,8 +134,8 @@ func TestPostgres(t *testing.T) { createTable := func(ctx flow.Context) error { db, err := sql.Open("pgx", dockerConnectionString) assert.NoError(t, err) - _, err = db.Exec("CREATE TABLE " + tableName + " (id INT, c1 TEXT, ts TEXT);") - assert.NoError(t, err) + _, err = db.Exec("CREATE TABLE " + tableName + " (id INT, c1 TEXT, ts TIMESTAMP);") + require.NoError(t, err) db.Close() return nil } @@ -140,14 +167,16 @@ func TestPostgresNetworkError(t *testing.T) { testExec := func(ctx flow.Context) error { client, err := daprClient.NewClientWithPort(fmt.Sprintf("%d", grpcPort)) - require.NoError(t, err, "Could not initialize dapr client.") - - metadata := make(map[string]string) + require.NoError(t, err, "Could not initialize dapr client") ctx.Log("Invoking output binding for exec operation!") - req := &daprClient.InvokeBindingRequest{Name: "standard-binding", Operation: "exec", Metadata: metadata} - req.Metadata["sql"] = "INSERT INTO " + tableName + " (id, c1, ts) VALUES (1, 'demo', '2020-09-24T11:45:05Z07:00');" - errBinding := client.InvokeOutputBinding(ctx, req) + errBinding := client.InvokeOutputBinding(ctx, &daprClient.InvokeBindingRequest{ + Name: "standard-binding", + Operation: "exec", + Metadata: map[string]string{ + "sql": "INSERT INTO " + tableName + " (id, c1, ts) VALUES (1, 'demo', '2020-09-24T11:45:05+07:00');", + }, + }) require.NoError(ctx, errBinding, "error in output binding - exec") return nil @@ -155,14 +184,16 @@ func TestPostgresNetworkError(t *testing.T) { testQuery := func(ctx flow.Context) error { client, err := daprClient.NewClientWithPort(fmt.Sprintf("%d", grpcPort)) - require.NoError(t, err, "Could not initialize dapr client.") - - metadata := make(map[string]string) + require.NoError(t, err, "Could not initialize dapr client") ctx.Log("Invoking output binding for query operation!") - req := &daprClient.InvokeBindingRequest{Name: "standard-binding", Operation: "query", Metadata: metadata} - req.Metadata["sql"] = "SELECT * FROM " + tableName + " WHERE id = 1;" - errBinding := client.InvokeOutputBinding(ctx, req) + errBinding := client.InvokeOutputBinding(ctx, &daprClient.InvokeBindingRequest{ + Name: "standard-binding", + Operation: "query", + Metadata: map[string]string{ + "sql": "SELECT * FROM " + tableName + " WHERE id = 1;", + }, + }) require.NoError(ctx, errBinding, "error in output binding - query") return nil @@ -171,7 +202,7 @@ func TestPostgresNetworkError(t *testing.T) { createTable := func(ctx flow.Context) error { db, err := sql.Open("pgx", dockerConnectionString) assert.NoError(t, err) - _, err = db.Exec("CREATE TABLE " + tableName + "(id INT, c1 TEXT, ts TEXT);") + _, err = db.Exec("CREATE TABLE " + tableName + " (id INT, c1 TEXT, ts TIMESTAMP);") assert.NoError(t, err) db.Close() return nil From 95045c4dfeeff85a9792955b87004a1e1a401c63 Mon Sep 17 00:00:00 2001 From: Shivam Kumar Singh Date: Fri, 14 Jul 2023 02:57:42 +0530 Subject: [PATCH 6/8] Add output binding for OpenAI (#2965) Signed-off-by: Shivam Kumar Singh Signed-off-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> --- bindings/azure/openai/metadata.yaml | 38 ++++ bindings/azure/openai/openai.go | 326 ++++++++++++++++++++++++++++ go.mod | 3 +- go.sum | 6 +- tests/certification/go.mod | 2 +- tests/certification/go.sum | 4 +- 6 files changed, 373 insertions(+), 6 deletions(-) create mode 100644 bindings/azure/openai/metadata.yaml create mode 100644 bindings/azure/openai/openai.go diff --git a/bindings/azure/openai/metadata.yaml b/bindings/azure/openai/metadata.yaml new file mode 100644 index 0000000000..f579916940 --- /dev/null +++ b/bindings/azure/openai/metadata.yaml @@ -0,0 +1,38 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: bindings +name: azure.openai +version: v1 +status: alpha +title: "Azure OpenAI" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-bindings/azure-openai/ +binding: + output: true + input: false + operations: + - name: completion + description: "Text completion" + - name: chat-completion + description: "Chat completion" +builtinAuthenticationProfiles: + - name: "azuread" +authenticationProfiles: + - title: "API Key" + description: "Authenticate using an API key" + metadata: + - name: apiKey + required: true + sensitive: true + description: "API Key" + example: '"1234567890abcdef"' +metadata: + - name: endpoint + required: true + description: "Endpoint of the Azure OpenAI service" + example: '"https://myopenai.openai.azure.com"' + - name: deploymentID + required: true + description: "ID of the model deployment in the Azure OpenAI service" + example: '"my-model"' diff --git a/bindings/azure/openai/openai.go b/bindings/azure/openai/openai.go new file mode 100644 index 0000000000..83d189203a --- /dev/null +++ b/bindings/azure/openai/openai.go @@ -0,0 +1,326 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openai + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai" + + "github.com/dapr/components-contrib/bindings" + azauth "github.com/dapr/components-contrib/internal/authentication/azure" + "github.com/dapr/components-contrib/metadata" + "github.com/dapr/kit/config" + "github.com/dapr/kit/logger" +) + +// List of operations. +const ( + CompletionOperation bindings.OperationKind = "completion" + ChatCompletionOperation bindings.OperationKind = "chat-completion" + + APIKey = "apiKey" + DeploymentID = "deploymentID" + Endpoint = "endpoint" + MessagesKey = "messages" + Temperature = "temperature" + MaxTokens = "maxTokens" + TopP = "topP" + N = "n" + Stop = "stop" + FrequencyPenalty = "frequencyPenalty" + LogitBias = "logitBias" + User = "user" +) + +// AzOpenAI represents OpenAI output binding. +type AzOpenAI struct { + logger logger.Logger + client *azopenai.Client +} + +type openAIMetadata struct { + // APIKey is the API key for the Azure OpenAI API. + APIKey string `mapstructure:"apiKey"` + // DeploymentID is the deployment ID for the Azure OpenAI API. + DeploymentID string `mapstructure:"deploymentID"` + // Endpoint is the endpoint for the Azure OpenAI API. + Endpoint string `mapstructure:"endpoint"` +} + +type ChatSettings struct { + Temperature float32 `mapstructure:"temperature"` + MaxTokens int32 `mapstructure:"maxTokens"` + TopP float32 `mapstructure:"topP"` + N int32 `mapstructure:"n"` + PresencePenalty float32 `mapstructure:"presencePenalty"` + FrequencyPenalty float32 `mapstructure:"frequencyPenalty"` +} + +// ChatMessages type for chat completion API. +type ChatMessages struct { + Messages []Message `json:"messages"` + Temperature float32 `json:"temperature"` + MaxTokens int32 `json:"maxTokens"` + TopP float32 `json:"topP"` + N int32 `json:"n"` + PresencePenalty float32 `json:"presencePenalty"` + FrequencyPenalty float32 `json:"frequencyPenalty"` +} + +// Message type stores the messages for bot conversation. +type Message struct { + Role string + Message string +} + +// Prompt type for completion API. +type Prompt struct { + Prompt string `json:"prompt"` + Temperature float32 `json:"temperature"` + MaxTokens int32 `json:"maxTokens"` + TopP float32 `json:"topP"` + N int32 `json:"n"` + PresencePenalty float32 `json:"presencePenalty"` + FrequencyPenalty float32 `json:"frequencyPenalty"` +} + +// NewOpenAI returns a new OpenAI output binding. +func NewOpenAI(logger logger.Logger) bindings.OutputBinding { + return &AzOpenAI{ + logger: logger, + } +} + +// Init initializes the OpenAI binding. +func (p *AzOpenAI) Init(ctx context.Context, meta bindings.Metadata) error { + m := openAIMetadata{} + err := metadata.DecodeMetadata(meta.Properties, &m) + if err != nil { + return fmt.Errorf("error decoding metadata: %w", err) + } + if m.Endpoint == "" { + return fmt.Errorf("required metadata not set: %s", Endpoint) + } + if m.DeploymentID == "" { + return fmt.Errorf("required metadata not set: %s", DeploymentID) + } + + if m.APIKey != "" { + // use API key authentication + var keyCredential azopenai.KeyCredential + keyCredential, err = azopenai.NewKeyCredential(m.APIKey) + if err != nil { + return fmt.Errorf("error getting credentials object: %w", err) + } + + p.client, err = azopenai.NewClientWithKeyCredential(m.Endpoint, keyCredential, m.DeploymentID, nil) + if err != nil { + return fmt.Errorf("error creating Azure OpenAI client: %w", err) + } + } else { + // fallback to Azure AD authentication + settings, innerErr := azauth.NewEnvironmentSettings(meta.Properties) + if innerErr != nil { + return fmt.Errorf("error creating environment settings: %w", innerErr) + } + + token, innerErr := settings.GetTokenCredential() + if innerErr != nil { + return fmt.Errorf("error getting token credential: %w", innerErr) + } + + p.client, err = azopenai.NewClient(m.Endpoint, token, m.DeploymentID, nil) + if err != nil { + return fmt.Errorf("error creating Azure OpenAI client: %w", err) + } + } + + return nil +} + +// Operations returns list of operations supported by OpenAI binding. +func (p *AzOpenAI) Operations() []bindings.OperationKind { + return []bindings.OperationKind{ + ChatCompletionOperation, + CompletionOperation, + } +} + +// Invoke handles all invoke operations. +func (p *AzOpenAI) Invoke(ctx context.Context, req *bindings.InvokeRequest) (resp *bindings.InvokeResponse, err error) { + if req == nil || len(req.Metadata) == 0 { + return nil, fmt.Errorf("invalid request: metadata is required") + } + + startTime := time.Now().UTC() + resp = &bindings.InvokeResponse{ + Metadata: map[string]string{ + "operation": string(req.Operation), + "start-time": startTime.Format(time.RFC3339Nano), + }, + } + + switch req.Operation { //nolint:exhaustive + case CompletionOperation: + response, err := p.completion(ctx, req.Data, req.Metadata) + if err != nil { + return nil, fmt.Errorf("error performing completion: %w", err) + } + responseAsBytes, _ := json.Marshal(response) + resp.Data = responseAsBytes + + case ChatCompletionOperation: + response, err := p.chatCompletion(ctx, req.Data, req.Metadata) + if err != nil { + return nil, fmt.Errorf("error performing chat completion: %w", err) + } + responseAsBytes, _ := json.Marshal(response) + resp.Data = responseAsBytes + + default: + return nil, fmt.Errorf( + "invalid operation type: %s. Expected %s, %s", + req.Operation, CompletionOperation, ChatCompletionOperation, + ) + } + + endTime := time.Now().UTC() + resp.Metadata["end-time"] = endTime.Format(time.RFC3339Nano) + resp.Metadata["duration"] = endTime.Sub(startTime).String() + + return resp, nil +} + +func (s *ChatSettings) Decode(in any) error { + return config.Decode(in, s) +} + +func (p *AzOpenAI) completion(ctx context.Context, message []byte, metadata map[string]string) (response []azopenai.Choice, err error) { + prompt := Prompt{ + Temperature: 1.0, + TopP: 1.0, + MaxTokens: 16, + N: 1, + PresencePenalty: 0.0, + FrequencyPenalty: 0.0, + } + err = json.Unmarshal(message, &prompt) + if err != nil { + return nil, fmt.Errorf("error unmarshalling the input object: %w", err) + } + + if prompt.Prompt == "" { + return nil, fmt.Errorf("prompt is required for completion operation") + } + + resp, err := p.client.GetCompletions(ctx, azopenai.CompletionsOptions{ + Prompt: []*string{&prompt.Prompt}, + MaxTokens: &prompt.MaxTokens, + Temperature: &prompt.Temperature, + TopP: &prompt.TopP, + N: &prompt.N, + }, nil) + if err != nil { + return nil, fmt.Errorf("error getting completion api: %w", err) + } + + // No choices returned + if len(resp.Completions.Choices) == 0 { + return []azopenai.Choice{}, nil + } + + choices := resp.Completions.Choices + response = make([]azopenai.Choice, len(choices)) + for i, c := range choices { + response[i] = *c + } + + return response, nil +} + +func (p *AzOpenAI) chatCompletion(ctx context.Context, messageRequest []byte, metadata map[string]string) (response []azopenai.ChatChoice, err error) { + messages := ChatMessages{ + Temperature: 1.0, + TopP: 1.0, + N: 1, + PresencePenalty: 0.0, + FrequencyPenalty: 0.0, + } + err = json.Unmarshal(messageRequest, &messages) + if err != nil { + return nil, fmt.Errorf("error unmarshalling the input object: %w", err) + } + + if len(messages.Messages) == 0 { + return nil, fmt.Errorf("messages are required for chat-completion operation") + } + + messageReq := make([]*azopenai.ChatMessage, len(messages.Messages)) + for i, m := range messages.Messages { + messageReq[i] = &azopenai.ChatMessage{ + Role: to.Ptr(azopenai.ChatRole(m.Role)), + Content: to.Ptr(m.Message), + } + } + + var maxTokens *int32 + if messages.MaxTokens != 0 { + maxTokens = &messages.MaxTokens + } + + res, err := p.client.GetChatCompletions(ctx, azopenai.ChatCompletionsOptions{ + MaxTokens: maxTokens, + Temperature: &messages.Temperature, + TopP: &messages.TopP, + N: &messages.N, + Messages: messageReq, + }, nil) + if err != nil { + return nil, fmt.Errorf("error getting chat completion api: %w", err) + } + + // No choices returned. + if len(res.ChatCompletions.Choices) == 0 { + return []azopenai.ChatChoice{}, nil + } + + choices := res.ChatCompletions.Choices + response = make([]azopenai.ChatChoice, len(choices)) + for i, c := range choices { + response[i] = *c + } + + return response, nil +} + +// Close Az OpenAI instance. +func (p *AzOpenAI) Close() error { + p.client = nil + + return nil +} + +// GetComponentMetadata returns the metadata of the component. +func (p *AzOpenAI) GetComponentMetadata() map[string]string { + metadataStruct := openAIMetadata{} + metadataInfo := map[string]string{} + metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) + return metadataInfo +} diff --git a/go.mod b/go.mod index 937f84ebff..98002f8790 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,9 @@ require ( cloud.google.com/go/secretmanager v1.10.0 cloud.google.com/go/storage v1.30.1 dubbo.apache.org/dubbo-go/v3 v3.0.3-0.20230118042253-4f159a2b38f3 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai v0.0.0-20230705184009-934612c4f2b5 github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v0.5.0 github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5 github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 diff --git a/go.sum b/go.sum index 33ff6e88ca..d952c1ce42 100644 --- a/go.sum +++ b/go.sum @@ -420,11 +420,13 @@ github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.2/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1/go.mod h1:gLa1CL2RNE4s7M3yopJ/p0iq5DdY6Yv5ZUt9MTRZOQM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= +github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai v0.0.0-20230705184009-934612c4f2b5 h1:DQCZXtoCPuwBMlAa2aC+B3CfpE6xz2xe1jqdqt8nIJY= +github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai v0.0.0-20230705184009-934612c4f2b5/go.mod h1:GQSjs1n073tbMa3e76+STZkyFb+NcEA4N7OB5vNvB3E= github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v0.5.0 h1:OrKZybbyagpgJiREiIVzH5mV/z9oS4rXqdX7i31DSF0= github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v0.5.0/go.mod h1:p74+tP95m8830ypJk53L93+BEsjTKY4SKQ75J2NmS5U= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5 h1:qS0Bp4do0cIvnuQgSGeO6ZCu/q/HlRKl4NPfv1eJ2p0= diff --git a/tests/certification/go.mod b/tests/certification/go.mod index 6c16649bd7..37efbf1384 100644 --- a/tests/certification/go.mod +++ b/tests/certification/go.mod @@ -52,7 +52,7 @@ require ( github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a // indirect github.com/AthenZ/athenz v1.10.39 // indirect github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 // indirect diff --git a/tests/certification/go.sum b/tests/certification/go.sum index 9f6d9eb0f8..39ae5032fe 100644 --- a/tests/certification/go.sum +++ b/tests/certification/go.sum @@ -70,8 +70,8 @@ github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.2/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1/go.mod h1:gLa1CL2RNE4s7M3yopJ/p0iq5DdY6Yv5ZUt9MTRZOQM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= From aa4d073cd5b372f062eb14e2bcca137dcb9b0e70 Mon Sep 17 00:00:00 2001 From: Roberto Rojas Date: Mon, 17 Jul 2023 17:38:58 -0400 Subject: [PATCH 7/8] [Bindings] Append Direction and Route as Built-in Metadata Properties. (#2945) Signed-off-by: Roberto Rojas Signed-off-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> --- .build-tools/pkg/metadataschema/validators.go | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/.build-tools/pkg/metadataschema/validators.go b/.build-tools/pkg/metadataschema/validators.go index 9527508fea..4cfc8b0efc 100644 --- a/.build-tools/pkg/metadataschema/validators.go +++ b/.build-tools/pkg/metadataschema/validators.go @@ -23,6 +23,13 @@ import ( mdutils "github.com/dapr/components-contrib/metadata" ) +const ( + bindingDirectionMetadataKey = "direction" + bindingDirectionInput = "input" + bindingDirectionOutput = "output" + bindingRouteMetadataKey = "route" +) + // IsValid performs additional validation and returns true if the object is valid. func (c *ComponentMetadata) IsValid() error { // Check valid component type @@ -135,6 +142,51 @@ func (c *ComponentMetadata) AppendBuiltin() error { }, }, ) + case mdutils.BindingType: + if c.Binding != nil { + if c.Metadata == nil { + c.Metadata = []Metadata{} + } + + if c.Binding.Input { + direction := bindingDirectionInput + allowedValues := []string{ + bindingDirectionInput, + } + + if c.Binding.Output { + direction = fmt.Sprintf("%s,%s", bindingDirectionInput, bindingDirectionOutput) + allowedValues = append(allowedValues, bindingDirectionOutput, direction) + } + + c.Metadata = append(c.Metadata, + Metadata{ + Name: bindingDirectionMetadataKey, + Type: "string", + Description: "Indicates the direction of the binding component.", + Example: `"`+direction+`"`, + URL: &URL{ + Title: "Documentation", + URL: "https://docs.dapr.io/reference/api/bindings_api/#binding-direction-optional", + }, + AllowedValues: allowedValues, + }, + ) + + c.Metadata = append(c.Metadata, + Metadata{ + Name: bindingRouteMetadataKey, + Type: "string", + Description: "Specifies a custom route for incoming events.", + Example: `"/custom-path"`, + URL: &URL{ + Title: "Documentation", + URL: "https://docs.dapr.io/developing-applications/building-blocks/bindings/howto-triggers/#specifying-a-custom-route", + }, + }, + ) + } + } } // Sanity check to ensure the data is in sync From ec05809ee63d5e7b1b695c47e91e2d860b52b86d Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Mon, 17 Jul 2023 15:17:22 -0700 Subject: [PATCH 8/8] [Metadata] Update validator and some other fixes (#2984) Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Bernd Verst --- .../builtin-authentication-profiles.yaml | 97 +++++++++++++ .build-tools/component-folders.json | 46 ------ .build-tools/component-folders.yaml | 43 ++++++ .build-tools/go.mod | 1 + .build-tools/main.go | 35 +++-- .../pkg/metadataanalyzer/analyzer.template | 33 +++-- .../builtin-authentication-profiles.go | 133 ++---------------- .build-tools/pkg/metadataschema/schema.go | 79 +++++------ .build-tools/pkg/metadataschema/validators.go | 12 ++ .golangci.yml | 1 + Makefile | 5 +- bindings/alicloud/dingtalk/webhook/webhook.go | 5 +- bindings/alicloud/oss/oss.go | 5 +- bindings/alicloud/sls/sls.go | 5 +- bindings/alicloud/tablestore/tablestore.go | 5 +- bindings/apns/apns.go | 5 +- bindings/aws/dynamodb/dynamodb.go | 5 +- bindings/aws/kinesis/kinesis.go | 5 +- bindings/aws/s3/s3.go | 11 +- bindings/aws/ses/ses.go | 5 +- bindings/aws/sns/sns.go | 5 +- bindings/aws/sqs/sqs.go | 5 +- bindings/azure/blobstorage/blobstorage.go | 5 +- bindings/azure/cosmosdb/cosmosdb.go | 5 +- .../cosmosdbgremlinapi/cosmosdbgremlinapi.go | 5 +- bindings/azure/eventgrid/eventgrid.go | 5 +- bindings/azure/eventgrid/metadata.yaml | 2 +- bindings/azure/eventhubs/eventhubs.go | 5 +- bindings/azure/eventhubs/metadata.yaml | 8 -- bindings/azure/openai/openai.go | 5 +- .../servicebusqueues/servicebusqueues.go | 5 +- bindings/azure/signalr/signalr.go | 5 +- bindings/azure/storagequeues/storagequeues.go | 5 +- bindings/cloudflare/queues/cfqueues.go | 5 +- bindings/commercetools/commercetools.go | 5 +- bindings/cron/cron.go | 5 +- bindings/dubbo/dubbo_output.go | 6 +- bindings/gcp/bucket/bucket.go | 11 +- bindings/gcp/pubsub/pubsub.go | 5 +- bindings/graphql/graphql.go | 5 +- bindings/http/http.go | 5 +- bindings/huawei/obs/obs.go | 5 +- bindings/influx/influx.go | 5 +- bindings/input_binding.go | 4 +- bindings/kafka/kafka.go | 5 +- bindings/kitex/kitex_output.go | 6 +- bindings/kubemq/kubemq.go | 5 +- bindings/kubernetes/kubernetes.go | 5 +- bindings/localstorage/localstorage.go | 5 +- bindings/mqtt3/mqtt.go | 5 +- bindings/mysql/mysql.go | 5 +- bindings/nacos/nacos.go | 5 +- bindings/output_binding.go | 4 +- bindings/postgres/postgres.go | 5 +- bindings/postmark/postmark.go | 5 +- bindings/rabbitmq/rabbitmq.go | 5 +- bindings/redis/redis.go | 5 +- bindings/rethinkdb/statechange/statechange.go | 5 +- bindings/smtp/smtp.go | 5 +- bindings/twilio/sendgrid/sendgrid.go | 5 +- bindings/twilio/sms/sms.go | 5 +- bindings/wasm/output.go | 5 +- bindings/zeebe/command/command.go | 5 +- bindings/zeebe/jobworker/jobworker.go | 5 +- component-metadata-schema.json | 2 +- configuration/azure/appconfig/appconfig.go | 5 +- configuration/postgres/postgres.go | 5 +- configuration/redis/redis.go | 5 +- configuration/store.go | 11 +- crypto/azure/keyvault/component.go | 5 +- crypto/jwks/component.go | 5 +- crypto/kubernetes/secrets/component.go | 9 +- crypto/localstorage/component.go | 5 +- crypto/subtlecrypto.go | 7 +- .../component/azure/blobstorage/metadata.go | 2 +- .../component/azure/eventhubs/metadata.go | 12 +- .../component/azure/servicebus/metadata.go | 2 +- internal/component/postgresql/postgresql.go | 5 +- internal/component/redis/settings.go | 18 +-- lock/redis/standalone.go | 5 +- lock/store.go | 11 +- metadata/componentmetadata.go | 23 +++ metadata/componentmetadata_metadata.go | 23 +++ metadata/utils.go | 62 ++++++-- metadata/utils_test.go | 60 +++++--- middleware/http/bearer/bearer_middleware.go | 5 +- middleware/http/oauth2/oauth2_middleware.go | 5 +- .../oauth2clientcredentials_middleware.go | 5 +- middleware/http/opa/middleware.go | 5 +- .../http/ratelimit/ratelimit_middleware.go | 5 +- .../routeralias/routeralias_middleware.go | 9 +- .../routerchecker/routerchecker_middleware.go | 5 +- middleware/http/sentinel/middleware.go | 5 +- middleware/http/wasm/httpwasm.go | 5 +- middleware/middleware.go | 1 - pubsub/aws/snssqs/snssqs.go | 5 +- pubsub/azure/eventhubs/eventhubs.go | 5 +- pubsub/azure/servicebus/queues/servicebus.go | 7 +- pubsub/azure/servicebus/topics/servicebus.go | 5 +- pubsub/gcp/pubsub/pubsub.go | 5 +- pubsub/in-memory/in-memory.go | 5 +- pubsub/jetstream/jetstream.go | 5 +- pubsub/kafka/kafka.go | 5 +- pubsub/kubemq/kubemq.go | 5 +- pubsub/mqtt3/mqtt.go | 5 +- pubsub/natsstreaming/natsstreaming.go | 5 +- pubsub/pubsub.go | 4 +- pubsub/pulsar/pulsar.go | 5 +- pubsub/rabbitmq/rabbitmq.go | 5 +- pubsub/redis/redis.go | 5 +- pubsub/rocketmq/rocketmq.go | 5 +- pubsub/solace/amqp/amqp.go | 5 +- .../alicloud/parameterstore/parameterstore.go | 5 +- .../aws/parameterstore/parameterstore.go | 5 +- .../aws/secretmanager/secretmanager.go | 5 +- secretstores/azure/keyvault/keyvault.go | 5 +- .../gcp/secretmanager/secretmanager.go | 5 +- secretstores/hashicorp/vault/vault.go | 5 +- secretstores/huaweicloud/csms/csms.go | 5 +- secretstores/kubernetes/kubernetes.go | 10 +- secretstores/local/env/envstore.go | 5 +- secretstores/local/file/filestore.go | 5 +- secretstores/secret_store.go | 5 +- secretstores/tencentcloud/ssm/ssm.go | 5 +- state/aerospike/aerospike.go | 5 +- state/alicloud/tablestore/tablestore.go | 5 +- state/aws/dynamodb/dynamodb.go | 5 +- state/azure/blobstorage/blobstorage.go | 5 +- state/azure/cosmosdb/cosmosdb.go | 5 +- state/azure/tablestorage/tablestorage.go | 5 +- state/bulk_test.go | 6 +- state/cassandra/cassandra.go | 5 +- state/cloudflare/workerskv/workerskv.go | 5 +- state/couchbase/couchbase.go | 5 +- state/etcd/etcd.go | 5 +- state/gcp/firestore/firestore.go | 5 +- state/hashicorp/consul/consul.go | 5 +- state/hazelcast/hazelcast.go | 5 +- state/in-memory/in_memory.go | 5 +- state/jetstream/jetstream.go | 5 +- state/memcached/memcached.go | 5 +- state/mongodb/mongodb.go | 5 +- state/mysql/mysql.go | 5 +- state/oci/objectstorage/objectstorage.go | 5 +- state/oracledatabase/oracledatabase.go | 5 +- state/redis/redis.go | 6 +- state/redis/redis_test.go | 2 - state/rethinkdb/rethinkdb.go | 5 +- state/sqlite/sqlite.go | 5 +- state/sqlserver/sqlserver.go | 8 +- state/store.go | 4 +- state/zookeeper/zk.go | 5 +- workflows/temporal/temporal.go | 5 +- workflows/workflow.go | 1 - 154 files changed, 703 insertions(+), 689 deletions(-) create mode 100644 .build-tools/builtin-authentication-profiles.yaml delete mode 100644 .build-tools/component-folders.json create mode 100644 .build-tools/component-folders.yaml create mode 100644 metadata/componentmetadata.go create mode 100644 metadata/componentmetadata_metadata.go diff --git a/.build-tools/builtin-authentication-profiles.yaml b/.build-tools/builtin-authentication-profiles.yaml new file mode 100644 index 0000000000..81e07c3635 --- /dev/null +++ b/.build-tools/builtin-authentication-profiles.yaml @@ -0,0 +1,97 @@ +aws: + - title: "AWS: Access Key ID and Secret Access Key" + description: | + Authenticate using an Access Key ID and Secret Access Key included in the metadata + metadata: + - name: accessKey + description: AWS access key associated with an IAM account + required: true + sensitive: true + example: '"AKIAIOSFODNN7EXAMPLE"' + - name: secretKey + description: The secret key associated with the access key + required: true + sensitive: true + example: '"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"' + - title: "AWS: Credentials from Environment Variables" + description: Use AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY from the environment + +azuread: + - title: "Azure AD: Managed identity" + description: Authenticate using Azure AD and a managed identity. + metadata: + - name: azureClientId + description: | + Client ID (application ID). Required if the service has multiple identities assigned. + example: '"c7dd251f-811f-4ba2-a905-acd4d3f8f08b"' + - name: azureEnvironment + description: | + Optional name for the Azure environment if using a different Azure cloud + default: AzurePublicCloud + example: '"AzurePublicCloud"' + allowedValues: + - AzurePublicCloud + - AzureChinaCloud + - AzureUSGovernmentCloud + - title: "Azure AD: Client credentials" + description: | + Authenticate using Azure AD with client credentials, also known as "service principals". + metadata: + - name: azureTenantId + description: ID of the Azure AD tenant + required: true + example: '"cd4b2887-304c-47e1-b4d5-65447fdd542a"' + - name: azureClientId + description: Client ID (application ID) + required: true + example: '"c7dd251f-811f-4ba2-a905-acd4d3f8f08b"' + - name: azureClientSecret + description: Client secret (application password) + required: true + sensitive: true + example: '"Ecy3XG7zVZK3/vl/a2NSB+a1zXLa8RnMum/IgD0E"' + - name: azureEnvironment + description: | + Optional name for the Azure environment if using a different Azure cloud + default: AzurePublicCloud + example: '"AzurePublicCloud"' + allowedValues: + - AzurePublicCloud + - AzureChinaCloud + - AzureUSGovernmentCloud + - title: "Azure AD: Client certificate" + description: | + Authenticate using Azure AD with a client certificate. One of "azureCertificate" and "azureCertificateFile" is required. + metadata: + - name: azureTenantId + description: ID of the Azure AD tenant + required: true + example: '"cd4b2887-304c-47e1-b4d5-65447fdd542a"' + - name: azureClientId + description: Client ID (application ID) + required: true + example: '"c7dd251f-811f-4ba2-a905-acd4d3f8f08b"' + - name: azureCertificate + description: | + Certificate and private key (in either a PEM file containing both the certificate and key, or in PFX/PKCS#12 format) + sensitive: true + example: | + "-----BEGIN PRIVATE KEY-----\n MIIEvgI... \n -----END PRIVATE KEY----- + \n -----BEGIN CERTIFICATE----- \n MIICoTC... \n -----END CERTIFICATE----- \n" + - name: azureCertificateFile + description: | + Path to PEM or PFX/PKCS#12 file on disk, containing the certificate and private key. + example: '"/path/to/file.pem"' + - name: azureCertificatePassword + description: Password for the certificate if encrypted. + sensitive: true + example: '"password"' + - name: azureEnvironment + description: | + Optional name for the Azure environment if using a different Azure cloud + default: AzurePublicCloud + example: '"AzurePublicCloud"' + allowedValues: + - AzurePublicCloud + - AzureChinaCloud + - AzureUSGovernmentCloud diff --git a/.build-tools/component-folders.json b/.build-tools/component-folders.json deleted file mode 100644 index ecea136ffb..0000000000 --- a/.build-tools/component-folders.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "componentFolders": [ - "bindings", - "configuration", - "crypto", - "lock", - "middleware/http", - "nameresolution", - "pubsub", - "secretstores", - "state", - "workflows" - ], - "excludeFolders": [ - "bindings/alicloud", - "bindings/aws", - "bindings/azure", - "bindings/gcp", - "bindings/huawei", - "bindings/rethinkdb", - "bindings/twilio", - "bindings/zeebe", - "configuration/azure", - "configuration/redis/internal", - "crypto/azure", - "crypto/kubernetes", - "pubsub/aws", - "pubsub/azure", - "pubsub/azure/servicebus", - "pubsub/gcp", - "secretstores/alicloud", - "secretstores/aws", - "secretstores/azure", - "secretstores/gcp", - "secretstores/hashicorp", - "secretstores/huaweicloud", - "secretstores/local", - "state/alicloud", - "state/aws", - "state/azure", - "state/gcp", - "state/hashicorp", - "state/oci", - "state/utils" - ] -} diff --git a/.build-tools/component-folders.yaml b/.build-tools/component-folders.yaml new file mode 100644 index 0000000000..13b0882176 --- /dev/null +++ b/.build-tools/component-folders.yaml @@ -0,0 +1,43 @@ +componentFolders: + - bindings + - configuration + - crypto + - lock + - middleware/http + - nameresolution + - pubsub + - secretstores + - state + - workflows + +excludeFolders: + - bindings/alicloud + - bindings/aws + - bindings/azure + - bindings/gcp + - bindings/huawei + - bindings/rethinkdb + - bindings/twilio + - bindings/zeebe + - configuration/azure + - configuration/redis/internal + - crypto/azure + - crypto/kubernetes + - pubsub/aws + - pubsub/azure + - pubsub/azure/servicebus + - pubsub/gcp + - secretstores/alicloud + - secretstores/aws + - secretstores/azure + - secretstores/gcp + - secretstores/hashicorp + - secretstores/huaweicloud + - secretstores/local + - state/alicloud + - state/aws + - state/azure + - state/gcp + - state/hashicorp + - state/oci + - state/utils diff --git a/.build-tools/go.mod b/.build-tools/go.mod index a75f66cde6..6b9370e53b 100644 --- a/.build-tools/go.mod +++ b/.build-tools/go.mod @@ -8,6 +8,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + gopkg.in/yaml.v3 v3.0.1 sigs.k8s.io/yaml v1.3.0 ) diff --git a/.build-tools/main.go b/.build-tools/main.go index fcfbf4adb3..ee602c3b3a 100644 --- a/.build-tools/main.go +++ b/.build-tools/main.go @@ -15,28 +15,39 @@ package main import ( _ "embed" - "encoding/json" + + "gopkg.in/yaml.v3" "github.com/dapr/components-contrib/build-tools/cmd" + "github.com/dapr/components-contrib/build-tools/pkg/metadataschema" ) -//go:embed component-folders.json -var componentFoldersJSON []byte +var ( + //go:embed component-folders.yaml + componentFoldersYAML []byte + //go:embed builtin-authentication-profiles.yaml + builtinAuthenticationProfilesYAML []byte +) -func init() { - parsed := struct { - ComponentFolders []string `json:"componentFolders"` - ExcludeFolders []string `json:"excludeFolders"` +func main() { + // Parse component-folders.json + parsedComponentFolders := struct { + ComponentFolders []string `json:"componentFolders" yaml:"componentFolders"` + ExcludeFolders []string `json:"excludeFolders" yaml:"excludeFolders"` }{} - err := json.Unmarshal(componentFoldersJSON, &parsed) + err := yaml.Unmarshal(componentFoldersYAML, &parsedComponentFolders) if err != nil { panic(err) } - cmd.ComponentFolders = parsed.ComponentFolders - cmd.ExcludeFolders = parsed.ExcludeFolders -} + cmd.ComponentFolders = parsedComponentFolders.ComponentFolders + cmd.ExcludeFolders = parsedComponentFolders.ExcludeFolders + + // Parse builtin-authentication-profiles.yaml + err = yaml.Unmarshal(builtinAuthenticationProfilesYAML, &metadataschema.BuiltinAuthenticationProfiles) + if err != nil { + panic(err) + } -func main() { cmd.Execute() } diff --git a/.build-tools/pkg/metadataanalyzer/analyzer.template b/.build-tools/pkg/metadataanalyzer/analyzer.template index b5fc62ecf3..6177c0050c 100644 --- a/.build-tools/pkg/metadataanalyzer/analyzer.template +++ b/.build-tools/pkg/metadataanalyzer/analyzer.template @@ -6,7 +6,7 @@ import ( "os" "strings" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/dapr/kit/logger" mdutils "github.com/dapr/components-contrib/metadata" @@ -17,24 +17,24 @@ import ( ) func main() { - if len(os.Args) < 2 { - fmt.Println("Please provide the path to the components-contrib root as an argument") - os.Exit(1) - } - basePath := os.Args[1] - log := logger.NewLogger("metadata") + if len(os.Args) < 2 { + fmt.Println("Please provide the path to the components-contrib root as an argument") + os.Exit(1) + } + basePath := os.Args[1] + log := logger.NewLogger("metadata") - var ( + var ( yamlMetadata *map[string]string - missing map[string]string + missing []string unexpected []string ) - missingByComponent := make(map[string]map[string]string) + missingByComponent := make(map[string][]string) unexpectedByComponent := make(map[string][]string) {{range $fullpkg, $val := .Pkgs}} instanceOf_{{index $val 0}} := {{index $val 0}}.{{index $val 1}}(log) - metadataFor_{{index $val 0}} := instanceOf_{{index $val 0}}.GetComponentMetadata() + metadataFor_{{index $val 0}} := instanceOf_{{index $val 0}}.(mdutils.ComponentWithMetadata).GetComponentMetadata() yamlMetadata = getYamlMetadata(basePath, "{{$fullpkg}}") missing = checkMissingMetadata(yamlMetadata, metadataFor_{{index $val 0}}) if len(missing) > 0 { @@ -127,14 +127,17 @@ func getYamlMetadata(basePath string, pkg string) *map[string]string { return &names } -func checkMissingMetadata(yamlMetadata *map[string]string, componentMetadata map[string]string) map[string]string { - missingMetadata := make(map[string]string) +func checkMissingMetadata(yamlMetadata *map[string]string, componentMetadata mdutils.MetadataMap) []string { + missingMetadata := make([]string, 0) // if there is no yaml metadata, then we are not missing anything yet if yamlMetadata != nil && len(*yamlMetadata) > 0 { - for key := range componentMetadata { + for key, md := range componentMetadata { + if md.Ignored { + continue + } lowerKey := strings.ToLower(key) if _, ok := (*yamlMetadata)[lowerKey]; !ok { - missingMetadata[lowerKey] = componentMetadata[key] + missingMetadata = append(missingMetadata, key) } // todo - check if the metadata is the same data type } diff --git a/.build-tools/pkg/metadataschema/builtin-authentication-profiles.go b/.build-tools/pkg/metadataschema/builtin-authentication-profiles.go index d2fb6983b8..bcb76d5ed0 100644 --- a/.build-tools/pkg/metadataschema/builtin-authentication-profiles.go +++ b/.build-tools/pkg/metadataschema/builtin-authentication-profiles.go @@ -17,131 +17,22 @@ import ( "fmt" ) +// Built-in authentication profiles +var BuiltinAuthenticationProfiles map[string][]AuthenticationProfile + // ParseBuiltinAuthenticationProfile returns an AuthenticationProfile(s) from a given BuiltinAuthenticationProfile. func ParseBuiltinAuthenticationProfile(bi BuiltinAuthenticationProfile) ([]AuthenticationProfile, error) { - switch bi.Name { - case "aws": - return []AuthenticationProfile{ - { - Title: "AWS: Access Key ID and Secret Access Key", - Description: "Authenticate using an Access Key ID and Secret Access Key included in the metadata", - Metadata: []Metadata{ - { - Name: "accessKey", - Required: true, - Sensitive: true, - Description: "AWS access key associated with an IAM account", - Example: `"AKIAIOSFODNN7EXAMPLE"`, - }, - { - Name: "secretKey", - Required: true, - Sensitive: true, - Description: "The secret key associated with the access key", - Example: `"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"`, - }, - }, - }, - { - Title: "AWS: Credentials from Environment Variables", - Description: "Use AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY from the environment", - Metadata: []Metadata{}, - }, - }, nil - case "azuread": - azureEnvironmentMetadata := Metadata{ - Name: "azureEnvironment", - Required: false, - Description: "Optional name for the Azure environment if using a different Azure cloud", - Example: `"AzurePublicCloud"`, - Default: "AzurePublicCloud", - AllowedValues: []string{"AzurePublicCloud", "AzureChinaCloud", "AzureUSGovernmentCloud"}, - } - profiles := []AuthenticationProfile{ - { - Title: "Azure AD: Managed identity", - Description: "Authenticate using Azure AD and a managed identity.", - Metadata: mergedMetadata(bi.Metadata, - Metadata{ - Name: "azureClientId", - Description: "Client ID (application ID). Required if the service has multiple identities assigned.", - Example: `"c7dd251f-811f-4ba2-a905-acd4d3f8f08b"`, - Required: false, - }, - azureEnvironmentMetadata, - ), - }, - { - Title: "Azure AD: Client credentials", - Description: "Authenticate using Azure AD with client credentials, also known as \"service principals\".", - Metadata: mergedMetadata(bi.Metadata, - Metadata{ - Name: "azureTenantId", - Description: "ID of the Azure AD tenant", - Example: `"cd4b2887-304c-47e1-b4d5-65447fdd542a"`, - Required: true, - }, - Metadata{ - Name: "azureClientId", - Description: "Client ID (application ID)", - Example: `"c7dd251f-811f-4ba2-a905-acd4d3f8f08b"`, - Required: true, - }, - Metadata{ - Name: "azureClientSecret", - Description: "Client secret (application password)", - Example: `"Ecy3XG7zVZK3/vl/a2NSB+a1zXLa8RnMum/IgD0E"`, - Required: true, - Sensitive: true, - }, - azureEnvironmentMetadata, - ), - }, - { - Title: "Azure AD: Client certificate", - Description: `Authenticate using Azure AD with a client certificate. One of "azureCertificate" and "azureCertificateFile" is required.`, - Metadata: mergedMetadata(bi.Metadata, - Metadata{ - Name: "azureTenantId", - Description: "ID of the Azure AD tenant", - Example: `"cd4b2887-304c-47e1-b4d5-65447fdd542a"`, - Required: true, - }, - Metadata{ - Name: "azureClientId", - Description: "Client ID (application ID)", - Example: `"c7dd251f-811f-4ba2-a905-acd4d3f8f08b"`, - Required: true, - }, - Metadata{ - Name: "azureCertificate", - Description: "Certificate and private key (in either a PEM file containing both the certificate and key, or in PFX/PKCS#12 format)", - Example: `"-----BEGIN PRIVATE KEY-----\n MIIEvgI... \n -----END PRIVATE KEY----- \n -----BEGIN CERTIFICATE----- \n MIICoTC... \n -----END CERTIFICATE----- \n"`, - Required: false, - Sensitive: true, - }, - Metadata{ - Name: "azureCertificateFile", - Description: "Path to PEM or PFX/PKCS#12 file on disk, containing the certificate and private key.", - Example: `"/path/to/file.pem"`, - Required: false, - Sensitive: false, - }, - Metadata{ - Name: "azureCertificatePassword", - Description: "Password for the certificate if encrypted.", - Example: `"password"`, - Required: false, - Sensitive: true, - }, - azureEnvironmentMetadata, - ), - }, - } - return profiles, nil - default: + profiles, ok := BuiltinAuthenticationProfiles[bi.Name] + if !ok { return nil, fmt.Errorf("built-in authentication profile %s does not exist", bi.Name) } + + res := make([]AuthenticationProfile, len(profiles)) + for i, profile := range profiles { + res[i] = profile + res[i].Metadata = mergedMetadata(bi.Metadata, res[i].Metadata...) + } + return res, nil } func mergedMetadata(base []Metadata, add ...Metadata) []Metadata { diff --git a/.build-tools/pkg/metadataschema/schema.go b/.build-tools/pkg/metadataschema/schema.go index c807f0f65c..2e49338aec 100644 --- a/.build-tools/pkg/metadataschema/schema.go +++ b/.build-tools/pkg/metadataschema/schema.go @@ -18,116 +18,113 @@ package metadataschema // ComponentMetadata is the schema for the metadata.yaml / metadata.json files. type ComponentMetadata struct { // Version of the component metadata schema. - SchemaVersion string `json:"schemaVersion" jsonschema:"enum=v1"` + SchemaVersion string `json:"schemaVersion" yaml:"schemaVersion" jsonschema:"enum=v1"` // Component type, of one of the allowed values. - Type string `json:"type" jsonschema:"enum=bindings,enum=state,enum=secretstores,enum=pubsub,enum=workflows,enum=configuration,enum=lock,enum=middleware"` + Type string `json:"type" yaml:"type" jsonschema:"enum=bindings,enum=state,enum=secretstores,enum=pubsub,enum=workflows,enum=configuration,enum=lock,enum=middleware"` // Name of the component (without the inital type, e.g. "http" instead of "bindings.http"). - Name string `json:"name"` + Name string `json:"name" yaml:"name"` // Version of the component, with the leading "v", e.g. "v1". - Version string `json:"version"` + Version string `json:"version" yaml:"version"` // Component status. - Status string `json:"status" jsonschema:"enum=stable,enum=beta,enum=alpha,enum=development-only"` + Status string `json:"status" yaml:"status" jsonschema:"enum=stable,enum=beta,enum=alpha,enum=development-only"` // Title of the component, e.g. "HTTP". - Title string `json:"title"` + Title string `json:"title" yaml:"title"` // Additional description for the component, optional. - Description string `json:"description,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` // URLs with additional resources for the component, such as docs. - URLs []URL `json:"urls"` + URLs []URL `json:"urls" yaml:"urls"` // Properties for bindings only. // This should not present unless "type" is "bindings". - Binding *Binding `json:"binding,omitempty"` + Binding *Binding `json:"binding,omitempty" yaml:"binding,omitempty"` // Component capabilities. // For state stores, the presence of "actorStateStore" implies that the metadata property "actorStateStore" can be set. In that case, do not manually specify "actorStateStore" as metadata option. - Capabilities []string `json:"capabilities,omitempty"` + Capabilities []string `json:"capabilities,omitempty" yaml:"capabilities,omitempty"` // Authentication profiles for the component. - AuthenticationProfiles []AuthenticationProfile `json:"authenticationProfiles,omitempty"` + AuthenticationProfiles []AuthenticationProfile `json:"authenticationProfiles,omitempty" yaml:"authenticationProfiles,omitempty"` // Built-in authentication profiles to import. - BuiltInAuthenticationProfiles []BuiltinAuthenticationProfile `json:"builtinAuthenticationProfiles,omitempty"` + BuiltInAuthenticationProfiles []BuiltinAuthenticationProfile `json:"builtinAuthenticationProfiles,omitempty" yaml:"builtinAuthenticationProfiles,omitempty"` // Metadata options for the component. - Metadata []Metadata `json:"metadata,omitempty"` + Metadata []Metadata `json:"metadata,omitempty" yaml:"metadata,omitempty"` } // URL represents one URL with additional resources. type URL struct { // Title of the URL. - Title string `json:"title"` + Title string `json:"title" yaml:"title"` // URL. - URL string `json:"url"` + URL string `json:"url" yaml:"url"` } // Binding represents properties that are specific to bindings type Binding struct { // If "true", the binding can be used as input binding. - Input bool `json:"input,omitempty"` + Input bool `json:"input,omitempty" yaml:"input,omitempty"` // If "true", the binding can be used as output binding. - Output bool `json:"output,omitempty"` + Output bool `json:"output,omitempty" yaml:"output,omitempty"` // List of operations that the output binding support. // Required in output bindings, and not allowed in input-only bindings. - Operations []BindingOperation `json:"operations"` + Operations []BindingOperation `json:"operations" yaml:"operations"` } // BindingOperation represents an operation offered by an output binding. type BindingOperation struct { // Name of the operation, such as "create", "post", "delete", etc. - Name string `json:"name"` + Name string `json:"name" yaml:"name"` // Descrption of the operation. - Description string `json:"description"` + Description string `json:"description" yaml:"description"` } // Metadata property. type Metadata struct { // Name of the metadata property. - Name string `json:"name"` + Name string `json:"name" yaml:"name"` // Description of the property. - Description string `json:"description"` + Description string `json:"description" yaml:"description"` // If "true", the property is required - Required bool `json:"required,omitempty"` + Required bool `json:"required,omitempty" yaml:"required,omitempty"` // If "true", the property represents a sensitive value such as a password. - Sensitive bool `json:"sensitive,omitempty"` + Sensitive bool `json:"sensitive,omitempty" yaml:"sensitive,omitempty"` // Type of the property. // If this is empty, it's interpreted as "string". - Type string `json:"type,omitempty" jsonschema:"enum=string,enum=number,enum=bool,enum=duration"` + Type string `json:"type,omitempty" yaml:"type,omitempty" jsonschema:"enum=string,enum=number,enum=bool,enum=duration"` // Default value for the property. // If it's a string, don't forget to add quotes. - Default string `json:"default,omitempty"` + Default string `json:"default,omitempty" yaml:"default,omitempty"` // Example value. - Example string `json:"example"` + Example string `json:"example" yaml:"example"` // If set, forces the value to be one of those specified in this allowlist. - AllowedValues []string `json:"allowedValues,omitempty"` + AllowedValues []string `json:"allowedValues,omitempty" yaml:"allowedValues,omitempty"` // If set, specifies that the property is only applicable to bindings of the type specified below. // At least one of "input" and "output" must be "true". - Binding *MetadataBinding `json:"binding,omitempty"` + Binding *MetadataBinding `json:"binding,omitempty" yaml:"binding,omitempty"` // URL with additional information, such as docs. - URL *URL `json:"url,omitempty"` + URL *URL `json:"url,omitempty" yaml:"url,omitempty"` // If set, specifies that the property is deprecated and should not be used in new configurations. - Deprecated bool `json:"deprecated,omitempty"` + Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"` } // MetadataBinding is the type for the "binding" property in the "metadata" object. type MetadataBinding struct { // If "true", the property can be used with the binding as input binding only. - Input bool `json:"input,omitempty"` + Input bool `json:"input,omitempty" yaml:"input,omitempty"` // If "true", the property can be used with the binding as output binding only. - Output bool `json:"output,omitempty"` + Output bool `json:"output,omitempty" yaml:"output,omitempty"` } // AuthenticationProfile is the type for an authentication profile. type AuthenticationProfile struct { // Title of the authentication profile. - Title string `json:"title"` + Title string `json:"title" yaml:"title"` // Additional description for the authentication profile, optional. - Description string `json:"description"` + Description string `json:"description" yaml:"description"` // Metadata options applicable when using this authentication profile. - Metadata []Metadata `json:"metadata,omitempty"` + Metadata []Metadata `json:"metadata,omitempty" yaml:"metadata,omitempty"` } // BuiltinAuthenticationProfile is a reference to a built-in authentication profile. type BuiltinAuthenticationProfile struct { // Name of the built-in authentication profile. - // Currently supports: - // - // - `azuread` (Azure AD, including Managed Identity). - Name string `json:"name"` + Name string `json:"name" yaml:"name"` // Additional metadata options applicable when using this authentication profile. - Metadata []Metadata `json:"metadata,omitempty"` + Metadata []Metadata `json:"metadata,omitempty" yaml:"metadata,omitempty"` } diff --git a/.build-tools/pkg/metadataschema/validators.go b/.build-tools/pkg/metadataschema/validators.go index 4cfc8b0efc..88b026e690 100644 --- a/.build-tools/pkg/metadataschema/validators.go +++ b/.build-tools/pkg/metadataschema/validators.go @@ -77,6 +77,18 @@ func (c *ComponentMetadata) IsValid() error { // Remove the property builtinAuthenticationProfiles now c.BuiltInAuthenticationProfiles = nil + // Trim newlines from all descriptions + c.Description = strings.TrimSpace(c.Description) + for i := range c.AuthenticationProfiles { + c.AuthenticationProfiles[i].Description = strings.TrimSpace(c.AuthenticationProfiles[i].Description) + for j := range c.AuthenticationProfiles[i].Metadata { + c.AuthenticationProfiles[i].Metadata[j].Description = strings.TrimSpace(c.AuthenticationProfiles[i].Metadata[j].Description) + } + } + for i := range c.Metadata { + c.Metadata[i].Description = strings.TrimSpace(c.Metadata[i].Description) + } + return nil } diff --git a/.golangci.yml b/.golangci.yml index 6b635ebc3c..73e3b28b38 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -15,6 +15,7 @@ run: # list of build tags, all linters use it. Default is empty list. build-tags: - certtests + - metadata # which dirs to skip: they won't be analyzed; # can use regexp here: generated.*, regexp is applied on full path; diff --git a/Makefile b/Makefile index 4b4743b177..4028525320 100644 --- a/Makefile +++ b/Makefile @@ -109,14 +109,13 @@ verify-linter-version: ################################################################################ .PHONY: test test: - CGO_ENABLED=$(CGO) go test ./... $(COVERAGE_OPTS) $(BUILDMODE) --timeout=15m + CGO_ENABLED=$(CGO) go test ./... $(COVERAGE_OPTS) $(BUILDMODE) -tags metadata --timeout=15m ################################################################################ # Target: lint # ################################################################################ .PHONY: lint lint: verify-linter-installed verify-linter-version - # Due to https://github.com/golangci/golangci-lint/issues/580, we need to add --fix for windows $(GOLANGCI_LINT) run --timeout=20m ################################################################################ @@ -228,7 +227,7 @@ check-component-metadata: go get "github.com/dapr/components-contrib@master" && \ go mod edit -replace "github.com/dapr/components-contrib"="../" && \ go mod tidy && \ - go build . && \ + go build -tags metadata . && \ rm ./go.mod && rm ./go.sum && rm ./main.go && \ ./metadataanalyzer ../ diff --git a/bindings/alicloud/dingtalk/webhook/webhook.go b/bindings/alicloud/dingtalk/webhook/webhook.go index 30d5f10dec..14964f3443 100644 --- a/bindings/alicloud/dingtalk/webhook/webhook.go +++ b/bindings/alicloud/dingtalk/webhook/webhook.go @@ -207,11 +207,10 @@ func (t *DingTalkWebhook) sendMessage(ctx context.Context, req *bindings.InvokeR } // GetComponentMetadata returns the metadata of the component. -func (t *DingTalkWebhook) GetComponentMetadata() map[string]string { +func (t *DingTalkWebhook) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := Settings{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } func getPostURL(urlPath, secret string) (string, error) { diff --git a/bindings/alicloud/oss/oss.go b/bindings/alicloud/oss/oss.go index 4d43c1fb80..c124098f0d 100644 --- a/bindings/alicloud/oss/oss.go +++ b/bindings/alicloud/oss/oss.go @@ -108,9 +108,8 @@ func (s *AliCloudOSS) getClient(metadata *ossMetadata) (*oss.Client, error) { } // GetComponentMetadata returns the metadata of the component. -func (s *AliCloudOSS) GetComponentMetadata() map[string]string { +func (s *AliCloudOSS) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := ossMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/alicloud/sls/sls.go b/bindings/alicloud/sls/sls.go index 5b16fa494c..04c509b9b3 100644 --- a/bindings/alicloud/sls/sls.go +++ b/bindings/alicloud/sls/sls.go @@ -128,9 +128,8 @@ func (callback *Callback) Fail(result *producer.Result) { } // GetComponentMetadata returns the metadata of the component. -func (s *AliCloudSlsLogstorage) GetComponentMetadata() map[string]string { +func (s *AliCloudSlsLogstorage) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := SlsLogstorageMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/alicloud/tablestore/tablestore.go b/bindings/alicloud/tablestore/tablestore.go index 4d0cf6be6d..abb34a43be 100644 --- a/bindings/alicloud/tablestore/tablestore.go +++ b/bindings/alicloud/tablestore/tablestore.go @@ -347,9 +347,8 @@ func contains(arr []string, str string) bool { } // GetComponentMetadata returns the metadata of the component. -func (s *AliCloudTableStore) GetComponentMetadata() map[string]string { +func (s *AliCloudTableStore) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := tablestoreMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/apns/apns.go b/bindings/apns/apns.go index 273244cb98..df15daf86e 100644 --- a/bindings/apns/apns.go +++ b/bindings/apns/apns.go @@ -261,9 +261,8 @@ func makeErrorResponse(httpResponse *http.Response) (*bindings.InvokeResponse, e } // GetComponentMetadata returns the metadata of the component. -func (a *APNS) GetComponentMetadata() map[string]string { +func (a *APNS) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := APNSmetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/aws/dynamodb/dynamodb.go b/bindings/aws/dynamodb/dynamodb.go index db297d8164..d9ef0e53a4 100644 --- a/bindings/aws/dynamodb/dynamodb.go +++ b/bindings/aws/dynamodb/dynamodb.go @@ -115,9 +115,8 @@ func (d *DynamoDB) getClient(metadata *dynamoDBMetadata) (*dynamodb.DynamoDB, er } // GetComponentMetadata returns the metadata of the component. -func (d *DynamoDB) GetComponentMetadata() map[string]string { +func (d *DynamoDB) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := dynamoDBMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/aws/kinesis/kinesis.go b/bindings/aws/kinesis/kinesis.go index 5c646e4090..0f6547f232 100644 --- a/bindings/aws/kinesis/kinesis.go +++ b/bindings/aws/kinesis/kinesis.go @@ -418,9 +418,8 @@ func (p *recordProcessor) Shutdown(input *interfaces.ShutdownInput) { } // GetComponentMetadata returns the metadata of the component. -func (a *AWSKinesis) GetComponentMetadata() map[string]string { +func (a *AWSKinesis) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := &kinesisMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/aws/s3/s3.go b/bindings/aws/s3/s3.go index a7ba93d6f7..ff84145e74 100644 --- a/bindings/aws/s3/s3.go +++ b/bindings/aws/s3/s3.go @@ -62,10 +62,12 @@ type AWSS3 struct { } type s3Metadata struct { + // Ignored by metadata parser because included in built-in authentication profile + AccessKey string `json:"accessKey" mapstructure:"accessKey" mdignore:"true"` + SecretKey string `json:"secretKey" mapstructure:"secretKey" mdignore:"true"` + Region string `json:"region" mapstructure:"region"` Endpoint string `json:"endpoint" mapstructure:"endpoint"` - AccessKey string `json:"accessKey" mapstructure:"accessKey"` - SecretKey string `json:"secretKey" mapstructure:"secretKey"` SessionToken string `json:"sessionToken" mapstructure:"sessionToken"` Bucket string `json:"bucket" mapstructure:"bucket"` DecodeBase64 bool `json:"decodeBase64,string" mapstructure:"decodeBase64"` @@ -416,9 +418,8 @@ func (metadata s3Metadata) mergeWithRequestMetadata(req *bindings.InvokeRequest) } // GetComponentMetadata returns the metadata of the component. -func (s *AWSS3) GetComponentMetadata() map[string]string { +func (s *AWSS3) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := s3Metadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/aws/ses/ses.go b/bindings/aws/ses/ses.go index e609c1467c..03b48c6fd1 100644 --- a/bindings/aws/ses/ses.go +++ b/bindings/aws/ses/ses.go @@ -171,9 +171,8 @@ func (a *AWSSES) getClient(metadata *sesMetadata) (*ses.SES, error) { } // GetComponentMetadata returns the metadata of the component. -func (a *AWSSES) GetComponentMetadata() map[string]string { +func (a *AWSSES) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := sesMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/aws/sns/sns.go b/bindings/aws/sns/sns.go index e2d587fda9..7731401171 100644 --- a/bindings/aws/sns/sns.go +++ b/bindings/aws/sns/sns.go @@ -117,9 +117,8 @@ func (a *AWSSNS) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*bind } // GetComponentMetadata returns the metadata of the component. -func (a *AWSSNS) GetComponentMetadata() map[string]string { +func (a *AWSSNS) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := snsMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/aws/sqs/sqs.go b/bindings/aws/sqs/sqs.go index 58c999b55d..231ac80151 100644 --- a/bindings/aws/sqs/sqs.go +++ b/bindings/aws/sqs/sqs.go @@ -187,9 +187,8 @@ func (a *AWSSQS) getClient(metadata *sqsMetadata) (*sqs.SQS, error) { } // GetComponentMetadata returns the metadata of the component. -func (a *AWSSQS) GetComponentMetadata() map[string]string { +func (a *AWSSQS) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := sqsMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/azure/blobstorage/blobstorage.go b/bindings/azure/blobstorage/blobstorage.go index 8779ec01e8..c28034edab 100644 --- a/bindings/azure/blobstorage/blobstorage.go +++ b/bindings/azure/blobstorage/blobstorage.go @@ -358,9 +358,8 @@ func (a *AzureBlobStorage) isValidDeleteSnapshotsOptionType(accessType azblob.De } // GetComponentMetadata returns the metadata of the component. -func (a *AzureBlobStorage) GetComponentMetadata() map[string]string { +func (a *AzureBlobStorage) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := storageinternal.BlobStorageMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/azure/cosmosdb/cosmosdb.go b/bindings/azure/cosmosdb/cosmosdb.go index 785514f74f..381f8e667a 100644 --- a/bindings/azure/cosmosdb/cosmosdb.go +++ b/bindings/azure/cosmosdb/cosmosdb.go @@ -192,9 +192,8 @@ func (c *CosmosDB) lookup(m map[string]interface{}, ks []string) (val interface{ } // GetComponentMetadata returns the metadata of the component. -func (c *CosmosDB) GetComponentMetadata() map[string]string { +func (c *CosmosDB) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := cosmosDBCredentials{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/azure/cosmosdbgremlinapi/cosmosdbgremlinapi.go b/bindings/azure/cosmosdbgremlinapi/cosmosdbgremlinapi.go index f235d1ee91..74109da384 100644 --- a/bindings/azure/cosmosdbgremlinapi/cosmosdbgremlinapi.go +++ b/bindings/azure/cosmosdbgremlinapi/cosmosdbgremlinapi.go @@ -130,9 +130,8 @@ func (c *CosmosDBGremlinAPI) Invoke(_ context.Context, req *bindings.InvokeReque } // GetComponentMetadata returns the metadata of the component. -func (c *CosmosDBGremlinAPI) GetComponentMetadata() map[string]string { +func (c *CosmosDBGremlinAPI) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := cosmosDBGremlinAPICredentials{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/azure/eventgrid/eventgrid.go b/bindings/azure/eventgrid/eventgrid.go index 4c7edb4da4..377f9f4bc2 100644 --- a/bindings/azure/eventgrid/eventgrid.go +++ b/bindings/azure/eventgrid/eventgrid.go @@ -535,9 +535,8 @@ func (a *AzureEventGrid) subscriptionNeedsUpdating(res armeventgrid.EventSubscri } // GetComponentMetadata returns the metadata of the component. -func (a *AzureEventGrid) GetComponentMetadata() map[string]string { +func (a *AzureEventGrid) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := azureEventGridMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/azure/eventgrid/metadata.yaml b/bindings/azure/eventgrid/metadata.yaml index 7c99fc44fa..dd8bbc4dda 100644 --- a/bindings/azure/eventgrid/metadata.yaml +++ b/bindings/azure/eventgrid/metadata.yaml @@ -27,7 +27,7 @@ metadata: output: false description: | The HTTPS endpoint of the webhook Event Grid sends events (formatted as - Cloud Events) to. If you’re not re-writing URLs on ingress, it should be + Cloud Events) to. If you're not re-writing URLs on ingress, it should be in the form of: `"https://[YOUR HOSTNAME]/"` If testing on your local machine, you can use something like `ngrok` to create a public endpoint. diff --git a/bindings/azure/eventhubs/eventhubs.go b/bindings/azure/eventhubs/eventhubs.go index cd48ab6bb2..ec68b4e378 100644 --- a/bindings/azure/eventhubs/eventhubs.go +++ b/bindings/azure/eventhubs/eventhubs.go @@ -96,9 +96,8 @@ func (a *AzureEventHubs) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (a *AzureEventHubs) GetComponentMetadata() map[string]string { +func (a *AzureEventHubs) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := impl.AzureEventHubsMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/azure/eventhubs/metadata.yaml b/bindings/azure/eventhubs/metadata.yaml index 77c221cfa2..f6250ce69d 100644 --- a/bindings/azure/eventhubs/metadata.yaml +++ b/bindings/azure/eventhubs/metadata.yaml @@ -89,14 +89,6 @@ builtinAuthenticationProfiles: Number of partitions for the new Event Hub namespace. Used only when entity management is enabled. metadata: - # Input and output metadata - - name: partitionId - type: string - required: false - description: | - DEPRECATED. - deprecated: true - example: "" # Input-only metadata # consumerGroup is an alias for consumerId, if both are defined consumerId takes precedence. - name: consumerId diff --git a/bindings/azure/openai/openai.go b/bindings/azure/openai/openai.go index 83d189203a..9df8009a33 100644 --- a/bindings/azure/openai/openai.go +++ b/bindings/azure/openai/openai.go @@ -318,9 +318,8 @@ func (p *AzOpenAI) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (p *AzOpenAI) GetComponentMetadata() map[string]string { +func (p *AzOpenAI) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := openAIMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/azure/servicebusqueues/servicebusqueues.go b/bindings/azure/servicebusqueues/servicebusqueues.go index 27d74f4088..3ab133fbe4 100644 --- a/bindings/azure/servicebusqueues/servicebusqueues.go +++ b/bindings/azure/servicebusqueues/servicebusqueues.go @@ -204,10 +204,9 @@ func (a *AzureServiceBusQueues) Close() (err error) { } // GetComponentMetadata returns the metadata of the component. -func (a *AzureServiceBusQueues) GetComponentMetadata() map[string]string { +func (a *AzureServiceBusQueues) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := impl.Metadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) delete(metadataInfo, "consumerID") // only applies to topics, not queues - return metadataInfo + return } diff --git a/bindings/azure/signalr/signalr.go b/bindings/azure/signalr/signalr.go index a446ffc9a1..2e4e3821fa 100644 --- a/bindings/azure/signalr/signalr.go +++ b/bindings/azure/signalr/signalr.go @@ -301,9 +301,8 @@ func (s *SignalR) getToken(ctx context.Context, url string) (string, error) { } // GetComponentMetadata returns the metadata of the component. -func (s *SignalR) GetComponentMetadata() map[string]string { +func (s *SignalR) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := SignalRMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/azure/storagequeues/storagequeues.go b/bindings/azure/storagequeues/storagequeues.go index 183b2f92cd..babfb2feab 100644 --- a/bindings/azure/storagequeues/storagequeues.go +++ b/bindings/azure/storagequeues/storagequeues.go @@ -370,9 +370,8 @@ func (a *AzureStorageQueues) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (a *AzureStorageQueues) GetComponentMetadata() map[string]string { +func (a *AzureStorageQueues) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := storageQueuesMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/cloudflare/queues/cfqueues.go b/bindings/cloudflare/queues/cfqueues.go index 1b3d04a106..30d89ccb49 100644 --- a/bindings/cloudflare/queues/cfqueues.go +++ b/bindings/cloudflare/queues/cfqueues.go @@ -136,9 +136,8 @@ func (q *CFQueues) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (q *CFQueues) GetComponentMetadata() map[string]string { +func (q *CFQueues) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := componentMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/commercetools/commercetools.go b/bindings/commercetools/commercetools.go index 55010f4e2e..7efd4053cf 100644 --- a/bindings/commercetools/commercetools.go +++ b/bindings/commercetools/commercetools.go @@ -201,9 +201,8 @@ func (ct *Binding) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (ct Binding) GetComponentMetadata() map[string]string { +func (ct Binding) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := commercetoolsMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/cron/cron.go b/bindings/cron/cron.go index 53a5754582..db892616b9 100644 --- a/bindings/cron/cron.go +++ b/bindings/cron/cron.go @@ -132,9 +132,8 @@ func (b *Binding) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (b *Binding) GetComponentMetadata() map[string]string { +func (b *Binding) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := metadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/dubbo/dubbo_output.go b/bindings/dubbo/dubbo_output.go index d57d823739..ab117589b3 100644 --- a/bindings/dubbo/dubbo_output.go +++ b/bindings/dubbo/dubbo_output.go @@ -27,6 +27,7 @@ import ( dubboImpl "dubbo.apache.org/dubbo-go/v3/protocol/dubbo/impl" "github.com/dapr/components-contrib/bindings" + "github.com/dapr/components-contrib/metadata" "github.com/dapr/kit/logger" ) @@ -93,7 +94,6 @@ func (out *DubboOutputBinding) Operations() []bindings.OperationKind { } // GetComponentMetadata returns the metadata of the component. -func (out *DubboOutputBinding) GetComponentMetadata() map[string]string { - metadataInfo := map[string]string{} - return metadataInfo +func (out *DubboOutputBinding) GetComponentMetadata() metadata.MetadataMap { + return metadata.MetadataMap{} } diff --git a/bindings/gcp/bucket/bucket.go b/bindings/gcp/bucket/bucket.go index e682c170b1..941b43ff4c 100644 --- a/bindings/gcp/bucket/bucket.go +++ b/bindings/gcp/bucket/bucket.go @@ -31,7 +31,7 @@ import ( "github.com/dapr/components-contrib/bindings" "github.com/dapr/components-contrib/internal/utils" - contribMetadata "github.com/dapr/components-contrib/metadata" + "github.com/dapr/components-contrib/metadata" "github.com/dapr/kit/logger" ) @@ -110,7 +110,7 @@ func (g *GCPStorage) Init(ctx context.Context, metadata bindings.Metadata) error func (g *GCPStorage) parseMetadata(meta bindings.Metadata) (*gcpMetadata, error) { m := gcpMetadata{} - err := contribMetadata.DecodeMetadata(meta.Properties, &m) + err := metadata.DecodeMetadata(meta.Properties, &m) if err != nil { return nil, err } @@ -311,9 +311,8 @@ func (g *GCPStorage) handleBackwardCompatibilityForMetadata(metadata map[string] } // GetComponentMetadata returns the metadata of the component. -func (g *GCPStorage) GetComponentMetadata() map[string]string { +func (g *GCPStorage) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := gcpMetadata{} - metadataInfo := map[string]string{} - contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) + return } diff --git a/bindings/gcp/pubsub/pubsub.go b/bindings/gcp/pubsub/pubsub.go index 7cc7483f09..dfc8a69787 100644 --- a/bindings/gcp/pubsub/pubsub.go +++ b/bindings/gcp/pubsub/pubsub.go @@ -151,9 +151,8 @@ func (g *GCPPubSub) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (g *GCPPubSub) GetComponentMetadata() map[string]string { +func (g *GCPPubSub) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := pubSubMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/graphql/graphql.go b/bindings/graphql/graphql.go index 91edb090e9..379c863e54 100644 --- a/bindings/graphql/graphql.go +++ b/bindings/graphql/graphql.go @@ -186,9 +186,8 @@ func (gql *GraphQL) runRequest(ctx context.Context, requestKey string, req *bind } // GetComponentMetadata returns the metadata of the component. -func (gql *GraphQL) GetComponentMetadata() map[string]string { +func (gql *GraphQL) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := graphQLMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/http/http.go b/bindings/http/http.go index 63fe4ee4f6..177a3af087 100644 --- a/bindings/http/http.go +++ b/bindings/http/http.go @@ -338,9 +338,8 @@ func (h *HTTPSource) Invoke(parentCtx context.Context, req *bindings.InvokeReque } // GetComponentMetadata returns the metadata of the component. -func (h *HTTPSource) GetComponentMetadata() map[string]string { +func (h *HTTPSource) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := httpMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/huawei/obs/obs.go b/bindings/huawei/obs/obs.go index 8e68e2d8d2..26cad599f3 100644 --- a/bindings/huawei/obs/obs.go +++ b/bindings/huawei/obs/obs.go @@ -322,9 +322,8 @@ func (o *HuaweiOBS) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*b } // GetComponentMetadata returns the metadata of the component. -func (o *HuaweiOBS) GetComponentMetadata() map[string]string { +func (o *HuaweiOBS) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := obsMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/influx/influx.go b/bindings/influx/influx.go index 87fad6b805..30bc43595e 100644 --- a/bindings/influx/influx.go +++ b/bindings/influx/influx.go @@ -167,9 +167,8 @@ func (i *Influx) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (i *Influx) GetComponentMetadata() map[string]string { +func (i *Influx) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := influxMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/input_binding.go b/bindings/input_binding.go index 5892bf2d72..b87f7b9d4c 100644 --- a/bindings/input_binding.go +++ b/bindings/input_binding.go @@ -19,10 +19,13 @@ import ( "io" "github.com/dapr/components-contrib/health" + "github.com/dapr/components-contrib/metadata" ) // InputBinding is the interface to define a binding that triggers on incoming events. type InputBinding interface { + metadata.ComponentWithMetadata + // Init passes connection and properties metadata to the binding implementation. Init(ctx context.Context, metadata Metadata) error // Read is a method that runs in background and triggers the callback function whenever an event arrives. @@ -30,7 +33,6 @@ type InputBinding interface { // Close is a method that closes the connection to the binding. Must be // called when the binding is no longer needed to free up resources. io.Closer - GetComponentMetadata() map[string]string } // Handler is the handler used to invoke the app handler. diff --git a/bindings/kafka/kafka.go b/bindings/kafka/kafka.go index ddcb899bb6..266c4a43b2 100644 --- a/bindings/kafka/kafka.go +++ b/bindings/kafka/kafka.go @@ -138,9 +138,8 @@ func adaptHandler(handler bindings.Handler) kafka.EventHandler { } // GetComponentMetadata returns the metadata of the component. -func (b *Binding) GetComponentMetadata() map[string]string { +func (b *Binding) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := kafka.KafkaMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.BindingType) - return metadataInfo + return } diff --git a/bindings/kitex/kitex_output.go b/bindings/kitex/kitex_output.go index 4709218df3..171a832837 100644 --- a/bindings/kitex/kitex_output.go +++ b/bindings/kitex/kitex_output.go @@ -18,6 +18,7 @@ import ( "sync" "github.com/dapr/components-contrib/bindings" + "github.com/dapr/components-contrib/metadata" "github.com/dapr/kit/logger" ) @@ -79,7 +80,6 @@ func (out *kitexOutputBinding) Operations() []bindings.OperationKind { } // GetComponentMetadata returns the metadata of the component. -func (out *kitexOutputBinding) GetComponentMetadata() map[string]string { - metadataInfo := map[string]string{} - return metadataInfo +func (out *kitexOutputBinding) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { + return } diff --git a/bindings/kubemq/kubemq.go b/bindings/kubemq/kubemq.go index dde819568a..bee647248e 100644 --- a/bindings/kubemq/kubemq.go +++ b/bindings/kubemq/kubemq.go @@ -172,9 +172,8 @@ func (k *kubeMQ) processQueueMessage(ctx context.Context, handler bindings.Handl } // GetComponentMetadata returns the metadata of the component. -func (k *kubeMQ) GetComponentMetadata() map[string]string { +func (k *kubeMQ) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := options{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/kubernetes/kubernetes.go b/bindings/kubernetes/kubernetes.go index f854fa6678..0312888db3 100644 --- a/bindings/kubernetes/kubernetes.go +++ b/bindings/kubernetes/kubernetes.go @@ -202,9 +202,8 @@ func (k *kubernetesInput) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (k *kubernetesInput) GetComponentMetadata() map[string]string { +func (k *kubernetesInput) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := kubernetesMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/localstorage/localstorage.go b/bindings/localstorage/localstorage.go index 654bc54346..79b10388ff 100644 --- a/bindings/localstorage/localstorage.go +++ b/bindings/localstorage/localstorage.go @@ -336,9 +336,8 @@ func (ls *LocalStorage) Invoke(_ context.Context, req *bindings.InvokeRequest) ( } // GetComponentMetadata returns the metadata of the component. -func (ls *LocalStorage) GetComponentMetadata() map[string]string { +func (ls *LocalStorage) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := Metadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/mqtt3/mqtt.go b/bindings/mqtt3/mqtt.go index d11ecb0b2c..bac9ae5ea6 100644 --- a/bindings/mqtt3/mqtt.go +++ b/bindings/mqtt3/mqtt.go @@ -384,9 +384,8 @@ func (m *MQTT) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (m *MQTT) GetComponentMetadata() map[string]string { +func (m *MQTT) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := mqtt3Metadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/mysql/mysql.go b/bindings/mysql/mysql.go index 2127caf0f3..5143a02096 100644 --- a/bindings/mysql/mysql.go +++ b/bindings/mysql/mysql.go @@ -370,9 +370,8 @@ func (m *Mysql) convert(columnTypes []*sql.ColumnType, values []any) map[string] } // GetComponentMetadata returns the metadata of the component. -func (m *Mysql) GetComponentMetadata() map[string]string { +func (m *Mysql) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := mysqlMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/nacos/nacos.go b/bindings/nacos/nacos.go index c0e506f5e9..7296e70f65 100644 --- a/bindings/nacos/nacos.go +++ b/bindings/nacos/nacos.go @@ -457,9 +457,8 @@ func parseServerURL(s string) (*constant.ServerConfig, error) { } // GetComponentMetadata returns the metadata of the component. -func (n *Nacos) GetComponentMetadata() map[string]string { +func (n *Nacos) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := Settings{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/output_binding.go b/bindings/output_binding.go index 39f52988c5..42c46bb9ab 100644 --- a/bindings/output_binding.go +++ b/bindings/output_binding.go @@ -18,14 +18,16 @@ import ( "fmt" "github.com/dapr/components-contrib/health" + "github.com/dapr/components-contrib/metadata" ) // OutputBinding is the interface for an output binding, allowing users to invoke remote systems with optional payloads. type OutputBinding interface { + metadata.ComponentWithMetadata + Init(ctx context.Context, metadata Metadata) error Invoke(ctx context.Context, req *InvokeRequest) (*InvokeResponse, error) Operations() []OperationKind - GetComponentMetadata() map[string]string } func PingOutBinding(ctx context.Context, outputBinding OutputBinding) error { diff --git a/bindings/postgres/postgres.go b/bindings/postgres/postgres.go index 637b88d344..3a0ab98252 100644 --- a/bindings/postgres/postgres.go +++ b/bindings/postgres/postgres.go @@ -213,9 +213,8 @@ func (p *Postgres) exec(ctx context.Context, sql string, args ...any) (result in } // GetComponentMetadata returns the metadata of the component. -func (p *Postgres) GetComponentMetadata() map[string]string { +func (p *Postgres) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := psqlMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/postmark/postmark.go b/bindings/postmark/postmark.go index 21bece8a93..7560bff28b 100644 --- a/bindings/postmark/postmark.go +++ b/bindings/postmark/postmark.go @@ -161,9 +161,8 @@ func (p *Postmark) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*bi } // GetComponentMetadata returns the metadata of the component. -func (p *Postmark) GetComponentMetadata() map[string]string { +func (p *Postmark) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := postmarkMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/rabbitmq/rabbitmq.go b/bindings/rabbitmq/rabbitmq.go index b8a59457f8..e10f7422f3 100644 --- a/bindings/rabbitmq/rabbitmq.go +++ b/bindings/rabbitmq/rabbitmq.go @@ -548,9 +548,8 @@ func (r *RabbitMQ) reset() (err error) { return err } -func (r *RabbitMQ) GetComponentMetadata() map[string]string { +func (r *RabbitMQ) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := rabbitMQMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/redis/redis.go b/bindings/redis/redis.go index 0da6043f0c..5d029886d3 100644 --- a/bindings/redis/redis.go +++ b/bindings/redis/redis.go @@ -145,9 +145,8 @@ func (r *Redis) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (r *Redis) GetComponentMetadata() map[string]string { +func (r *Redis) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := rediscomponent.Settings{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/rethinkdb/statechange/statechange.go b/bindings/rethinkdb/statechange/statechange.go index 0d3188b80e..81639d473c 100644 --- a/bindings/rethinkdb/statechange/statechange.go +++ b/bindings/rethinkdb/statechange/statechange.go @@ -166,9 +166,8 @@ func metadataToConfig(cfg map[string]string, logger logger.Logger) (StateConfig, } // GetComponentMetadata returns the metadata of the component. -func (b *Binding) GetComponentMetadata() map[string]string { +func (b *Binding) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := StateConfig{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/smtp/smtp.go b/bindings/smtp/smtp.go index 668b88ddda..12ff6473af 100644 --- a/bindings/smtp/smtp.go +++ b/bindings/smtp/smtp.go @@ -231,9 +231,8 @@ func (metadata Metadata) parseAddresses(addresses string) []string { } // GetComponentMetadata returns the metadata of the component. -func (s *Mailer) GetComponentMetadata() map[string]string { +func (s *Mailer) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := Metadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/twilio/sendgrid/sendgrid.go b/bindings/twilio/sendgrid/sendgrid.go index 1bb75ffbb6..1204c977a0 100644 --- a/bindings/twilio/sendgrid/sendgrid.go +++ b/bindings/twilio/sendgrid/sendgrid.go @@ -264,11 +264,10 @@ func (sg *SendGrid) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*b } // GetComponentMetadata returns the metadata of the component. -func (sg *SendGrid) GetComponentMetadata() map[string]string { +func (sg *SendGrid) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := sendGridMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } // Function that unmarshals the Dynamic Template Data JSON String into a map[string]any. diff --git a/bindings/twilio/sms/sms.go b/bindings/twilio/sms/sms.go index b4e35c5a5d..13723c6f0a 100644 --- a/bindings/twilio/sms/sms.go +++ b/bindings/twilio/sms/sms.go @@ -135,9 +135,8 @@ func (t *SMS) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*binding } // GetComponentMetadata returns the metadata of the component. -func (t *SMS) GetComponentMetadata() map[string]string { +func (t *SMS) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := twilioMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/wasm/output.go b/bindings/wasm/output.go index 9d80d5511e..429eb8f6a0 100644 --- a/bindings/wasm/output.go +++ b/bindings/wasm/output.go @@ -168,9 +168,8 @@ func detectImports(imports []api.FunctionDefinition) importMode { } // GetComponentMetadata returns the metadata of the component. -func (out *outputBinding) GetComponentMetadata() map[string]string { +func (out *outputBinding) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := wasm.InitMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/zeebe/command/command.go b/bindings/zeebe/command/command.go index 1e7ab1b71e..a0f8a76011 100644 --- a/bindings/zeebe/command/command.go +++ b/bindings/zeebe/command/command.go @@ -131,9 +131,8 @@ func (z *ZeebeCommand) Invoke(ctx context.Context, req *bindings.InvokeRequest) } // GetComponentMetadata returns the metadata of the component. -func (z *ZeebeCommand) GetComponentMetadata() map[string]string { +func (z *ZeebeCommand) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := zeebe.ClientMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/bindings/zeebe/jobworker/jobworker.go b/bindings/zeebe/jobworker/jobworker.go index 60703a4f02..21638ca26a 100644 --- a/bindings/zeebe/jobworker/jobworker.go +++ b/bindings/zeebe/jobworker/jobworker.go @@ -256,9 +256,8 @@ func (h *jobHandler) failJob(ctx context.Context, client worker.JobClient, job e } // GetComponentMetadata returns the metadata of the component. -func (z *ZeebeJobWorker) GetComponentMetadata() map[string]string { +func (z *ZeebeJobWorker) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := jobWorkerMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.BindingType) - return metadataInfo + return } diff --git a/component-metadata-schema.json b/component-metadata-schema.json index be4333ad68..b6d7020727 100644 --- a/component-metadata-schema.json +++ b/component-metadata-schema.json @@ -76,7 +76,7 @@ "properties": { "name": { "type": "string", - "description": "Name of the built-in authentication profile.\nCurrently supports:\n\n- `azuread` (Azure AD, including Managed Identity)." + "description": "Name of the built-in authentication profile." }, "metadata": { "items": { diff --git a/configuration/azure/appconfig/appconfig.go b/configuration/azure/appconfig/appconfig.go index 2a27d410ee..a18f5d367c 100644 --- a/configuration/azure/appconfig/appconfig.go +++ b/configuration/azure/appconfig/appconfig.go @@ -338,9 +338,8 @@ func (r *ConfigurationStore) Unsubscribe(ctx context.Context, req *configuration } // GetComponentMetadata returns the metadata of the component. -func (r *ConfigurationStore) GetComponentMetadata() map[string]string { +func (r *ConfigurationStore) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := metadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.ConfigurationStoreType) - return metadataInfo + return } diff --git a/configuration/postgres/postgres.go b/configuration/postgres/postgres.go index 3a072da851..f5cef79234 100644 --- a/configuration/postgres/postgres.go +++ b/configuration/postgres/postgres.go @@ -376,9 +376,8 @@ func (p *ConfigurationStore) subscribeToChannel(ctx context.Context, pgNotifyCha } // GetComponentMetadata returns the metadata of the component. -func (p *ConfigurationStore) GetComponentMetadata() map[string]string { +func (p *ConfigurationStore) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := metadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.ConfigurationStoreType) - return metadataInfo + return } diff --git a/configuration/redis/redis.go b/configuration/redis/redis.go index 2ddb62012f..e5125e5601 100644 --- a/configuration/redis/redis.go +++ b/configuration/redis/redis.go @@ -245,9 +245,8 @@ func (r *ConfigurationStore) handleSubscribedChange(ctx context.Context, req *co } // GetComponentMetadata returns the metadata of the component. -func (r *ConfigurationStore) GetComponentMetadata() map[string]string { +func (r *ConfigurationStore) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := rediscomponent.Settings{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.ConfigurationStoreType) - return metadataInfo + return } diff --git a/configuration/store.go b/configuration/store.go index 0a82b4d8a2..c0f167fe57 100644 --- a/configuration/store.go +++ b/configuration/store.go @@ -13,10 +13,16 @@ limitations under the License. package configuration -import "context" +import ( + "context" + + "github.com/dapr/components-contrib/metadata" +) // Store is an interface to perform operations on store. type Store interface { + metadata.ComponentWithMetadata + // Init configuration store. Init(ctx context.Context, metadata Metadata) error @@ -28,9 +34,6 @@ type Store interface { // Unsubscribe configuration with keys Unsubscribe(ctx context.Context, req *UnsubscribeRequest) error - - // GetComponentMetadata returns information on the component's metadata. - GetComponentMetadata() map[string]string } // UpdateHandler is the handler used to send event to daprd. diff --git a/crypto/azure/keyvault/component.go b/crypto/azure/keyvault/component.go index b009ed00a5..558775b919 100644 --- a/crypto/azure/keyvault/component.go +++ b/crypto/azure/keyvault/component.go @@ -406,11 +406,10 @@ func (keyvaultCrypto) SupportedSignatureAlgorithms() []string { return signatureAlgsList } -func (keyvaultCrypto) GetComponentMetadata() map[string]string { +func (keyvaultCrypto) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := keyvaultMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.CryptoType) - return metadataInfo + return } type keyID struct { diff --git a/crypto/jwks/component.go b/crypto/jwks/component.go index 56e22c1184..42a18d11d9 100644 --- a/crypto/jwks/component.go +++ b/crypto/jwks/component.go @@ -126,9 +126,8 @@ func (k *jwksCrypto) retrieveKeyFromSecretFn(parentCtx context.Context, kid stri return key, nil } -func (k *jwksCrypto) GetComponentMetadata() map[string]string { +func (k *jwksCrypto) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := jwksMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.CryptoType) - return metadataInfo + return } diff --git a/crypto/kubernetes/secrets/component.go b/crypto/kubernetes/secrets/component.go index c4a7436533..7acd88238d 100644 --- a/crypto/kubernetes/secrets/component.go +++ b/crypto/kubernetes/secrets/component.go @@ -28,7 +28,7 @@ import ( contribCrypto "github.com/dapr/components-contrib/crypto" kubeclient "github.com/dapr/components-contrib/internal/authentication/kubernetes" - contribMetadata "github.com/dapr/components-contrib/metadata" + "github.com/dapr/components-contrib/metadata" internals "github.com/dapr/kit/crypto" "github.com/dapr/kit/logger" ) @@ -136,9 +136,8 @@ func (k *kubeSecretsCrypto) parseKeyString(param string) (namespace string, secr return } -func (kubeSecretsCrypto) GetComponentMetadata() map[string]string { +func (kubeSecretsCrypto) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := secretsMetadata{} - metadataInfo := map[string]string{} - contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.CryptoType) - return metadataInfo + metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.CryptoType) + return } diff --git a/crypto/localstorage/component.go b/crypto/localstorage/component.go index 45cc7f654c..f74bae7d03 100644 --- a/crypto/localstorage/component.go +++ b/crypto/localstorage/component.go @@ -105,9 +105,8 @@ func (l *localStorageCrypto) retrieveKey(parentCtx context.Context, key string) return jwkObj, nil } -func (localStorageCrypto) GetComponentMetadata() map[string]string { +func (localStorageCrypto) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := localStorageMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.CryptoType) - return metadataInfo + return } diff --git a/crypto/subtlecrypto.go b/crypto/subtlecrypto.go index dad9dba3c1..6f227f807d 100644 --- a/crypto/subtlecrypto.go +++ b/crypto/subtlecrypto.go @@ -17,10 +17,14 @@ import ( "context" "github.com/lestrrat-go/jwx/v2/jwk" + + "github.com/dapr/components-contrib/metadata" ) // SubtleCrypto offers an interface to perform low-level ("subtle") cryptographic operations with keys stored in a vault. type SubtleCrypto interface { + metadata.ComponentWithMetadata + SubtleCryptoAlgorithms // Init the component. @@ -161,9 +165,6 @@ type SubtleCrypto interface { valid bool, err error, ) - - // GetComponentMetadata returns information on the component's metadata. - GetComponentMetadata() map[string]string } // SubtleCryptoAlgorithms is an extension to SubtleCrypto that includes methods to return information on the supported algorithms. diff --git a/internal/component/azure/blobstorage/metadata.go b/internal/component/azure/blobstorage/metadata.go index 4a24f72f09..743ecbeebd 100644 --- a/internal/component/azure/blobstorage/metadata.go +++ b/internal/component/azure/blobstorage/metadata.go @@ -25,7 +25,7 @@ import ( type BlobStorageMetadata struct { ContainerClientOpts `json:",inline" mapstructure:",squash"` - DecodeBase64 bool `json:"decodeBase64,string" mapstructure:"decodeBase64" only:"bindings"` + DecodeBase64 bool `json:"decodeBase64,string" mapstructure:"decodeBase64" mdonly:"bindings"` PublicAccessLevel azblob.PublicAccessType } diff --git a/internal/component/azure/eventhubs/metadata.go b/internal/component/azure/eventhubs/metadata.go index 4f6152c198..2547f7fb73 100644 --- a/internal/component/azure/eventhubs/metadata.go +++ b/internal/component/azure/eventhubs/metadata.go @@ -40,9 +40,8 @@ type AzureEventHubsMetadata struct { ResourceGroupName string `json:"resourceGroupName" mapstructure:"resourceGroupName"` // Binding only - EventHub string `json:"eventHub" mapstructure:"eventHub" only:"bindings"` - ConsumerGroup string `json:"consumerGroup" mapstructure:"consumerGroup" only:"bindings"` // Alias for ConsumerID - PartitionID string `json:"partitionID" mapstructure:"partitionID" only:"bindings"` // Deprecated + EventHub string `json:"eventHub" mapstructure:"eventHub" mdonly:"bindings"` + ConsumerGroup string `json:"consumerGroup" mapstructure:"consumerGroup" mdonly:"bindings"` // Alias for ConsumerID // Internal properties namespaceName string @@ -91,16 +90,9 @@ func parseEventHubsMetadata(meta map[string]string, isBinding bool, log logger.L return nil, errors.New("the provided connection string does not contain a value for 'EntityPath' and no 'eventHub' property was passed") } } - - // Property partitionID is deprecated - if m.PartitionID != "" { - log.Info("Property partitionID is deprecated and will be ignored") - m.PartitionID = "" - } } else { // Ignored when not a binding m.EventHub = "" - m.PartitionID = "" // If connecting using a connection string, parse hubName if m.ConnectionString != "" { diff --git a/internal/component/azure/servicebus/metadata.go b/internal/component/azure/servicebus/metadata.go index 84950e9583..5f49e23e9c 100644 --- a/internal/component/azure/servicebus/metadata.go +++ b/internal/component/azure/servicebus/metadata.go @@ -49,7 +49,7 @@ type Metadata struct { NamespaceName string `mapstructure:"namespaceName"` // Only for Azure AD /** For bindings only **/ - QueueName string `mapstructure:"queueName" only:"bindings"` // Only queues + QueueName string `mapstructure:"queueName" mdonly:"bindings"` // Only queues } // Keys. diff --git a/internal/component/postgresql/postgresql.go b/internal/component/postgresql/postgresql.go index f44d8abdfa..76af6bb568 100644 --- a/internal/component/postgresql/postgresql.go +++ b/internal/component/postgresql/postgresql.go @@ -120,9 +120,8 @@ func (p *PostgreSQL) GetDBAccess() dbAccess { return p.dbaccess } -func (p *PostgreSQL) GetComponentMetadata() map[string]string { +func (p *PostgreSQL) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := postgresMetadataStruct{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/internal/component/redis/settings.go b/internal/component/redis/settings.go index f71afff8ef..172e7a183e 100644 --- a/internal/component/redis/settings.go +++ b/internal/component/redis/settings.go @@ -79,23 +79,23 @@ type Settings struct { EnableTLS bool `mapstructure:"enableTLS"` // == state only properties == - TTLInSeconds *int `mapstructure:"ttlInSeconds" only:"state"` - QueryIndexes string `mapstructure:"queryIndexes" only:"state"` + TTLInSeconds *int `mapstructure:"ttlInSeconds" mdonly:"state"` + QueryIndexes string `mapstructure:"queryIndexes" mdonly:"state"` // == pubsub only properties == // The consumer identifier - ConsumerID string `mapstructure:"consumerID" only:"pubsub"` + ConsumerID string `mapstructure:"consumerID" mdonly:"pubsub"` // The interval between checking for pending messages to redelivery (0 disables redelivery) - RedeliverInterval time.Duration `mapstructure:"-" only:"pubsub"` + RedeliverInterval time.Duration `mapstructure:"-" mdonly:"pubsub"` // The amount time a message must be pending before attempting to redeliver it (0 disables redelivery) - ProcessingTimeout time.Duration `mapstructure:"processingTimeout" only:"pubsub"` + ProcessingTimeout time.Duration `mapstructure:"processingTimeout" mdonly:"pubsub"` // The size of the message queue for processing - QueueDepth uint `mapstructure:"queueDepth" only:"pubsub"` + QueueDepth uint `mapstructure:"queueDepth" mdonly:"pubsub"` // The number of concurrent workers that are processing messages - Concurrency uint `mapstructure:"concurrency" only:"pubsub"` + Concurrency uint `mapstructure:"concurrency" mdonly:"pubsub"` - // the max len of stream - MaxLenApprox int64 `mapstructure:"maxLenApprox" only:"pubsub"` + // The max len of stream + MaxLenApprox int64 `mapstructure:"maxLenApprox" mdonly:"pubsub"` } func (s *Settings) Decode(in interface{}) error { diff --git a/lock/redis/standalone.go b/lock/redis/standalone.go index 672b78e85c..12f3305de0 100644 --- a/lock/redis/standalone.go +++ b/lock/redis/standalone.go @@ -183,9 +183,8 @@ func (r *StandaloneRedisLock) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (r *StandaloneRedisLock) GetComponentMetadata() map[string]string { +func (r *StandaloneRedisLock) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := rediscomponent.Settings{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.LockStoreType) - return metadataInfo + return } diff --git a/lock/store.go b/lock/store.go index 9617a9d11b..0fb1758e2c 100644 --- a/lock/store.go +++ b/lock/store.go @@ -13,9 +13,15 @@ limitations under the License. package lock -import "context" +import ( + "context" + + "github.com/dapr/components-contrib/metadata" +) type Store interface { + metadata.ComponentWithMetadata + // Init this component. InitLockStore(ctx context.Context, metadata Metadata) error @@ -24,7 +30,4 @@ type Store interface { // Unlock tries to release a lock. Unlock(ctx context.Context, req *UnlockRequest) (*UnlockResponse, error) - - // GetComponentMetadata returns information on the component's metadata. - GetComponentMetadata() map[string]string } diff --git a/metadata/componentmetadata.go b/metadata/componentmetadata.go new file mode 100644 index 0000000000..d7dd83e6b6 --- /dev/null +++ b/metadata/componentmetadata.go @@ -0,0 +1,23 @@ +//go:build !metadata +// +build !metadata + +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metadata + +// ComponentWithMetadata is empty when the `metadata` build tag is not present. +// The build tag is present when running the linter. +type ComponentWithMetadata interface { + // Empty +} diff --git a/metadata/componentmetadata_metadata.go b/metadata/componentmetadata_metadata.go new file mode 100644 index 0000000000..3aa007d8e6 --- /dev/null +++ b/metadata/componentmetadata_metadata.go @@ -0,0 +1,23 @@ +//go:build metadata +// +build metadata + +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metadata + +// ComponentWithMetadata includes the GetComponentMetadata method when the `metadata` build tag is present. +// The build tag is present when running the linter. +type ComponentWithMetadata interface { + GetComponentMetadata() MetadataMap +} diff --git a/metadata/utils.go b/metadata/utils.go index 49d1811a84..1c9248c9ab 100644 --- a/metadata/utils.go +++ b/metadata/utils.go @@ -140,7 +140,7 @@ func GetMetadataProperty(props map[string]string, keys ...string) (val string, o // DecodeMetadata decodes metadata into a struct // This is an extension of mitchellh/mapstructure which also supports decoding durations -func DecodeMetadata(input interface{}, result interface{}) error { +func DecodeMetadata(input any, result any) error { // avoids a common mistake of passing the metadata struct, instead of the properties map // if input is of type struct, case it to metadata.Base and access the Properties instead v := reflect.ValueOf(input) @@ -174,8 +174,8 @@ func toTruthyBoolHookFunc() mapstructure.DecodeHookFunc { return func( f reflect.Type, t reflect.Type, - data interface{}, - ) (interface{}, error) { + data any, + ) (any, error) { if f == reflect.TypeOf("") && t == reflect.TypeOf(true) { val := data.(string) return utils.IsTruthy(val), nil @@ -192,8 +192,8 @@ func toStringArrayHookFunc() mapstructure.DecodeHookFunc { return func( f reflect.Type, t reflect.Type, - data interface{}, - ) (interface{}, error) { + data any, + ) (any, error) { if f == reflect.TypeOf("") && t == reflect.TypeOf([]string{}) { val := data.(string) return strings.Split(val, ","), nil @@ -231,8 +231,8 @@ func toTimeDurationArrayHookFunc() mapstructure.DecodeHookFunc { return func( f reflect.Type, t reflect.Type, - data interface{}, - ) (interface{}, error) { + data any, + ) (any, error) { if f == reflect.TypeOf("") && t == reflect.TypeOf([]time.Duration{}) { inputArrayString := data.(string) return convert(inputArrayString) @@ -296,9 +296,22 @@ func (t ComponentType) BuiltInMetadataProperties() []string { } } +type MetadataField struct { + // Field type + Type string + // True if the field should be ignored by the metadata analyzer + Ignored bool + // True if the field is deprecated + Deprecated bool + // Aliases used for old, deprecated names + Aliases []string +} + +type MetadataMap map[string]MetadataField + // GetMetadataInfoFromStructType converts a struct to a map of field name (or struct tag) to field type. // This is used to generate metadata documentation for components. -func GetMetadataInfoFromStructType(t reflect.Type, metadataMap *map[string]string, componentType ComponentType) error { +func GetMetadataInfoFromStructType(t reflect.Type, metadataMap *MetadataMap, componentType ComponentType) error { // Return if not struct or pointer to struct. if t.Kind() == reflect.Ptr { t = t.Elem() @@ -307,6 +320,10 @@ func GetMetadataInfoFromStructType(t reflect.Type, metadataMap *map[string]strin return fmt.Errorf("not a struct: %s", t.Kind().String()) } + if *metadataMap == nil { + *metadataMap = MetadataMap{} + } + for i := 0; i < t.NumField(); i++ { currentField := t.Field(i) // fields that are not exported cannot be set via the mapstructure metadata decoding mechanism @@ -318,10 +335,11 @@ func GetMetadataInfoFromStructType(t reflect.Type, metadataMap *map[string]strin if mapStructureTag == "-" { continue } - onlyTag := currentField.Tag.Get("only") - if onlyTag != "" { + + // If there's a "mdonly" tag, that metadata option is only included for certain component types + if mdOnlyTag := currentField.Tag.Get("mdonly"); mdOnlyTag != "" { include := false - onlyTags := strings.Split(onlyTag, ",") + onlyTags := strings.Split(mdOnlyTag, ",") for _, tag := range onlyTags { if tag == string(componentType) { include = true @@ -332,6 +350,24 @@ func GetMetadataInfoFromStructType(t reflect.Type, metadataMap *map[string]strin continue } } + + mdField := MetadataField{ + Type: currentField.Type.String(), + } + + // If there's a mdignore tag and that's truthy, the field should be ignored by the metadata analyzer + mdField.Ignored = utils.IsTruthy(currentField.Tag.Get("mdignore")) + + // If there's a "mddeprecated" tag, the field may be deprecated + mdField.Deprecated = utils.IsTruthy(currentField.Tag.Get("mddeprecated")) + + // If there's a "mdaliases" tag, the field contains aliases + // The value is a comma-separated string + if mdAliasesTag := currentField.Tag.Get("mdaliases"); mdAliasesTag != "" { + mdField.Aliases = strings.Split(mdAliasesTag, ",") + } + + // Handle mapstructure tags and get the field name mapStructureTags := strings.Split(mapStructureTag, ",") numTags := len(mapStructureTags) if numTags > 1 && mapStructureTags[numTags-1] == "squash" && currentField.Anonymous { @@ -345,7 +381,9 @@ func GetMetadataInfoFromStructType(t reflect.Type, metadataMap *map[string]strin } else { fieldName = currentField.Name } - (*metadataMap)[fieldName] = currentField.Type.String() + + // Add the field + (*metadataMap)[fieldName] = mdField } return nil diff --git a/metadata/utils_test.go b/metadata/utils_test.go index a26f9a608a..33a4af23c5 100644 --- a/metadata/utils_test.go +++ b/metadata/utils_test.go @@ -245,33 +245,61 @@ func TestMetadataStructToStringMap(t *testing.T) { Mybool *bool MyRegularDuration time.Duration SomethingWithCustomName string `mapstructure:"something_with_custom_name"` - PubSubOnlyProperty string `mapstructure:"pubsub_only_property" only:"pubsub"` - BindingOnlyProperty string `mapstructure:"binding_only_property" only:"bindings"` - PubSubAndBindingProperty string `mapstructure:"pubsub_and_binding_property" only:"pubsub,bindings"` + PubSubOnlyProperty string `mapstructure:"pubsub_only_property" mdonly:"pubsub"` + BindingOnlyProperty string `mapstructure:"binding_only_property" mdonly:"bindings"` + PubSubAndBindingProperty string `mapstructure:"pubsub_and_binding_property" mdonly:"pubsub,bindings"` MyDurationArray []time.Duration NotExportedByMapStructure string `mapstructure:"-"` notExported string //nolint:structcheck,unused + DeprecatedProperty string `mapstructure:"something_deprecated" mddeprecated:"true"` + Aliased string `mapstructure:"aliased" mdaliases:"another,name"` + Ignored string `mapstructure:"ignored" mdignore:"true"` } m := testMetadata{} - metadatainfo := map[string]string{} + metadatainfo := MetadataMap{} GetMetadataInfoFromStructType(reflect.TypeOf(m), &metadatainfo, BindingType) - assert.Equal(t, "string", metadatainfo["Mystring"]) - assert.Equal(t, "metadata.Duration", metadatainfo["Myduration"]) - assert.Equal(t, "int", metadatainfo["Myinteger"]) - assert.Equal(t, "float64", metadatainfo["Myfloat64"]) - assert.Equal(t, "*bool", metadatainfo["Mybool"]) - assert.Equal(t, "time.Duration", metadatainfo["MyRegularDuration"]) - assert.Equal(t, "string", metadatainfo["something_with_custom_name"]) + _ = assert.NotEmpty(t, metadatainfo["Mystring"]) && + assert.Equal(t, "string", metadatainfo["Mystring"].Type) + _ = assert.NotEmpty(t, metadatainfo["Myduration"]) && + assert.Equal(t, "metadata.Duration", metadatainfo["Myduration"].Type) + _ = assert.NotEmpty(t, metadatainfo["Myinteger"]) && + assert.Equal(t, "int", metadatainfo["Myinteger"].Type) + _ = assert.NotEmpty(t, metadatainfo["Myfloat64"]) && + assert.Equal(t, "float64", metadatainfo["Myfloat64"].Type) + _ = assert.NotEmpty(t, metadatainfo["Mybool"]) && + assert.Equal(t, "*bool", metadatainfo["Mybool"].Type) + _ = assert.NotEmpty(t, metadatainfo["MyRegularDuration"]) && + assert.Equal(t, "time.Duration", metadatainfo["MyRegularDuration"].Type) + _ = assert.NotEmpty(t, metadatainfo["something_with_custom_name"]) && + assert.Equal(t, "string", metadatainfo["something_with_custom_name"].Type) assert.NotContains(t, metadatainfo, "NestedStruct") assert.NotContains(t, metadatainfo, "SomethingWithCustomName") - assert.Equal(t, "string", metadatainfo["nested_string_custom"]) - assert.Equal(t, "string", metadatainfo["NestedString"]) + _ = assert.NotEmpty(t, metadatainfo["nested_string_custom"]) && + assert.Equal(t, "string", metadatainfo["nested_string_custom"].Type) + _ = assert.NotEmpty(t, metadatainfo["NestedString"]) && + assert.Equal(t, "string", metadatainfo["NestedString"].Type) assert.NotContains(t, metadatainfo, "pubsub_only_property") - assert.Equal(t, "string", metadatainfo["binding_only_property"]) - assert.Equal(t, "string", metadatainfo["pubsub_and_binding_property"]) - assert.Equal(t, "[]time.Duration", metadatainfo["MyDurationArray"]) + _ = assert.NotEmpty(t, metadatainfo["binding_only_property"]) && + assert.Equal(t, "string", metadatainfo["binding_only_property"].Type) + _ = assert.NotEmpty(t, metadatainfo["pubsub_and_binding_property"]) && + assert.Equal(t, "string", metadatainfo["pubsub_and_binding_property"].Type) + _ = assert.NotEmpty(t, metadatainfo["MyDurationArray"]) && + assert.Equal(t, "[]time.Duration", metadatainfo["MyDurationArray"].Type) assert.NotContains(t, metadatainfo, "NotExportedByMapStructure") assert.NotContains(t, metadatainfo, "notExported") + _ = assert.NotEmpty(t, metadatainfo["something_deprecated"]) && + assert.Equal(t, "string", metadatainfo["something_deprecated"].Type) && + assert.True(t, metadatainfo["something_deprecated"].Deprecated) + _ = assert.NotEmpty(t, metadatainfo["aliased"]) && + assert.Equal(t, "string", metadatainfo["aliased"].Type) && + assert.False(t, metadatainfo["aliased"].Deprecated) && + assert.False(t, metadatainfo["aliased"].Ignored) && + assert.Equal(t, []string{"another", "name"}, metadatainfo["aliased"].Aliases) + _ = assert.NotEmpty(t, metadatainfo["ignored"]) && + assert.Equal(t, "string", metadatainfo["ignored"].Type) && + assert.False(t, metadatainfo["ignored"].Deprecated) && + assert.True(t, metadatainfo["ignored"].Ignored) && + assert.Empty(t, metadatainfo["ignored"].Aliases) }) } diff --git a/middleware/http/bearer/bearer_middleware.go b/middleware/http/bearer/bearer_middleware.go index 404dd2dcd0..030403716a 100644 --- a/middleware/http/bearer/bearer_middleware.go +++ b/middleware/http/bearer/bearer_middleware.go @@ -126,9 +126,8 @@ func (m *Middleware) GetHandler(ctx context.Context, metadata middleware.Metadat }, nil } -func (m *Middleware) GetComponentMetadata() map[string]string { +func (m *Middleware) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := bearerMiddlewareMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.MiddlewareType) - return metadataInfo + return } diff --git a/middleware/http/oauth2/oauth2_middleware.go b/middleware/http/oauth2/oauth2_middleware.go index 7a7cb78c72..6ad773ea8d 100644 --- a/middleware/http/oauth2/oauth2_middleware.go +++ b/middleware/http/oauth2/oauth2_middleware.go @@ -155,9 +155,8 @@ func (m *Middleware) getNativeMetadata(metadata middleware.Metadata) (*oAuth2Mid return &middlewareMetadata, nil } -func (m *Middleware) GetComponentMetadata() map[string]string { +func (m *Middleware) GetComponentMetadata() (metadataInfo mdutils.MetadataMap) { metadataStruct := oAuth2MiddlewareMetadata{} - metadataInfo := map[string]string{} mdutils.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, mdutils.MiddlewareType) - return metadataInfo + return } diff --git a/middleware/http/oauth2clientcredentials/oauth2clientcredentials_middleware.go b/middleware/http/oauth2clientcredentials/oauth2clientcredentials_middleware.go index f71a7ae36a..a3cfff4036 100644 --- a/middleware/http/oauth2clientcredentials/oauth2clientcredentials_middleware.go +++ b/middleware/http/oauth2clientcredentials/oauth2clientcredentials_middleware.go @@ -178,9 +178,8 @@ func (m *Middleware) GetToken(ctx context.Context, conf *clientcredentials.Confi return tokenSource.Token() } -func (m *Middleware) GetComponentMetadata() map[string]string { +func (m *Middleware) GetComponentMetadata() (metadataInfo mdutils.MetadataMap) { metadataStruct := oAuth2ClientCredentialsMiddlewareMetadata{} - metadataInfo := map[string]string{} mdutils.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, mdutils.MiddlewareType) - return metadataInfo + return } diff --git a/middleware/http/opa/middleware.go b/middleware/http/opa/middleware.go index 12175152ae..c0c664441f 100644 --- a/middleware/http/opa/middleware.go +++ b/middleware/http/opa/middleware.go @@ -260,9 +260,8 @@ func (m *Middleware) getNativeMetadata(metadata middleware.Metadata) (*middlewar return &meta, nil } -func (m *Middleware) GetComponentMetadata() map[string]string { +func (m *Middleware) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := middlewareMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.MiddlewareType) - return metadataInfo + return } diff --git a/middleware/http/ratelimit/ratelimit_middleware.go b/middleware/http/ratelimit/ratelimit_middleware.go index 1bd342b425..b3caa87821 100644 --- a/middleware/http/ratelimit/ratelimit_middleware.go +++ b/middleware/http/ratelimit/ratelimit_middleware.go @@ -101,9 +101,8 @@ func (m *Middleware) getNativeMetadata(metadata middleware.Metadata) (*rateLimit return &middlewareMetadata, nil } -func (m *Middleware) GetComponentMetadata() map[string]string { +func (m *Middleware) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := rateLimitMiddlewareMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.MiddlewareType) - return metadataInfo + return } diff --git a/middleware/http/routeralias/routeralias_middleware.go b/middleware/http/routeralias/routeralias_middleware.go index 36ed67310e..1ac1929955 100644 --- a/middleware/http/routeralias/routeralias_middleware.go +++ b/middleware/http/routeralias/routeralias_middleware.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "net/http" + "reflect" "github.com/gorilla/mux" "gopkg.in/yaml.v3" @@ -118,6 +119,10 @@ func vars(r *http.Request) map[string]string { return nil } -func (m *Middleware) GetComponentMetadata() map[string]string { - return map[string]string{} +func (m *Middleware) GetComponentMetadata() (metadataInfo mdutils.MetadataMap) { + metadataStruct := struct { + Routes string `mapstructure:"routes"` + }{} + mdutils.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, mdutils.MiddlewareType) + return } diff --git a/middleware/http/routerchecker/routerchecker_middleware.go b/middleware/http/routerchecker/routerchecker_middleware.go index ed9bd5ce32..eaf01825eb 100644 --- a/middleware/http/routerchecker/routerchecker_middleware.go +++ b/middleware/http/routerchecker/routerchecker_middleware.go @@ -74,9 +74,8 @@ func (m *Middleware) getNativeMetadata(metadata middleware.Metadata) (*Metadata, return &middlewareMetadata, nil } -func (m *Middleware) GetComponentMetadata() map[string]string { +func (m *Middleware) GetComponentMetadata() (metadataInfo mdutils.MetadataMap) { metadataStruct := Metadata{} - metadataInfo := map[string]string{} mdutils.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, mdutils.MiddlewareType) - return metadataInfo + return } diff --git a/middleware/http/sentinel/middleware.go b/middleware/http/sentinel/middleware.go index 8ea851ac33..4680824d07 100644 --- a/middleware/http/sentinel/middleware.go +++ b/middleware/http/sentinel/middleware.go @@ -157,9 +157,8 @@ func getNativeMetadata(metadata middleware.Metadata) (*middlewareMetadata, error return &md, nil } -func (m *Middleware) GetComponentMetadata() map[string]string { +func (m *Middleware) GetComponentMetadata() (metadataInfo mdutils.MetadataMap) { metadataStruct := middlewareMetadata{} - metadataInfo := map[string]string{} mdutils.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, mdutils.MiddlewareType) - return metadataInfo + return } diff --git a/middleware/http/wasm/httpwasm.go b/middleware/http/wasm/httpwasm.go index aa024f6a55..af707c8e2d 100644 --- a/middleware/http/wasm/httpwasm.go +++ b/middleware/http/wasm/httpwasm.go @@ -121,9 +121,8 @@ func (rh *requestHandler) Close() error { return rh.mw.Close(ctx) } -func (m *middleware) GetComponentMetadata() map[string]string { +func (m *middleware) GetComponentMetadata() (metadataInfo mdutils.MetadataMap) { metadataStruct := wasm.InitMetadata{} - metadataInfo := map[string]string{} mdutils.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, mdutils.MiddlewareType) - return metadataInfo + return } diff --git a/middleware/middleware.go b/middleware/middleware.go index 4e61da34f3..88e13660e5 100644 --- a/middleware/middleware.go +++ b/middleware/middleware.go @@ -21,5 +21,4 @@ import ( // Middleware is the interface for a middleware. type Middleware interface { GetHandler(ctx context.Context, metadata Metadata) (func(next http.Handler) http.Handler, error) - GetComponentMetadata() map[string]string } diff --git a/pubsub/aws/snssqs/snssqs.go b/pubsub/aws/snssqs/snssqs.go index 9c8fd09aad..1d71d21ae9 100644 --- a/pubsub/aws/snssqs/snssqs.go +++ b/pubsub/aws/snssqs/snssqs.go @@ -931,9 +931,8 @@ func (s *snsSqs) Features() []pubsub.Feature { } // GetComponentMetadata returns the metadata of the component. -func (s *snsSqs) GetComponentMetadata() map[string]string { +func (s *snsSqs) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := snsSqsMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.PubSubType) - return metadataInfo + return } diff --git a/pubsub/azure/eventhubs/eventhubs.go b/pubsub/azure/eventhubs/eventhubs.go index 7fca9570c7..766c904a0d 100644 --- a/pubsub/azure/eventhubs/eventhubs.go +++ b/pubsub/azure/eventhubs/eventhubs.go @@ -147,9 +147,8 @@ func (aeh *AzureEventHubs) Close() (err error) { } // GetComponentMetadata returns the metadata of the component. -func (aeh *AzureEventHubs) GetComponentMetadata() map[string]string { +func (aeh *AzureEventHubs) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := impl.AzureEventHubsMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.PubSubType) - return metadataInfo + return } diff --git a/pubsub/azure/servicebus/queues/servicebus.go b/pubsub/azure/servicebus/queues/servicebus.go index 2870efc08f..61328d3b61 100644 --- a/pubsub/azure/servicebus/queues/servicebus.go +++ b/pubsub/azure/servicebus/queues/servicebus.go @@ -227,10 +227,9 @@ func (a *azureServiceBus) Features() []pubsub.Feature { } // GetComponentMetadata returns the metadata of the component. -func (a *azureServiceBus) GetComponentMetadata() map[string]string { +func (a *azureServiceBus) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := impl.Metadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.PubSubType) - delete(metadataInfo, "consumerID") // does not apply to queues - return metadataInfo + delete(metadataInfo, "consumerID") // only applies to topics, not queues + return } diff --git a/pubsub/azure/servicebus/topics/servicebus.go b/pubsub/azure/servicebus/topics/servicebus.go index 244c045740..97fbb6baef 100644 --- a/pubsub/azure/servicebus/topics/servicebus.go +++ b/pubsub/azure/servicebus/topics/servicebus.go @@ -317,9 +317,8 @@ func (a *azureServiceBus) connectAndReceiveWithSessions(ctx context.Context, req } // GetComponentMetadata returns the metadata of the component. -func (a *azureServiceBus) GetComponentMetadata() map[string]string { +func (a *azureServiceBus) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := impl.Metadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.PubSubType) - return metadataInfo + return } diff --git a/pubsub/gcp/pubsub/pubsub.go b/pubsub/gcp/pubsub/pubsub.go index d960371737..43db539871 100644 --- a/pubsub/gcp/pubsub/pubsub.go +++ b/pubsub/gcp/pubsub/pubsub.go @@ -402,9 +402,8 @@ func (g *GCPPubSub) Features() []pubsub.Feature { } // GetComponentMetadata returns the metadata of the component. -func (g *GCPPubSub) GetComponentMetadata() map[string]string { +func (g *GCPPubSub) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := metadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.PubSubType) - return metadataInfo + return } diff --git a/pubsub/in-memory/in-memory.go b/pubsub/in-memory/in-memory.go index 1521230453..b6b4ab2696 100644 --- a/pubsub/in-memory/in-memory.go +++ b/pubsub/in-memory/in-memory.go @@ -21,6 +21,7 @@ import ( "time" "github.com/dapr/components-contrib/internal/eventbus" + "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/pubsub" "github.com/dapr/kit/logger" ) @@ -112,6 +113,6 @@ func (a *bus) Subscribe(ctx context.Context, req pubsub.SubscribeRequest, handle } // GetComponentMetadata returns the metadata of the component. -func (a *bus) GetComponentMetadata() map[string]string { - return map[string]string{} +func (a *bus) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { + return } diff --git a/pubsub/jetstream/jetstream.go b/pubsub/jetstream/jetstream.go index 1c5f743360..e991a157c8 100644 --- a/pubsub/jetstream/jetstream.go +++ b/pubsub/jetstream/jetstream.go @@ -299,9 +299,8 @@ func sigHandler(seedKey string, nonce []byte) ([]byte, error) { } // GetComponentMetadata returns the metadata of the component. -func (js *jetstreamPubSub) GetComponentMetadata() map[string]string { +func (js *jetstreamPubSub) GetComponentMetadata() (metadataInfo mdutils.MetadataMap) { metadataStruct := metadata{} - metadataInfo := map[string]string{} mdutils.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, mdutils.PubSubType) - return metadataInfo + return } diff --git a/pubsub/kafka/kafka.go b/pubsub/kafka/kafka.go index c2902d55ac..c341c77ea8 100644 --- a/pubsub/kafka/kafka.go +++ b/pubsub/kafka/kafka.go @@ -177,9 +177,8 @@ func adaptBulkHandler(handler pubsub.BulkHandler) kafka.BulkEventHandler { } // GetComponentMetadata returns the metadata of the component. -func (p *PubSub) GetComponentMetadata() map[string]string { +func (p *PubSub) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := kafka.KafkaMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.PubSubType) - return metadataInfo + return } diff --git a/pubsub/kubemq/kubemq.go b/pubsub/kubemq/kubemq.go index b0760fc694..164a9f2a58 100644 --- a/pubsub/kubemq/kubemq.go +++ b/pubsub/kubemq/kubemq.go @@ -79,9 +79,8 @@ func getRandomID() string { } // GetComponentMetadata returns the metadata of the component. -func (k *kubeMQ) GetComponentMetadata() map[string]string { +func (k *kubeMQ) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := &kubemqMetadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.PubSubType) - return metadataInfo + return } diff --git a/pubsub/mqtt3/mqtt.go b/pubsub/mqtt3/mqtt.go index 1f5c9fab13..2d50fbcc89 100644 --- a/pubsub/mqtt3/mqtt.go +++ b/pubsub/mqtt3/mqtt.go @@ -495,9 +495,8 @@ func buildRegexForTopic(topicName string) string { } // GetComponentMetadata returns the metadata of the component. -func (m *mqttPubSub) GetComponentMetadata() map[string]string { +func (m *mqttPubSub) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := mqttMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.PubSubType) - return metadataInfo + return } diff --git a/pubsub/natsstreaming/natsstreaming.go b/pubsub/natsstreaming/natsstreaming.go index a45f66707e..51817ac721 100644 --- a/pubsub/natsstreaming/natsstreaming.go +++ b/pubsub/natsstreaming/natsstreaming.go @@ -363,9 +363,8 @@ func (n *natsStreamingPubSub) Features() []pubsub.Feature { } // GetComponentMetadata returns the metadata of the component. -func (n *natsStreamingPubSub) GetComponentMetadata() map[string]string { +func (n *natsStreamingPubSub) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := natsMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.PubSubType) - return metadataInfo + return } diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index c17ed652ad..fb83761a8a 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -18,16 +18,18 @@ import ( "fmt" "github.com/dapr/components-contrib/health" + "github.com/dapr/components-contrib/metadata" ) // PubSub is the interface for message buses. type PubSub interface { + metadata.ComponentWithMetadata + Init(ctx context.Context, metadata Metadata) error Features() []Feature Publish(ctx context.Context, req *PublishRequest) error Subscribe(ctx context.Context, req SubscribeRequest, handler Handler) error Close() error - GetComponentMetadata() map[string]string } // BulkPublisher is the interface that wraps the BulkPublish method. diff --git a/pubsub/pulsar/pulsar.go b/pubsub/pulsar/pulsar.go index bf49defa68..a04bac1acb 100644 --- a/pubsub/pulsar/pulsar.go +++ b/pubsub/pulsar/pulsar.go @@ -496,11 +496,10 @@ func (p *Pulsar) formatTopic(topic string) string { } // GetComponentMetadata returns the metadata of the component. -func (p *Pulsar) GetComponentMetadata() map[string]string { +func (p *Pulsar) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := pulsarMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.PubSubType) - return metadataInfo + return } // isValidPEM validates the provided input has PEM formatted block. diff --git a/pubsub/rabbitmq/rabbitmq.go b/pubsub/rabbitmq/rabbitmq.go index 249d5d8b70..e332b58ade 100644 --- a/pubsub/rabbitmq/rabbitmq.go +++ b/pubsub/rabbitmq/rabbitmq.go @@ -677,9 +677,8 @@ func mustReconnect(channel rabbitMQChannelBroker, err error) bool { } // GetComponentMetadata returns the metadata of the component. -func (r *rabbitMQ) GetComponentMetadata() map[string]string { +func (r *rabbitMQ) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := rabbitmqMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.PubSubType) - return metadataInfo + return } diff --git a/pubsub/redis/redis.go b/pubsub/redis/redis.go index bd501bd531..7beca3fe69 100644 --- a/pubsub/redis/redis.go +++ b/pubsub/redis/redis.go @@ -401,9 +401,8 @@ func (r *redisStreams) Ping(ctx context.Context) error { return nil } -func (r *redisStreams) GetComponentMetadata() map[string]string { +func (r *redisStreams) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := rediscomponent.Settings{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.PubSubType) - return metadataInfo + return } diff --git a/pubsub/rocketmq/rocketmq.go b/pubsub/rocketmq/rocketmq.go index 761680d928..83c54e74c6 100644 --- a/pubsub/rocketmq/rocketmq.go +++ b/pubsub/rocketmq/rocketmq.go @@ -554,9 +554,8 @@ func (r *rocketMQ) Close() error { } // GetComponentMetadata returns the metadata of the component. -func (r *rocketMQ) GetComponentMetadata() map[string]string { +func (r *rocketMQ) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := rocketMQMetaData{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.PubSubType) - return metadataInfo + return } diff --git a/pubsub/solace/amqp/amqp.go b/pubsub/solace/amqp/amqp.go index 8e56aaaa06..11a95e1977 100644 --- a/pubsub/solace/amqp/amqp.go +++ b/pubsub/solace/amqp/amqp.go @@ -327,9 +327,8 @@ func (a *amqpPubSub) Features() []pubsub.Feature { } // GetComponentMetadata returns the metadata of the component. -func (a *amqpPubSub) GetComponentMetadata() map[string]string { +func (a *amqpPubSub) GetComponentMetadata() (metadataInfo contribMetadata.MetadataMap) { metadataStruct := metadata{} - metadataInfo := map[string]string{} contribMetadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribMetadata.PubSubType) - return metadataInfo + return } diff --git a/secretstores/alicloud/parameterstore/parameterstore.go b/secretstores/alicloud/parameterstore/parameterstore.go index 66ce2f85d9..df42109129 100644 --- a/secretstores/alicloud/parameterstore/parameterstore.go +++ b/secretstores/alicloud/parameterstore/parameterstore.go @@ -197,9 +197,8 @@ func (o *oosSecretStore) Features() []secretstores.Feature { return []secretstores.Feature{} // No Feature supported. } -func (o *oosSecretStore) GetComponentMetadata() map[string]string { +func (o *oosSecretStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := ParameterStoreMetaData{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.SecretStoreType) - return metadataInfo + return } diff --git a/secretstores/aws/parameterstore/parameterstore.go b/secretstores/aws/parameterstore/parameterstore.go index e66632aab5..fef4294f32 100644 --- a/secretstores/aws/parameterstore/parameterstore.go +++ b/secretstores/aws/parameterstore/parameterstore.go @@ -172,9 +172,8 @@ func (s *ssmSecretStore) Features() []secretstores.Feature { return []secretstores.Feature{} // No Feature supported. } -func (s *ssmSecretStore) GetComponentMetadata() map[string]string { +func (s *ssmSecretStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := ParameterStoreMetaData{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.SecretStoreType) - return metadataInfo + return } diff --git a/secretstores/aws/secretmanager/secretmanager.go b/secretstores/aws/secretmanager/secretmanager.go index 1403b2535e..d30ce79e96 100644 --- a/secretstores/aws/secretmanager/secretmanager.go +++ b/secretstores/aws/secretmanager/secretmanager.go @@ -165,9 +165,8 @@ func (s *smSecretStore) Features() []secretstores.Feature { return []secretstores.Feature{} // No Feature supported. } -func (s *smSecretStore) GetComponentMetadata() map[string]string { +func (s *smSecretStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := SecretManagerMetaData{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.SecretStoreType) - return metadataInfo + return } diff --git a/secretstores/azure/keyvault/keyvault.go b/secretstores/azure/keyvault/keyvault.go index 80ca7c077f..3dc97d1b62 100644 --- a/secretstores/azure/keyvault/keyvault.go +++ b/secretstores/azure/keyvault/keyvault.go @@ -206,9 +206,8 @@ func (k *keyvaultSecretStore) Features() []secretstores.Feature { return []secretstores.Feature{} // No Feature supported. } -func (k *keyvaultSecretStore) GetComponentMetadata() map[string]string { +func (k *keyvaultSecretStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := KeyvaultMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.SecretStoreType) - return metadataInfo + return } diff --git a/secretstores/gcp/secretmanager/secretmanager.go b/secretstores/gcp/secretmanager/secretmanager.go index ef6dc5a2a6..01ea2133f1 100644 --- a/secretstores/gcp/secretmanager/secretmanager.go +++ b/secretstores/gcp/secretmanager/secretmanager.go @@ -203,9 +203,8 @@ func (s *Store) Features() []secretstores.Feature { return []secretstores.Feature{} // No Feature supported. } -func (s *Store) GetComponentMetadata() map[string]string { +func (s *Store) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := GcpSecretManagerMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.SecretStoreType) - return metadataInfo + return } diff --git a/secretstores/hashicorp/vault/vault.go b/secretstores/hashicorp/vault/vault.go index f9daa7a6ec..8fabf2819d 100644 --- a/secretstores/hashicorp/vault/vault.go +++ b/secretstores/hashicorp/vault/vault.go @@ -531,9 +531,8 @@ func (v *vaultSecretStore) Features() []secretstores.Feature { return []secretstores.Feature{secretstores.FeatureMultipleKeyValuesPerSecret} } -func (v *vaultSecretStore) GetComponentMetadata() map[string]string { +func (v *vaultSecretStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := VaultMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.SecretStoreType) - return metadataInfo + return } diff --git a/secretstores/huaweicloud/csms/csms.go b/secretstores/huaweicloud/csms/csms.go index c7d73e01e3..797b48cb14 100644 --- a/secretstores/huaweicloud/csms/csms.go +++ b/secretstores/huaweicloud/csms/csms.go @@ -157,9 +157,8 @@ func (c *csmsSecretStore) Features() []secretstores.Feature { return []secretstores.Feature{} // No Feature supported. } -func (c *csmsSecretStore) GetComponentMetadata() map[string]string { +func (c *csmsSecretStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := CsmsSecretStoreMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.SecretStoreType) - return metadataInfo + return } diff --git a/secretstores/kubernetes/kubernetes.go b/secretstores/kubernetes/kubernetes.go index 030b7a980c..3147c1ba37 100644 --- a/secretstores/kubernetes/kubernetes.go +++ b/secretstores/kubernetes/kubernetes.go @@ -18,7 +18,6 @@ import ( "context" "errors" "os" - "reflect" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -117,10 +116,7 @@ func (k *kubernetesSecretStore) Features() []secretstores.Feature { return []secretstores.Feature{} } -func (k *kubernetesSecretStore) GetComponentMetadata() map[string]string { - type unusedMetadataStruct struct{} - metadataStruct := unusedMetadataStruct{} - metadataInfo := map[string]string{} - metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.SecretStoreType) - return metadataInfo +func (k *kubernetesSecretStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { + // No component metadata + return } diff --git a/secretstores/local/env/envstore.go b/secretstores/local/env/envstore.go index 4395f227fe..11dbf2b005 100644 --- a/secretstores/local/env/envstore.go +++ b/secretstores/local/env/envstore.go @@ -114,11 +114,10 @@ func (s *envSecretStore) Features() []secretstores.Feature { return []secretstores.Feature{} // No Feature supported. } -func (s *envSecretStore) GetComponentMetadata() map[string]string { +func (s *envSecretStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := Metadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.SecretStoreType) - return metadataInfo + return } func (s *envSecretStore) isKeyAllowed(key string) bool { diff --git a/secretstores/local/file/filestore.go b/secretstores/local/file/filestore.go index 4373b075ae..a9b065a4b4 100644 --- a/secretstores/local/file/filestore.go +++ b/secretstores/local/file/filestore.go @@ -277,9 +277,8 @@ func (j *localSecretStore) Features() []secretstores.Feature { return j.features } -func (j *localSecretStore) GetComponentMetadata() map[string]string { +func (j *localSecretStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := localSecretStoreMetaData{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.SecretStoreType) - return metadataInfo + return } diff --git a/secretstores/secret_store.go b/secretstores/secret_store.go index 4257222b57..2241ba7580 100644 --- a/secretstores/secret_store.go +++ b/secretstores/secret_store.go @@ -18,10 +18,13 @@ import ( "fmt" "github.com/dapr/components-contrib/health" + "github.com/dapr/components-contrib/metadata" ) // SecretStore is the interface for a component that handles secrets management. type SecretStore interface { + metadata.ComponentWithMetadata + // Init authenticates with the actual secret store and performs other init operation Init(ctx context.Context, metadata Metadata) error // GetSecret retrieves a secret using a key and returns a map of decrypted string/string values. @@ -30,8 +33,6 @@ type SecretStore interface { BulkGetSecret(ctx context.Context, req BulkGetSecretRequest) (BulkGetSecretResponse, error) // Features lists the features supported by the secret store. Features() []Feature - // GetComponentMetadata returns the metadata options for the secret store. - GetComponentMetadata() map[string]string } func Ping(ctx context.Context, secretStore SecretStore) error { diff --git a/secretstores/tencentcloud/ssm/ssm.go b/secretstores/tencentcloud/ssm/ssm.go index 9f7834c3af..d40887661a 100644 --- a/secretstores/tencentcloud/ssm/ssm.go +++ b/secretstores/tencentcloud/ssm/ssm.go @@ -193,9 +193,8 @@ func (s *ssmSecretStore) Features() []secretstores.Feature { return []secretstores.Feature{} // No Feature supported. } -func (s *ssmSecretStore) GetComponentMetadata() map[string]string { +func (s *ssmSecretStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := SsmMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.SecretStoreType) - return metadataInfo + return } diff --git a/state/aerospike/aerospike.go b/state/aerospike/aerospike.go index 9568a76296..0a27094376 100644 --- a/state/aerospike/aerospike.go +++ b/state/aerospike/aerospike.go @@ -271,9 +271,8 @@ func convertETag(eTag string) (uint32, error) { return uint32(i), nil } -func (aspike *Aerospike) GetComponentMetadata() map[string]string { +func (aspike *Aerospike) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := aerospikeMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/state/alicloud/tablestore/tablestore.go b/state/alicloud/tablestore/tablestore.go index 1ea6cf3cf9..cf31709c79 100644 --- a/state/alicloud/tablestore/tablestore.go +++ b/state/alicloud/tablestore/tablestore.go @@ -228,9 +228,8 @@ func (s *AliCloudTableStore) primaryKey(key string) *tablestore.PrimaryKey { return pk } -func (s *AliCloudTableStore) GetComponentMetadata() map[string]string { +func (s *AliCloudTableStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := tablestoreMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/state/aws/dynamodb/dynamodb.go b/state/aws/dynamodb/dynamodb.go index 094a801f00..ba7c3ab96b 100644 --- a/state/aws/dynamodb/dynamodb.go +++ b/state/aws/dynamodb/dynamodb.go @@ -223,11 +223,10 @@ func (d *StateStore) Delete(ctx context.Context, req *state.DeleteRequest) error return err } -func (d *StateStore) GetComponentMetadata() map[string]string { +func (d *StateStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := dynamoDBMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } func (d *StateStore) getDynamoDBMetadata(meta state.Metadata) (*dynamoDBMetadata, error) { diff --git a/state/azure/blobstorage/blobstorage.go b/state/azure/blobstorage/blobstorage.go index 65644c5c51..803d50bb50 100644 --- a/state/azure/blobstorage/blobstorage.go +++ b/state/azure/blobstorage/blobstorage.go @@ -106,11 +106,10 @@ func (r *StateStore) Ping(ctx context.Context) error { return nil } -func (r *StateStore) GetComponentMetadata() map[string]string { +func (r *StateStore) GetComponentMetadata() (metadataInfo mdutils.MetadataMap) { metadataStruct := storageinternal.BlobStorageMetadata{} - metadataInfo := map[string]string{} mdutils.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, mdutils.StateStoreType) - return metadataInfo + return } // NewAzureBlobStorageStore instance. diff --git a/state/azure/cosmosdb/cosmosdb.go b/state/azure/cosmosdb/cosmosdb.go index c809634747..622c091b06 100644 --- a/state/azure/cosmosdb/cosmosdb.go +++ b/state/azure/cosmosdb/cosmosdb.go @@ -106,11 +106,10 @@ func NewCosmosDBStateStore(logger logger.Logger) state.Store { return s } -func (c *StateStore) GetComponentMetadata() map[string]string { +func (c *StateStore) GetComponentMetadata() (metadataInfo contribmeta.MetadataMap) { metadataStruct := metadata{} - metadataInfo := map[string]string{} contribmeta.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, contribmeta.StateStoreType) - return metadataInfo + return } // Init does metadata and connection parsing. diff --git a/state/azure/tablestorage/tablestorage.go b/state/azure/tablestorage/tablestorage.go index 8c89c66766..41a5e558c2 100644 --- a/state/azure/tablestorage/tablestorage.go +++ b/state/azure/tablestorage/tablestorage.go @@ -204,11 +204,10 @@ func (r *StateStore) Set(ctx context.Context, req *state.SetRequest) error { return r.writeRow(ctx, req) } -func (r *StateStore) GetComponentMetadata() map[string]string { +func (r *StateStore) GetComponentMetadata() (metadataInfo mdutils.MetadataMap) { metadataStruct := tablesMetadata{} - metadataInfo := map[string]string{} mdutils.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, mdutils.StateStoreType) - return metadataInfo + return } func NewAzureTablesStateStore(logger logger.Logger) state.Store { diff --git a/state/bulk_test.go b/state/bulk_test.go index d9e098d7ce..9c179fe985 100644 --- a/state/bulk_test.go +++ b/state/bulk_test.go @@ -22,6 +22,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/dapr/components-contrib/metadata" ) var errSimulated = errors.New("simulated") @@ -162,8 +164,8 @@ func (s *storeBulk) Set(ctx context.Context, req *SetRequest) error { return nil } -func (s *storeBulk) GetComponentMetadata() map[string]string { - return map[string]string{} +func (s *storeBulk) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { + return } func (s *storeBulk) Features() []Feature { diff --git a/state/cassandra/cassandra.go b/state/cassandra/cassandra.go index 9a74a962a0..3ac0cda60a 100644 --- a/state/cassandra/cassandra.go +++ b/state/cassandra/cassandra.go @@ -320,11 +320,10 @@ func (c *Cassandra) createSession(consistency gocql.Consistency) (*gocql.Session return session, nil } -func (c *Cassandra) GetComponentMetadata() map[string]string { +func (c *Cassandra) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := cassandraMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } // Close the connection to Cassandra. diff --git a/state/cloudflare/workerskv/workerskv.go b/state/cloudflare/workerskv/workerskv.go index 3e54e785c8..c1a8609f6b 100644 --- a/state/cloudflare/workerskv/workerskv.go +++ b/state/cloudflare/workerskv/workerskv.go @@ -82,11 +82,10 @@ func (q *CFWorkersKV) Init(_ context.Context, metadata state.Metadata) error { return q.Base.Init(workerBindings, componentDocsURL, infoResponseValidate) } -func (q *CFWorkersKV) GetComponentMetadata() map[string]string { +func (q *CFWorkersKV) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := componentMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } // Features returns the features supported by this state store. diff --git a/state/couchbase/couchbase.go b/state/couchbase/couchbase.go index 0ea4293dc9..3da43f1567 100644 --- a/state/couchbase/couchbase.go +++ b/state/couchbase/couchbase.go @@ -266,9 +266,8 @@ func eTagToCas(eTag string) (gocb.Cas, error) { return cas, nil } -func (cbs *Couchbase) GetComponentMetadata() map[string]string { +func (cbs *Couchbase) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := couchbaseMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/state/etcd/etcd.go b/state/etcd/etcd.go index 30b5c93df1..4bac16a911 100644 --- a/state/etcd/etcd.go +++ b/state/etcd/etcd.go @@ -303,11 +303,10 @@ func (e *Etcd) doValidateEtag(key string, etag *string, concurrency string) erro return nil } -func (e *Etcd) GetComponentMetadata() map[string]string { +func (e *Etcd) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := etcdConfig{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } func (e *Etcd) Close() error { diff --git a/state/gcp/firestore/firestore.go b/state/gcp/firestore/firestore.go index 8697651a8c..efda8e7d9c 100644 --- a/state/gcp/firestore/firestore.go +++ b/state/gcp/firestore/firestore.go @@ -168,11 +168,10 @@ func (f *Firestore) Delete(ctx context.Context, req *state.DeleteRequest) error return nil } -func (f *Firestore) GetComponentMetadata() map[string]string { +func (f *Firestore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := firestoreMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } func getFirestoreMetadata(meta state.Metadata) (*firestoreMetadata, error) { diff --git a/state/hashicorp/consul/consul.go b/state/hashicorp/consul/consul.go index eb79055204..6c5a2c5a92 100644 --- a/state/hashicorp/consul/consul.go +++ b/state/hashicorp/consul/consul.go @@ -157,9 +157,8 @@ func (c *Consul) Delete(ctx context.Context, req *state.DeleteRequest) error { return nil } -func (c *Consul) GetComponentMetadata() map[string]string { +func (c *Consul) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := consulConfig{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/state/hazelcast/hazelcast.go b/state/hazelcast/hazelcast.go index 6fae9efb62..0a41f04848 100644 --- a/state/hazelcast/hazelcast.go +++ b/state/hazelcast/hazelcast.go @@ -159,9 +159,8 @@ func (store *Hazelcast) Delete(ctx context.Context, req *state.DeleteRequest) er return nil } -func (store *Hazelcast) GetComponentMetadata() map[string]string { +func (store *Hazelcast) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := hazelcastMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/state/in-memory/in_memory.go b/state/in-memory/in_memory.go index 1d3d0e4753..a05f2bd94e 100644 --- a/state/in-memory/in_memory.go +++ b/state/in-memory/in_memory.go @@ -26,6 +26,7 @@ import ( "github.com/google/uuid" "k8s.io/utils/clock" + "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/state" "github.com/dapr/components-contrib/state/utils" "github.com/dapr/kit/logger" @@ -410,9 +411,9 @@ func (store *inMemoryStore) doCleanExpiredItems() { } } -func (store *inMemoryStore) GetComponentMetadata() map[string]string { +func (store *inMemoryStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { // no metadata, hence no metadata struct to convert here - return map[string]string{} + return } type inMemStateStoreItem struct { diff --git a/state/jetstream/jetstream.go b/state/jetstream/jetstream.go index c493d82d1a..fedb1e2f68 100644 --- a/state/jetstream/jetstream.go +++ b/state/jetstream/jetstream.go @@ -170,9 +170,8 @@ func escape(key string) string { return strings.ReplaceAll(key, "||", ".") } -func (js *StateStore) GetComponentMetadata() map[string]string { +func (js *StateStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := jetstreamMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/state/memcached/memcached.go b/state/memcached/memcached.go index be2f427bb3..d4171e13ff 100644 --- a/state/memcached/memcached.go +++ b/state/memcached/memcached.go @@ -217,9 +217,8 @@ func (m *Memcached) Close() (err error) { return nil } -func (m *Memcached) GetComponentMetadata() map[string]string { +func (m *Memcached) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := memcachedMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/state/mongodb/mongodb.go b/state/mongodb/mongodb.go index 26a8847f6d..c8dac85c20 100644 --- a/state/mongodb/mongodb.go +++ b/state/mongodb/mongodb.go @@ -675,11 +675,10 @@ func getReadConcernObject(cn string) (*readconcern.ReadConcern, error) { return nil, fmt.Errorf("readConcern %s not found", cn) } -func (m *MongoDB) GetComponentMetadata() map[string]string { +func (m *MongoDB) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := mongoDBMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } // Close connection to the database. diff --git a/state/mysql/mysql.go b/state/mysql/mysql.go index 4c94a41ba8..6e5f6414b9 100644 --- a/state/mysql/mysql.go +++ b/state/mysql/mysql.go @@ -893,9 +893,8 @@ type querier interface { QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row } -func (m *MySQL) GetComponentMetadata() map[string]string { +func (m *MySQL) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := mySQLMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/state/oci/objectstorage/objectstorage.go b/state/oci/objectstorage/objectstorage.go index 2612d086df..7ed5748eca 100644 --- a/state/oci/objectstorage/objectstorage.go +++ b/state/oci/objectstorage/objectstorage.go @@ -517,9 +517,8 @@ func (c *ociObjectStorageClient) pingBucket(ctx context.Context) error { return nil } -func (r *StateStore) GetComponentMetadata() map[string]string { +func (r *StateStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := objectStoreMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/state/oracledatabase/oracledatabase.go b/state/oracledatabase/oracledatabase.go index 570f36453e..32e883287b 100644 --- a/state/oracledatabase/oracledatabase.go +++ b/state/oracledatabase/oracledatabase.go @@ -98,9 +98,8 @@ func (o *OracleDatabase) getDB() *sql.DB { return o.dbaccess.(*oracleDatabaseAccess).db } -func (o *OracleDatabase) GetComponentMetadata() map[string]string { +func (o *OracleDatabase) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := oracleDatabaseMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/state/redis/redis.go b/state/redis/redis.go index 2435a26da8..d5e1c33ee4 100644 --- a/state/redis/redis.go +++ b/state/redis/redis.go @@ -547,10 +547,8 @@ func (r *StateStore) Close() error { return r.client.Close() } -func (r *StateStore) GetComponentMetadata() map[string]string { +func (r *StateStore) GetComponentMetadata() (metadataInfo daprmetadata.MetadataMap) { settingsStruct := rediscomponent.Settings{} - metadataInfo := map[string]string{} daprmetadata.GetMetadataInfoFromStructType(reflect.TypeOf(settingsStruct), &metadataInfo, daprmetadata.StateStoreType) - - return metadataInfo + return } diff --git a/state/redis/redis_test.go b/state/redis/redis_test.go index fe2fed70e9..599de3ae41 100644 --- a/state/redis/redis_test.go +++ b/state/redis/redis_test.go @@ -507,8 +507,6 @@ func TestGetMetadata(t *testing.T) { metadataInfo := ss.GetComponentMetadata() assert.Contains(t, metadataInfo, "redisHost") assert.Contains(t, metadataInfo, "idleCheckFrequency") - assert.Equal(t, metadataInfo["redisHost"], "string") - assert.Equal(t, metadataInfo["idleCheckFrequency"], "redis.Duration") } func setupMiniredis() (*miniredis.Miniredis, rediscomponent.RedisClient) { diff --git a/state/rethinkdb/rethinkdb.go b/state/rethinkdb/rethinkdb.go index f506920a5d..b725ea17bc 100644 --- a/state/rethinkdb/rethinkdb.go +++ b/state/rethinkdb/rethinkdb.go @@ -306,9 +306,8 @@ func metadataToConfig(cfg map[string]string, logger logger.Logger) (*stateConfig return &c, nil } -func (s *RethinkDB) GetComponentMetadata() map[string]string { +func (s *RethinkDB) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := stateConfig{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/state/sqlite/sqlite.go b/state/sqlite/sqlite.go index fcb5f28635..218c16508f 100644 --- a/state/sqlite/sqlite.go +++ b/state/sqlite/sqlite.go @@ -58,11 +58,10 @@ func (s *SQLiteStore) Init(ctx context.Context, metadata state.Metadata) error { return s.dbaccess.Init(ctx, metadata) } -func (s SQLiteStore) GetComponentMetadata() map[string]string { +func (s SQLiteStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := sqliteMetadataStruct{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } // Features returns the features available in this state store. diff --git a/state/sqlserver/sqlserver.go b/state/sqlserver/sqlserver.go index 89806f3ee9..c87cafa491 100644 --- a/state/sqlserver/sqlserver.go +++ b/state/sqlserver/sqlserver.go @@ -20,9 +20,11 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "time" internalsql "github.com/dapr/components-contrib/internal/component/sql" + "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/state" "github.com/dapr/components-contrib/state/utils" "github.com/dapr/kit/logger" @@ -347,8 +349,10 @@ func (s *SQLServer) executeSet(ctx context.Context, db dbExecutor, req *state.Se return nil } -func (s *SQLServer) GetComponentMetadata() map[string]string { - return map[string]string{} +func (s *SQLServer) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { + settingsStruct := sqlServerMetadata{} + metadata.GetMetadataInfoFromStructType(reflect.TypeOf(settingsStruct), &metadataInfo, metadata.StateStoreType) + return } // Close implements io.Closer. diff --git a/state/store.go b/state/store.go index 9d23b50750..fa86851bd7 100644 --- a/state/store.go +++ b/state/store.go @@ -18,10 +18,13 @@ import ( "errors" "github.com/dapr/components-contrib/health" + "github.com/dapr/components-contrib/metadata" ) // Store is an interface to perform operations on store. type Store interface { + metadata.ComponentWithMetadata + BaseStore BulkStore } @@ -33,7 +36,6 @@ type BaseStore interface { Delete(ctx context.Context, req *DeleteRequest) error Get(ctx context.Context, req *GetRequest) (*GetResponse, error) Set(ctx context.Context, req *SetRequest) error - GetComponentMetadata() map[string]string } // TransactionalStore is an interface for initialization and support multiple transactional requests. diff --git a/state/zookeeper/zk.go b/state/zookeeper/zk.go index b4c92e2871..fe1985b941 100644 --- a/state/zookeeper/zk.go +++ b/state/zookeeper/zk.go @@ -303,9 +303,8 @@ func (s *StateStore) marshalData(v interface{}) ([]byte, error) { return jsoniter.ConfigFastest.Marshal(v) } -func (s *StateStore) GetComponentMetadata() map[string]string { +func (s *StateStore) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := properties{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) - return metadataInfo + return } diff --git a/workflows/temporal/temporal.go b/workflows/temporal/temporal.go index 0b528c04c0..9b28bd555c 100644 --- a/workflows/temporal/temporal.go +++ b/workflows/temporal/temporal.go @@ -172,11 +172,10 @@ func (c *TemporalWF) parseMetadata(meta workflows.Metadata) (*temporalMetadata, return &m, err } -func (c *TemporalWF) GetComponentMetadata() map[string]string { +func (c *TemporalWF) GetComponentMetadata() (metadataInfo metadata.MetadataMap) { metadataStruct := temporalMetadata{} - metadataInfo := map[string]string{} metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.WorkflowType) - return metadataInfo + return } func lookupStatus(status enums.WorkflowExecutionStatus) string { diff --git a/workflows/workflow.go b/workflows/workflow.go index 1dee2ba398..93a4591d79 100644 --- a/workflows/workflow.go +++ b/workflows/workflow.go @@ -30,5 +30,4 @@ type Workflow interface { Purge(ctx context.Context, req *PurgeRequest) error Pause(ctx context.Context, req *PauseRequest) error Resume(ctx context.Context, req *ResumeRequest) error - GetComponentMetadata() map[string]string }