Merge branch 'master' into comp-md-secretstores.aws.parameterstore-2582

.build-tools/builtin-authentication-profiles.yaml

@@ -13,6 +13,14 @@ aws:
         required: true
         sensitive: true
         example: '"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"'
+      - name: sessionToken
+        required: false
+        sensitive: true
+        description: |
+          AWS session token to use. A session token is only required if you are using
+          temporary security credentials.
+        example: '"TOKEN"'
+        type: string
   - title: "AWS: Credentials from Environment Variables"
     description: Use AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY from the environment
 

.build-tools/pkg/metadataschema/validators.go

@@ -153,6 +153,46 @@ func (c *ComponentMetadata) AppendBuiltin() error {
 					URL:   "https://docs.dapr.io/developing-applications/building-blocks/pubsub/howto-namespace/",
 				},
 			},
+			Metadata{
+				Name:        "allowedTopics",
+				Type:        "string",
+				Description: "A comma-separated list of allowed topics for all applications. If empty (default) apps can publish and subscribe to all topics, notwithstanding `publishingScopes` and `subscriptionScopes`.",
+				Example:     `"app1=topic1;app2=topic2,topic3"`,
+				URL: &URL{
+					Title: "Documentation",
+					URL:   "https://docs.dapr.io/developing-applications/building-blocks/pubsub/pubsub-scopes/",
+				},
+			},
+			Metadata{
+				Name:        "publishingScopes",
+				Type:        "string",
+				Description: "A semicolon-separated list of applications and comma-separated topic lists, allowing that app to publish to that list of topics. If empty (default), apps can publish to all topics.",
+				Example:     `"app1=topic1;app2=topic2,topic3;app3="`,
+				URL: &URL{
+					Title: "Documentation",
+					URL:   "https://docs.dapr.io/developing-applications/building-blocks/pubsub/pubsub-scopes/",
+				},
+			},
+			Metadata{
+				Name:        "subscriptionScopes",
+				Type:        "string",
+				Description: "A semicolon-separated list of applications and comma-separated topic lists, allowing that app to subscribe to that list of topics. If empty (default), apps can subscribe to all topics.",
+				Example:     `"app1=topic1;app2=topic2,topic3"`,
+				URL: &URL{
+					Title: "Documentation",
+					URL:   "https://docs.dapr.io/developing-applications/building-blocks/pubsub/pubsub-scopes/",
+				},
+			},
+			Metadata{
+				Name:        "protectedTopics",
+				Type:        "string",
+				Description: `A comma-separated list of topics marked as "protected" for all applications. If a topic is marked as protected then an application must be explicitly granted publish or subscribe permissions through 'publishingScopes' or 'subscriptionScopes' to publish or subscribe to it.`,
+				Example:     `"topic1,topic2"`,
+				URL: &URL{
+					Title: "Documentation",
+					URL:   "https://docs.dapr.io/developing-applications/building-blocks/pubsub/pubsub-scopes/",
+				},
+			},
 		)
 	case mdutils.BindingType:
 		if c.Binding != nil {
@@ -176,7 +216,7 @@ func (c *ComponentMetadata) AppendBuiltin() error {
 						Name:        bindingDirectionMetadataKey,
 						Type:        "string",
 						Description: "Indicates the direction of the binding component.",
-						Example:     `"`+direction+`"`,
+						Example:     `"` + direction + `"`,
 						URL: &URL{
 							Title: "Documentation",
 							URL:   "https://docs.dapr.io/reference/api/bindings_api/#binding-direction-optional",

bindings/aws/s3/metadata.yaml

@@ -0,0 +1,78 @@
+# yaml-language-server: $schema=../../../component-metadata-schema.json
+schemaVersion: v1
+type: bindings
+name: aws.s3
+version: v1
+status: stable
+title: "AWS S3"
+urls:
+  - title: Reference
+    url: https://docs.dapr.io/reference/components-reference/supported-bindings/s3/
+binding:
+  output: true
+  operations:
+    - name: create
+      description: "Create blob"
+    - name: get
+      description: "Get blob"
+    - name: delete
+      description: "Delete blob"
+    - name: list
+      description: "List blob"
+capabilities: []
+builtinAuthenticationProfiles:
+  - name: "aws"
+metadata:
+  - name: bucket
+    required: true
+    description: |
+      The name of the S3 bucket to write to.
+    example: '"bucket"'
+    type: string
+  - name: region
+    required: true
+    description: |
+      The specific AWS region where the S3 bucket is located.
+    example: '"us-east-1"'
+    type: string
+  - name: endpoint
+    required: false
+    description: |
+      AWS endpoint for the component to use, to connect to S3-compatible services or emulators.
+      Do not use this when running against production AWS.
+    example: '"http://localhost:4566"'
+    type: string
+  - name: forcePathStyle
+    description: |
+      Currently Amazon S3 SDK supports virtual-hosted-style and path-style access.
+      When false (the default), uses virtual-hosted-style format, i.e.: `https://<your bucket>.<endpoint>/<key>`. 
+      When true, uses path-style format, i.e.: `https://<endpoint>/<your bucket>/<key>`.
+    type: bool
+    default: 'false'
+    example: '"true", "false"'
+  - name: decodeBase64
+    description: |
+      Configuration to decode base64 file content before saving to bucket storage.
+      (In case of saving a file with binary content).
+    type: bool
+    default: 'false'
+    example: '"true", "false"'
+  - name: encodeBase64
+    description: |
+      Configuration to encode base64 file content before returning the content.
+      (In case of opening a file with binary content).
+    type: bool
+    default: 'false'
+    example: '"true", "false"'
+  - name: disableSSL
+    description: |
+      Allows to connect to non-`https://` endpoints. 
+    type: bool
+    default: 'false'
+    example: '"true", "false"'
+  - name: insecureSSL
+    description: |
+      When connecting to `https://` endpoints, accepts self-signed or invalid certificates.
+    type: bool
+    default: 'false'
+    example: '"true", "false"'

bindings/aws/s3/s3.go

@@ -63,20 +63,20 @@ type AWSS3 struct {
 
 type s3Metadata struct {
 	// Ignored by metadata parser because included in built-in authentication profile
-	AccessKey string `json:"accessKey" mapstructure:"accessKey" mdignore:"true"`
-	SecretKey string `json:"secretKey" mapstructure:"secretKey" mdignore:"true"`
+	AccessKey    string `json:"accessKey" mapstructure:"accessKey" mdignore:"true"`
+	SecretKey    string `json:"secretKey" mapstructure:"secretKey" mdignore:"true"`
+	SessionToken string `json:"sessionToken" mapstructure:"sessionToken" mdignore:"true"`
 
 	Region         string `json:"region" mapstructure:"region"`
 	Endpoint       string `json:"endpoint" mapstructure:"endpoint"`
-	SessionToken   string `json:"sessionToken" mapstructure:"sessionToken"`
 	Bucket         string `json:"bucket" mapstructure:"bucket"`
 	DecodeBase64   bool   `json:"decodeBase64,string" mapstructure:"decodeBase64"`
 	EncodeBase64   bool   `json:"encodeBase64,string" mapstructure:"encodeBase64"`
 	ForcePathStyle bool   `json:"forcePathStyle,string" mapstructure:"forcePathStyle"`
 	DisableSSL     bool   `json:"disableSSL,string" mapstructure:"disableSSL"`
 	InsecureSSL    bool   `json:"insecureSSL,string" mapstructure:"insecureSSL"`
-	FilePath       string `mapstructure:"filePath"`
-	PresignTTL     string `mapstructure:"presignTTL"`
+	FilePath       string `json:"filePath" mapstructure:"filePath"   mdignore:"true"`
+	PresignTTL     string `json:"presignTTL" mapstructure:"presignTTL"  mdignore:"true"`
 }
 
 type createResponse struct {

bindings/azure/openai/metadata.yaml

@@ -31,8 +31,4 @@ metadata:
   - name: endpoint
     required: true
     description: "Endpoint of the Azure OpenAI service"
-    example: '"https://myopenai.openai.azure.com"'
-  - name: deploymentID
-    required: true
-    description: "ID of the model deployment in the Azure OpenAI service"
-    example: '"my-model"'
+    example: '"https://myopenai.openai.azure.com"'

bindings/azure/openai/openai.go

@@ -33,6 +33,7 @@ import (
 const (
 	CompletionOperation     bindings.OperationKind = "completion"
 	ChatCompletionOperation bindings.OperationKind = "chat-completion"
+	GetEmbeddingOperation   bindings.OperationKind = "get-embedding"
 
 	APIKey           = "apiKey"
 	DeploymentID     = "deploymentID"
@@ -50,22 +51,20 @@ const (
 
 // AzOpenAI represents OpenAI output binding.
 type AzOpenAI struct {
-	logger       logger.Logger
-	client       *azopenai.Client
-	deploymentID string
+	logger logger.Logger
+	client *azopenai.Client
 }
 
 type openAIMetadata struct {
 	// APIKey is the API key for the Azure OpenAI API.
 	APIKey string `mapstructure:"apiKey"`
-	// DeploymentID is the deployment ID for the Azure OpenAI API.
-	DeploymentID string `mapstructure:"deploymentID"`
 	// Endpoint is the endpoint for the Azure OpenAI API.
 	Endpoint string `mapstructure:"endpoint"`
 }
 
 // ChatMessages type for chat completion API.
 type ChatMessages struct {
+	DeploymentID     string    `json:"deploymentID"`
 	Messages         []Message `json:"messages"`
 	Temperature      float32   `json:"temperature"`
 	MaxTokens        int32     `json:"maxTokens"`
@@ -84,6 +83,7 @@ type Message struct {
 
 // Prompt type for completion API.
 type Prompt struct {
+	DeploymentID     string   `json:"deploymentID"`
 	Prompt           string   `json:"prompt"`
 	Temperature      float32  `json:"temperature"`
 	MaxTokens        int32    `json:"maxTokens"`
@@ -94,6 +94,11 @@ type Prompt struct {
 	Stop             []string `json:"stop"`
 }
 
+type EmbeddingMessage struct {
+	DeploymentID string `json:"deploymentID"`
+	Message      string `json:"message"`
+}
+
 // NewOpenAI returns a new OpenAI output binding.
 func NewOpenAI(logger logger.Logger) bindings.OutputBinding {
 	return &AzOpenAI{
@@ -111,9 +116,6 @@ func (p *AzOpenAI) Init(ctx context.Context, meta bindings.Metadata) error {
 	if m.Endpoint == "" {
 		return fmt.Errorf("required metadata not set: %s", Endpoint)
 	}
-	if m.DeploymentID == "" {
-		return fmt.Errorf("required metadata not set: %s", DeploymentID)
-	}
 
 	if m.APIKey != "" {
 		// use API key authentication
@@ -144,7 +146,6 @@ func (p *AzOpenAI) Init(ctx context.Context, meta bindings.Metadata) error {
 			return fmt.Errorf("error creating Azure OpenAI client: %w", err)
 		}
 	}
-	p.deploymentID = m.DeploymentID
 
 	return nil
 }
@@ -154,6 +155,7 @@ func (p *AzOpenAI) Operations() []bindings.OperationKind {
 	return []bindings.OperationKind{
 		ChatCompletionOperation,
 		CompletionOperation,
+		GetEmbeddingOperation,
 	}
 }
 
@@ -188,6 +190,14 @@ func (p *AzOpenAI) Invoke(ctx context.Context, req *bindings.InvokeRequest) (res
 		responseAsBytes, _ := json.Marshal(response)
 		resp.Data = responseAsBytes
 
+	case GetEmbeddingOperation:
+		response, err := p.getEmbedding(ctx, req.Data, req.Metadata)
+		if err != nil {
+			return nil, fmt.Errorf("error performing get embedding operation: %w", err)
+		}
+		responseAsBytes, _ := json.Marshal(response)
+		resp.Data = responseAsBytes
+
 	default:
 		return nil, fmt.Errorf(
 			"invalid operation type: %s. Expected %s, %s",
@@ -220,12 +230,16 @@ func (p *AzOpenAI) completion(ctx context.Context, message []byte, metadata map[
 		return nil, fmt.Errorf("prompt is required for completion operation")
 	}
 
+	if prompt.DeploymentID == "" {
+		return nil, fmt.Errorf("required metadata not set: %s", DeploymentID)
+	}
+
 	if len(prompt.Stop) == 0 {
 		prompt.Stop = nil
 	}
 
 	resp, err := p.client.GetCompletions(ctx, azopenai.CompletionsOptions{
-		DeploymentID: p.deploymentID,
+		DeploymentID: prompt.DeploymentID,
 		Prompt:       []string{prompt.Prompt},
 		MaxTokens:    &prompt.MaxTokens,
 		Temperature:  &prompt.Temperature,
@@ -268,6 +282,10 @@ func (p *AzOpenAI) chatCompletion(ctx context.Context, messageRequest []byte, me
 		return nil, fmt.Errorf("messages are required for chat-completion operation")
 	}
 
+	if messages.DeploymentID == "" {
+		return nil, fmt.Errorf("required metadata not set: %s", DeploymentID)
+	}
+
 	if len(messages.Stop) == 0 {
 		messages.Stop = nil
 	}
@@ -286,7 +304,7 @@ func (p *AzOpenAI) chatCompletion(ctx context.Context, messageRequest []byte, me
 	}
 
 	res, err := p.client.GetChatCompletions(ctx, azopenai.ChatCompletionsOptions{
-		DeploymentID: p.deploymentID,
+		DeploymentID: messages.DeploymentID,
 		MaxTokens:    maxTokens,
 		Temperature:  &messages.Temperature,
 		TopP:         &messages.TopP,
@@ -312,6 +330,34 @@ func (p *AzOpenAI) chatCompletion(ctx context.Context, messageRequest []byte, me
 	return response, nil
 }
 
+func (p *AzOpenAI) getEmbedding(ctx context.Context, messageRequest []byte, metadata map[string]string) (response []float32, err error) {
+	message := EmbeddingMessage{}
+	err = json.Unmarshal(messageRequest, &message)
+	if err != nil {
+		return nil, fmt.Errorf("error unmarshalling the input object: %w", err)
+	}
+
+	if message.DeploymentID == "" {
+		return nil, fmt.Errorf("required metadata not set: %s", DeploymentID)
+	}
+
+	res, err := p.client.GetEmbeddings(ctx, azopenai.EmbeddingsOptions{
+		DeploymentID: message.DeploymentID,
+		Input:        []string{message.Message},
+	}, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error getting embedding api: %w", err)
+	}
+
+	// No embedding returned.
+	if len(res.Data) == 0 {
+		return []float32{}, nil
+	}
+
+	response = res.Data[0].Embedding
+	return response, nil
+}
+
 // Close Az OpenAI instance.
 func (p *AzOpenAI) Close() error {
 	p.client = nil