diff --git a/.build-tools/builtin-authentication-profiles.yaml b/.build-tools/builtin-authentication-profiles.yaml index 548fbf5352..0f545d356f 100644 --- a/.build-tools/builtin-authentication-profiles.yaml +++ b/.build-tools/builtin-authentication-profiles.yaml @@ -103,3 +103,72 @@ azuread: - AzurePublicCloud - AzureChinaCloud - AzureUSGovernmentCloud + +gcp: + - title: "GCP API Authentication with Service Account Key" + description: | + Authenticate authenticates API calls with the given service account or refresh token JSON credentials. + metadata: + - name: privateKeyID + required: true + sensitive: true + description: | + The GCP private key id. Replace with the value of "private_key_id" field of the Service Account Key file. + example: '"privateKeyID"' + - name: privateKey + required: true + sensitive: true + description: | + The GCP credentials private key. Replace with the value of "private_key" field of the Service Account Key file. + example: '"-----BEGIN PRIVATE KEY-----\nMIIE...\\n-----END PRIVATE KEY-----\n"' + - name: type + type: string + required: false + description: | + The GCP credentials type. + example: '"service_account"' + allowedValues: + - service_account + - name: projectID + type: string + required: true + description: | + GCP project id. + example: '"projectID"' + - name: clientEmail + type: string + required: true + description: | + GCP client email. + example: '"client@email.com"' + - name: clientID + type: string + required: true + description: | + The GCP client ID. + example: '"0123456789-0123456789"' + - name: authURI + type: string + required: false + description: | + The GCP account OAuth2 authorization server endpoint URI. + example: '"https://accounts.google.com/o/oauth2/auth"' + - name: tokenURI + type: string + required: false + description: | + The GCP account token server endpoint URI. + example: '"https://oauth2.googleapis.com/token"' + - name: authProviderX509CertURL + type: string + required: false + description: | + The GCP URL of the public x509 certificate, used to verify the signature + on JWTs, such as ID tokens, signed by the authentication provider. + example: '"https://www.googleapis.com/oauth2/v1/certs"' + - name: clientX509CertURL + type: string + required: false + description: | + The GCP URL of the public x509 certificate, used to verify JWTs signed by the client. + example: '"https://www.googleapis.com/robot/v1/metadata/x509/.iam.gserviceaccount.com"' diff --git a/.build-tools/go.mod b/.build-tools/go.mod index 710651c41e..ad9f1a5798 100644 --- a/.build-tools/go.mod +++ b/.build-tools/go.mod @@ -7,7 +7,7 @@ require ( github.com/invopop/jsonschema v0.6.0 github.com/spf13/cobra v1.6.1 github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5 - golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb + golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b gopkg.in/yaml.v3 v3.0.1 sigs.k8s.io/yaml v1.3.0 ) diff --git a/.build-tools/go.sum b/.build-tools/go.sum index f03aa9a1bb..a86c9d6df6 100644 --- a/.build-tools/go.sum +++ b/.build-tools/go.sum @@ -37,8 +37,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5 h1:ImnGIsrcG8vwbovhYvvSY8fagVV6QhCWSWXfzwGDLVs= github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/.github/infrastructure/terraform/conformance/state/aws/dynamodb/dynamodb.tf b/.github/infrastructure/terraform/conformance/state/aws/dynamodb/dynamodb.tf index cfd0660139..350a0cd319 100644 --- a/.github/infrastructure/terraform/conformance/state/aws/dynamodb/dynamodb.tf +++ b/.github/infrastructure/terraform/conformance/state/aws/dynamodb/dynamodb.tf @@ -34,6 +34,10 @@ resource "aws_dynamodb_table" "conformance_test_basic_table" { billing_mode = "PROVISIONED" read_capacity = "10" write_capacity = "10" + ttl { + attribute_name = "expiresAt" + enabled = true + } attribute { name = "key" type = "S" diff --git a/.github/workflows/certification.yml b/.github/workflows/certification.yml index dfa2fe39ba..5482b986a1 100644 --- a/.github/workflows/certification.yml +++ b/.github/workflows/certification.yml @@ -258,7 +258,7 @@ jobs: set +e gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json \ --junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.xml --format standard-quiet -- \ - -coverprofile=cover.out -covermode=set -tags=certtests -coverpkg=${{ matrix.source-pkg }} + -coverprofile=cover.out -covermode=set -tags=certtests -timeout=30m -coverpkg=${{ matrix.source-pkg }} status=$? echo "Completed certification tests for ${{ matrix.component }} ... " if test $status -ne 0; then diff --git a/.golangci.yml b/.golangci.yml index 73e3b28b38..c72308bc64 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -124,6 +124,8 @@ linters-settings: - "golang.org/x/net/context": "must use context" - "github.com/pkg/errors": "must use standard library (errors package and/or fmt.Errorf)" - "github.com/Sirupsen/logrus": "must use github.com/dapr/kit/logger" + - "github.com/labstack/gommon/log": "must use github.com/dapr/kit/logger" + - "github.com/gobuffalo/logger": "must use github.com/dapr/kit/logger" - "github.com/agrea/ptr": "must use github.com/dapr/kit/ptr" - "github.com/cenkalti/backoff$": "must use github.com/cenkalti/backoff/v4" - "github.com/cenkalti/backoff/v2": "must use github.com/cenkalti/backoff/v4" @@ -133,6 +135,14 @@ linters-settings: - "github.com/golang-jwt/jwt/v4": "must use github.com/lestrrat-go/jwx/v2" - "github.com/lestrrat-go/jwx/jwa": "must use github.com/lestrrat-go/jwx/v2" - "github.com/lestrrat-go/jwx/jwt": "must use github.com/lestrrat-go/jwx/v2" + - "github.com/lestrrat-go/jwx/jws": "must use github.com/lestrrat-go/jwx/v2" + - "github.com/gogo/status": "must use google.golang.org/grpc/status" + - "github.com/gogo/protobuf": "must use google.golang.org/protobuf" + - "k8s.io/utils/pointer": "must use github.com/dapr/kit/ptr" + - "k8s.io/utils/ptr": "must use github.com/dapr/kit/ptr" + - "github.com/ghodss/yaml": "must use sigs.k8s.io/yaml" + - "gopkg.in/yaml.v2": "must use gopkg.in/yaml.v3" + - "github.com/go-chi/chi$": "must use github.com/go-chi/chi/v5" misspell: # Correct spellings using locale preferences for US or UK. # Default is to use a neutral variety of English. diff --git a/bindings/azure/openai/metadata.yaml b/bindings/azure/openai/metadata.yaml index f579916940..7822f5ec05 100644 --- a/bindings/azure/openai/metadata.yaml +++ b/bindings/azure/openai/metadata.yaml @@ -31,8 +31,4 @@ metadata: - name: endpoint required: true description: "Endpoint of the Azure OpenAI service" - example: '"https://myopenai.openai.azure.com"' - - name: deploymentID - required: true - description: "ID of the model deployment in the Azure OpenAI service" - example: '"my-model"' + example: '"https://myopenai.openai.azure.com"' \ No newline at end of file diff --git a/bindings/azure/openai/openai.go b/bindings/azure/openai/openai.go index 81743b5c81..bb14293c6e 100644 --- a/bindings/azure/openai/openai.go +++ b/bindings/azure/openai/openai.go @@ -20,8 +20,8 @@ import ( "reflect" "time" + "github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai" "github.com/dapr/components-contrib/bindings" azauth "github.com/dapr/components-contrib/internal/authentication/azure" @@ -33,6 +33,7 @@ import ( const ( CompletionOperation bindings.OperationKind = "completion" ChatCompletionOperation bindings.OperationKind = "chat-completion" + GetEmbeddingOperation bindings.OperationKind = "get-embedding" APIKey = "apiKey" DeploymentID = "deploymentID" @@ -50,22 +51,20 @@ const ( // AzOpenAI represents OpenAI output binding. type AzOpenAI struct { - logger logger.Logger - client *azopenai.Client - deploymentID string + logger logger.Logger + client *azopenai.Client } type openAIMetadata struct { // APIKey is the API key for the Azure OpenAI API. APIKey string `mapstructure:"apiKey"` - // DeploymentID is the deployment ID for the Azure OpenAI API. - DeploymentID string `mapstructure:"deploymentID"` // Endpoint is the endpoint for the Azure OpenAI API. Endpoint string `mapstructure:"endpoint"` } // ChatMessages type for chat completion API. type ChatMessages struct { + DeploymentID string `json:"deploymentID"` Messages []Message `json:"messages"` Temperature float32 `json:"temperature"` MaxTokens int32 `json:"maxTokens"` @@ -84,6 +83,7 @@ type Message struct { // Prompt type for completion API. type Prompt struct { + DeploymentID string `json:"deploymentID"` Prompt string `json:"prompt"` Temperature float32 `json:"temperature"` MaxTokens int32 `json:"maxTokens"` @@ -94,6 +94,11 @@ type Prompt struct { Stop []string `json:"stop"` } +type EmbeddingMessage struct { + DeploymentID string `json:"deploymentID"` + Message string `json:"message"` +} + // NewOpenAI returns a new OpenAI output binding. func NewOpenAI(logger logger.Logger) bindings.OutputBinding { return &AzOpenAI{ @@ -111,9 +116,6 @@ func (p *AzOpenAI) Init(ctx context.Context, meta bindings.Metadata) error { if m.Endpoint == "" { return fmt.Errorf("required metadata not set: %s", Endpoint) } - if m.DeploymentID == "" { - return fmt.Errorf("required metadata not set: %s", DeploymentID) - } if m.APIKey != "" { // use API key authentication @@ -144,7 +146,6 @@ func (p *AzOpenAI) Init(ctx context.Context, meta bindings.Metadata) error { return fmt.Errorf("error creating Azure OpenAI client: %w", err) } } - p.deploymentID = m.DeploymentID return nil } @@ -154,6 +155,7 @@ func (p *AzOpenAI) Operations() []bindings.OperationKind { return []bindings.OperationKind{ ChatCompletionOperation, CompletionOperation, + GetEmbeddingOperation, } } @@ -188,6 +190,14 @@ func (p *AzOpenAI) Invoke(ctx context.Context, req *bindings.InvokeRequest) (res responseAsBytes, _ := json.Marshal(response) resp.Data = responseAsBytes + case GetEmbeddingOperation: + response, err := p.getEmbedding(ctx, req.Data, req.Metadata) + if err != nil { + return nil, fmt.Errorf("error performing get embedding operation: %w", err) + } + responseAsBytes, _ := json.Marshal(response) + resp.Data = responseAsBytes + default: return nil, fmt.Errorf( "invalid operation type: %s. Expected %s, %s", @@ -220,12 +230,16 @@ func (p *AzOpenAI) completion(ctx context.Context, message []byte, metadata map[ return nil, fmt.Errorf("prompt is required for completion operation") } + if prompt.DeploymentID == "" { + return nil, fmt.Errorf("required metadata not set: %s", DeploymentID) + } + if len(prompt.Stop) == 0 { prompt.Stop = nil } resp, err := p.client.GetCompletions(ctx, azopenai.CompletionsOptions{ - DeploymentID: p.deploymentID, + DeploymentID: prompt.DeploymentID, Prompt: []string{prompt.Prompt}, MaxTokens: &prompt.MaxTokens, Temperature: &prompt.Temperature, @@ -268,6 +282,10 @@ func (p *AzOpenAI) chatCompletion(ctx context.Context, messageRequest []byte, me return nil, fmt.Errorf("messages are required for chat-completion operation") } + if messages.DeploymentID == "" { + return nil, fmt.Errorf("required metadata not set: %s", DeploymentID) + } + if len(messages.Stop) == 0 { messages.Stop = nil } @@ -286,7 +304,7 @@ func (p *AzOpenAI) chatCompletion(ctx context.Context, messageRequest []byte, me } res, err := p.client.GetChatCompletions(ctx, azopenai.ChatCompletionsOptions{ - DeploymentID: p.deploymentID, + DeploymentID: messages.DeploymentID, MaxTokens: maxTokens, Temperature: &messages.Temperature, TopP: &messages.TopP, @@ -312,6 +330,34 @@ func (p *AzOpenAI) chatCompletion(ctx context.Context, messageRequest []byte, me return response, nil } +func (p *AzOpenAI) getEmbedding(ctx context.Context, messageRequest []byte, metadata map[string]string) (response []float32, err error) { + message := EmbeddingMessage{} + err = json.Unmarshal(messageRequest, &message) + if err != nil { + return nil, fmt.Errorf("error unmarshalling the input object: %w", err) + } + + if message.DeploymentID == "" { + return nil, fmt.Errorf("required metadata not set: %s", DeploymentID) + } + + res, err := p.client.GetEmbeddings(ctx, azopenai.EmbeddingsOptions{ + DeploymentID: message.DeploymentID, + Input: []string{message.Message}, + }, nil) + if err != nil { + return nil, fmt.Errorf("error getting embedding api: %w", err) + } + + // No embedding returned. + if len(res.Data) == 0 { + return []float32{}, nil + } + + response = res.Data[0].Embedding + return response, nil +} + // Close Az OpenAI instance. func (p *AzOpenAI) Close() error { p.client = nil diff --git a/bindings/azure/storagequeues/storagequeues.go b/bindings/azure/storagequeues/storagequeues.go index babfb2feab..d637d16685 100644 --- a/bindings/azure/storagequeues/storagequeues.go +++ b/bindings/azure/storagequeues/storagequeues.go @@ -38,6 +38,12 @@ const ( defaultTTL = 10 * time.Minute defaultVisibilityTimeout = 30 * time.Second defaultPollingInterval = 10 * time.Second + dequeueCount = "dequeueCount" + insertionTime = "insertionTime" + expirationTime = "expirationTime" + nextVisibleTime = "nextVisibleTime" + popReceipt = "popReceipt" + messageID = "messageID" ) type consumer struct { @@ -177,9 +183,30 @@ func (d *AzureQueueHelper) Read(ctx context.Context, consumer *consumer) error { } } + metadata := make(map[string]string, 6) + + if res.Messages[0].MessageID != nil { + metadata[messageID] = *res.Messages[0].MessageID + } + if res.Messages[0].PopReceipt != nil { + metadata[popReceipt] = *res.Messages[0].PopReceipt + } + if res.Messages[0].InsertionTime != nil { + metadata[insertionTime] = res.Messages[0].InsertionTime.Format(time.RFC3339) + } + if res.Messages[0].ExpirationTime != nil { + metadata[expirationTime] = res.Messages[0].ExpirationTime.Format(time.RFC3339) + } + if res.Messages[0].TimeNextVisible != nil { + metadata[nextVisibleTime] = res.Messages[0].TimeNextVisible.Format(time.RFC3339) + } + if res.Messages[0].DequeueCount != nil { + metadata[dequeueCount] = strconv.FormatInt(*res.Messages[0].DequeueCount, 10) + } + _, err = consumer.callback(ctx, &bindings.ReadResponse{ Data: data, - Metadata: map[string]string{}, + Metadata: metadata, }) if err != nil { return err diff --git a/bindings/gcp/bucket/bucket.go b/bindings/gcp/bucket/bucket.go index 941b43ff4c..6507773c6c 100644 --- a/bindings/gcp/bucket/bucket.go +++ b/bindings/gcp/bucket/bucket.go @@ -54,19 +54,21 @@ type GCPStorage struct { } type gcpMetadata struct { - Bucket string `json:"bucket" mapstructure:"bucket"` - Type string `json:"type" mapstructure:"type"` - ProjectID string `json:"project_id" mapstructure:"project_id"` - PrivateKeyID string `json:"private_key_id" mapstructure:"private_key_id"` - PrivateKey string `json:"private_key" mapstructure:"private_key"` - ClientEmail string `json:"client_email " mapstructure:"client_email"` - ClientID string `json:"client_id" mapstructure:"client_id"` - AuthURI string `json:"auth_uri" mapstructure:"auth_uri"` - TokenURI string `json:"token_uri" mapstructure:"token_uri"` - AuthProviderCertURL string `json:"auth_provider_x509_cert_url" mapstructure:"auth_provider_x509_cert_url"` - ClientCertURL string `json:"client_x509_cert_url" mapstructure:"client_x509_cert_url"` - DecodeBase64 bool `json:"decodeBase64,string" mapstructure:"decodeBase64"` - EncodeBase64 bool `json:"encodeBase64,string" mapstructure:"encodeBase64"` + // Ignored by metadata parser because included in built-in authentication profile + Type string `json:"type" mapstructure:"type" mdignore:"true"` + ProjectID string `json:"project_id" mapstructure:"projectID" mdignore:"true" mapstructurealiases:"project_id"` + PrivateKeyID string `json:"private_key_id" mapstructure:"privateKeyID" mdignore:"true" mapstructurealiases:"private_key_id"` + PrivateKey string `json:"private_key" mapstructure:"privateKey" mdignore:"true" mapstructurealiases:"private_key"` + ClientEmail string `json:"client_email " mapstructure:"clientEmail" mdignore:"true" mapstructurealiases:"client_email"` + ClientID string `json:"client_id" mapstructure:"clientID" mdignore:"true" mapstructurealiases:"client_id"` + AuthURI string `json:"auth_uri" mapstructure:"authURI" mdignore:"true" mapstructurealiases:"auth_uri"` + TokenURI string `json:"token_uri" mapstructure:"tokenURI" mdignore:"true" mapstructurealiases:"token_uri"` + AuthProviderCertURL string `json:"auth_provider_x509_cert_url" mapstructure:"authProviderX509CertURL" mdignore:"true" mapstructurealiases:"auth_provider_x509_cert_url"` + ClientCertURL string `json:"client_x509_cert_url" mapstructure:"clientX509CertURL" mdignore:"true" mapstructurealiases:"client_x509_cert_url"` + + Bucket string `json:"bucket" mapstructure:"bucket"` + DecodeBase64 bool `json:"decodeBase64,string" mapstructure:"decodeBase64"` + EncodeBase64 bool `json:"encodeBase64,string" mapstructure:"encodeBase64"` } type listPayload struct { diff --git a/bindings/gcp/bucket/bucket_test.go b/bindings/gcp/bucket/bucket_test.go index 9bd249e87e..c1057c11bc 100644 --- a/bindings/gcp/bucket/bucket_test.go +++ b/bindings/gcp/bucket/bucket_test.go @@ -27,17 +27,17 @@ func TestParseMetadata(t *testing.T) { t.Run("Has correct metadata", func(t *testing.T) { m := bindings.Metadata{} m.Properties = map[string]string{ - "auth_provider_x509_cert_url": "my_auth_provider_x509", - "auth_uri": "my_auth_uri", - "Bucket": "my_bucket", - "client_x509_cert_url": "my_client_x509", - "client_email": "my_email@mail.dapr", - "client_id": "my_client_id", - "private_key": "my_private_key", - "private_key_id": "my_private_key_id", - "project_id": "my_project_id", - "token_uri": "my_token_uri", - "type": "my_type", + "authProviderX509CertURL": "my_auth_provider_x509", + "authURI": "my_auth_uri", + "Bucket": "my_bucket", + "clientX509CertURL": "my_client_x509", + "clientEmail": "my_email@mail.dapr", + "clientID": "my_client_id", + "privateKey": "my_private_key", + "privateKeyID": "my_private_key_id", + "projectID": "my_project_id", + "tokenURI": "my_token_uri", + "type": "my_type", } gs := GCPStorage{logger: logger.NewLogger("test")} meta, err := gs.parseMetadata(m) @@ -73,18 +73,18 @@ func TestMergeWithRequestMetadata(t *testing.T) { t.Run("Has merged metadata", func(t *testing.T) { m := bindings.Metadata{} m.Properties = map[string]string{ - "auth_provider_x509_cert_url": "my_auth_provider_x509", - "auth_uri": "my_auth_uri", - "Bucket": "my_bucket", - "client_x509_cert_url": "my_client_x509", - "client_email": "my_email@mail.dapr", - "client_id": "my_client_id", - "private_key": "my_private_key", - "private_key_id": "my_private_key_id", - "project_id": "my_project_id", - "token_uri": "my_token_uri", - "type": "my_type", - "decodeBase64": "false", + "authProviderX509CertURL": "my_auth_provider_x509", + "authURI": "my_auth_uri", + "Bucket": "my_bucket", + "clientX509CertURL": "my_client_x509", + "clientEmail": "my_email@mail.dapr", + "clientID": "my_client_id", + "privateKey": "my_private_key", + "privateKeyID": "my_private_key_id", + "projectID": "my_project_id", + "tokenURI": "my_token_uri", + "type": "my_type", + "decodeBase64": "false", } gs := GCPStorage{logger: logger.NewLogger("test")} meta, err := gs.parseMetadata(m) @@ -129,18 +129,18 @@ func TestMergeWithRequestMetadata(t *testing.T) { t.Run("Has invalid merged metadata decodeBase64", func(t *testing.T) { m := bindings.Metadata{} m.Properties = map[string]string{ - "auth_provider_x509_cert_url": "my_auth_provider_x509", - "auth_uri": "my_auth_uri", - "Bucket": "my_bucket", - "client_x509_cert_url": "my_client_x509", - "client_email": "my_email@mail.dapr", - "client_id": "my_client_id", - "private_key": "my_private_key", - "private_key_id": "my_private_key_id", - "project_id": "my_project_id", - "token_uri": "my_token_uri", - "type": "my_type", - "decodeBase64": "false", + "authProviderX509CertURL": "my_auth_provider_x509", + "authURI": "my_auth_uri", + "Bucket": "my_bucket", + "clientX509CertURL": "my_client_x509", + "clientEmail": "my_email@mail.dapr", + "clientID": "my_client_id", + "privateKey": "my_private_key", + "privateKeyID": "my_private_key_id", + "projectID": "my_project_id", + "tokenURI": "my_token_uri", + "type": "my_type", + "decodeBase64": "false", } gs := GCPStorage{logger: logger.NewLogger("test")} meta, err := gs.parseMetadata(m) @@ -173,19 +173,19 @@ func TestMergeWithRequestMetadata(t *testing.T) { t.Run("Has invalid merged metadata encodeBase64", func(t *testing.T) { m := bindings.Metadata{} m.Properties = map[string]string{ - "auth_provider_x509_cert_url": "my_auth_provider_x509", - "auth_uri": "my_auth_uri", - "Bucket": "my_bucket", - "client_x509_cert_url": "my_client_x509", - "client_email": "my_email@mail.dapr", - "client_id": "my_client_id", - "private_key": "my_private_key", - "private_key_id": "my_private_key_id", - "project_id": "my_project_id", - "token_uri": "my_token_uri", - "type": "my_type", - "decodeBase64": "false", - "encodeBase64": "true", + "authProviderX509CertURL": "my_auth_provider_x509", + "authURI": "my_auth_uri", + "Bucket": "my_bucket", + "clientX509CertURL": "my_client_x509", + "clientEmail": "my_email@mail.dapr", + "clientID": "my_client_id", + "privateKey": "my_private_key", + "privateKeyID": "my_private_key_id", + "projectID": "my_project_id", + "tokenURI": "my_token_uri", + "type": "my_type", + "decodeBase64": "false", + "encodeBase64": "true", } gs := GCPStorage{logger: logger.NewLogger("test")} meta, err := gs.parseMetadata(m) diff --git a/bindings/gcp/bucket/metadata.yaml b/bindings/gcp/bucket/metadata.yaml new file mode 100644 index 0000000000..e45a072a21 --- /dev/null +++ b/bindings/gcp/bucket/metadata.yaml @@ -0,0 +1,41 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: bindings +name: gcp.bucket +version: v1 +status: alpha +title: "GCP Storage Bucket" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-bindings/gcpbucket/ +binding: + output: true + operations: + - name: create + description: "Create an item." +capabilities: [] +builtinAuthenticationProfiles: + - name: "gcp" +metadata: + - name: bucket + required: true + description: | + The bucket name. + example: '"mybucket"' + type: string + - name: decodeBase64 + type: bool + required: false + default: 'false' + description: | + Configuration to decode base64 file content before saving to bucket storage. + (In case of opening a file with binary content). + example: '"true, false"' + - name: encodeBase64 + type: bool + required: false + default: 'false' + description: | + Configuration to encode base64 file content before return the content. + (In case of saving a file with binary content). + example: '"true, false"' \ No newline at end of file diff --git a/bindings/rabbitmq/rabbitmq.go b/bindings/rabbitmq/rabbitmq.go index e10f7422f3..4188851342 100644 --- a/bindings/rabbitmq/rabbitmq.go +++ b/bindings/rabbitmq/rabbitmq.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "math" + "net/url" "reflect" "strconv" "sync" @@ -464,8 +465,20 @@ func (r *RabbitMQ) handleMessage(ctx context.Context, handler bindings.Handler, r.logger.Info("Input binding channel closed") return } + + metadata := make(map[string]string, len(d.Headers)) + // Passthrough any custom metadata to the handler. + for k, v := range d.Headers { + if s, ok := v.(string); ok { + // Escape the key and value to ensure they are valid URL query parameters. + // This is necessary for them to be sent as HTTP Metadata. + metadata[url.QueryEscape(k)] = url.QueryEscape(s) + } + } + _, err := handler(ctx, &bindings.ReadResponse{ - Data: d.Body, + Data: d.Body, + Metadata: metadata, }) if err != nil { ch.Nack(d.DeliveryTag, false, true) diff --git a/go.mod b/go.mod index 3ad6bd2b6d..50a6d8b617 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,24 @@ module github.com/dapr/components-contrib go 1.20 require ( - cloud.google.com/go/datastore v1.12.1 - cloud.google.com/go/pubsub v1.32.0 + cloud.google.com/go/datastore v1.13.0 + cloud.google.com/go/pubsub v1.33.0 cloud.google.com/go/secretmanager v1.11.1 cloud.google.com/go/storage v1.31.0 dubbo.apache.org/dubbo-go/v3 v3.0.3-0.20230118042253-4f159a2b38f3 + github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai v0.1.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 - github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai v0.1.0 github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v0.5.0 github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5 - github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 + github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2 github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.0.1 github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventgrid/armeventgrid/v2 v2.1.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1.1 - github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v0.12.0 - github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0 github.com/Azure/go-amqp v1.0.1 github.com/DATA-DOG/go-sqlmock v1.5.0 @@ -39,7 +39,7 @@ require ( github.com/apache/pulsar-client-go v0.11.0 github.com/apache/rocketmq-client-go/v2 v2.1.2-0.20230412142645-25003f6f083d github.com/apache/thrift v0.13.0 - github.com/aws/aws-sdk-go v1.44.299 + github.com/aws/aws-sdk-go v1.44.315 github.com/benbjohnson/clock v1.3.5 github.com/bradfitz/gomemcache v0.0.0-20230611145640-acc696258285 github.com/camunda/zeebe/clients/go/v8 v8.2.8 @@ -56,7 +56,6 @@ require ( github.com/didip/tollbooth/v7 v7.0.1 github.com/eclipse/paho.mqtt.golang v1.4.3 github.com/fasthttp-contrib/sessions v0.0.0-20160905201309-74f6ac73d5d5 - github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-redis/redis/v8 v8.11.5 github.com/go-sql-driver/mysql v1.7.1 github.com/go-zookeeper/zk v1.0.3 @@ -82,7 +81,7 @@ require ( github.com/lestrrat-go/jwx/v2 v2.0.11 github.com/machinebox/graphql v0.2.2 github.com/matoous/go-nanoid/v2 v2.0.0 - github.com/microsoft/go-mssqldb v1.3.0 + github.com/microsoft/go-mssqldb v1.5.0 github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 github.com/mrz1836/postmark v1.4.0 github.com/nacos-group/nacos-sdk-go/v2 v2.2.2 @@ -105,7 +104,7 @@ require ( github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.608 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/ssm v1.0.608 github.com/tetratelabs/wazero v1.3.0 - github.com/valyala/fasthttp v1.47.0 + github.com/valyala/fasthttp v1.48.0 github.com/vmware/vmware-go-kcl v1.5.0 github.com/xdg-go/scram v1.1.2 go.etcd.io/etcd/client/v3 v3.5.9 @@ -115,9 +114,9 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/ratelimit v0.3.0 golang.org/x/crypto v0.11.0 - golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb + golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b golang.org/x/mod v0.12.0 - golang.org/x/net v0.12.0 + golang.org/x/net v0.13.0 golang.org/x/oauth2 v0.10.0 google.golang.org/api v0.128.0 google.golang.org/grpc v1.56.1 @@ -129,8 +128,9 @@ require ( k8s.io/apiextensions-apiserver v0.26.3 k8s.io/apimachinery v0.26.3 k8s.io/client-go v0.26.3 - k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b modernc.org/sqlite v1.24.0 + sigs.k8s.io/yaml v1.3.0 ) require ( @@ -204,6 +204,7 @@ require ( github.com/fatih/color v1.15.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gavv/httpexpect v2.0.0+incompatible // indirect + github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect github.com/go-kit/kit v0.10.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect @@ -403,7 +404,6 @@ require ( modernc.org/token v1.1.0 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect stathat.com/c/consistent v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 6d8fabcffb..c17e675d8b 100644 --- a/go.sum +++ b/go.sum @@ -155,8 +155,8 @@ cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= -cloud.google.com/go/datastore v1.12.1 h1:i8HMKsqg/Sl3ZlOTGl471Z8j2uKtbRDT9VXJUIVlMik= -cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.13.0 h1:ktbC66bOQB3HJPQe8qNI1/aiQ77PMu7hD4mzE6uxe3w= +cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= @@ -286,8 +286,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= -cloud.google.com/go/pubsub v1.32.0 h1:JOEkgEYBuUTHSyHS4TcqOFuWr+vD6qO/imsFqShUCp4= -cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= @@ -417,19 +417,19 @@ github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai v0.1.1 h1:CZwHAPNp2pS80XfUr4xlC1n2M1xsGZ1UfnDW4EzHZGA= +github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai v0.1.1/go.mod h1:zPJgGMjMheJJrYgrQ4W8NrNCWtWXAkjI3KWYFnTtwdA= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= -github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai v0.1.0 h1:lkflJSWI6jicmEBImjpliUOWCr1PdJO/GcZj3bWx19Q= -github.com/Azure/azure-sdk-for-go/sdk/cognitiveservices/azopenai v0.1.0/go.mod h1:NwVkXm5Ty88Xd7cx6b53fGNeGG3W3ZDXgOXBNHLUy84= github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v0.5.0 h1:OrKZybbyagpgJiREiIVzH5mV/z9oS4rXqdX7i31DSF0= github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v0.5.0/go.mod h1:p74+tP95m8830ypJk53L93+BEsjTKY4SKQ75J2NmS5U= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5 h1:qS0Bp4do0cIvnuQgSGeO6ZCu/q/HlRKl4NPfv1eJ2p0= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5/go.mod h1:Beh5cHIXJ0oWEDWk9lNFtuklCojLLQ5hl+LqSNTTs0I= -github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 h1:bFa9IcjvrCber6gGgDAUZ+I2bO8J7s8JxXmu9fhi2ss= -github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1/go.mod h1:l3wvZkG9oW07GLBW5Cd0WwG5asOfJ8aqE8raUvNzLpk= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2 h1:iXFUCl7NK2DPVKfixcYDPGj3uLV7yf5eolBsoWD8Sc4= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2/go.mod h1:E1WPwLx0wZyus7NBHjhrHE1QgWwKJPE81fnUbT+FxqI= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.0.1 h1:7G4EhZbWFwfgkNfJkNoZmFL8FfWT6P96YVwG71uhNxY= @@ -442,14 +442,15 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1.1/go.mod h1:7fQVOnRA11ScLE8dOCWanXHQa2NMFOM2i0u/1VRICXA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v0.12.0 h1:4Kynh6Hn2ekyIsBgNQJb3dn1+/MyvzfUJebti2emB/A= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v0.12.0/go.mod h1:Q28U+75mpCaSCDowNEmhIo/rmgdkqmkmzI7N6TGR4UY= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0 h1:upXr9dsOnTJk3eHQ3ldyvIXAIGggHtkrfrgbcas6DXU= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0/go.mod h1:w2K61Z8eppIuGbQRx1SKYld2Lrr5vrGvnUwWAhF4nso= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.0 h1:yfJe15aSwEQ6Oo6J+gdfdulPNoZ3TEhmbhLIoxZcA+U= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.0/go.mod h1:Q28U+75mpCaSCDowNEmhIo/rmgdkqmkmzI7N6TGR4UY= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0 h1:qvCB+Za4z8dtU3R5CC7zhlxTLlT3eaEMugglVvjUWtk= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0/go.mod h1:w2K61Z8eppIuGbQRx1SKYld2Lrr5vrGvnUwWAhF4nso= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 h1:T028gtTPiYt/RMUfs8nVsAL7FDQrfLlrm/NnRG/zcC4= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0/go.mod h1:cw4zVQgBby0Z5f2v0itn6se2dDP17nTjbZFXW5uPyHA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0 h1:lJwNFV+xYjHREUTHJKx/ZF6CJSt9znxmLw9DqSTvyRU= github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0/go.mod h1:GfT0aGew8Qj5yiQVqOO5v7N8fanbJGyUoHqXg56qcVY= github.com/Azure/go-amqp v1.0.1 h1:Jf8OQCKzRDMZ3pCiH4onM7yrhl5curkRSGkRLTyP35o= @@ -592,8 +593,8 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ github.com/aws/aws-sdk-go v1.19.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.44.299 h1:HVD9lU4CAFHGxleMJp95FV/sRhtg7P4miHD1v88JAQk= -github.com/aws/aws-sdk-go v1.44.299/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.315 h1:kYTC+Y/bJ9M7QQRvkI/LN5OWvhkIOL/YuFFRhS5QAOo= +github.com/aws/aws-sdk-go v1.44.315/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= @@ -1503,8 +1504,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/microsoft/go-mssqldb v1.3.0 h1:JcPVl+acL8Z/cQcJc9zP0OkjQ+l20bco/cCDpMbmGJk= -github.com/microsoft/go-mssqldb v1.3.0/go.mod h1:lmWsjHD8XX/Txr0f8ZqgbEZSC+BZjmEQy/Ms+rLrvho= +github.com/microsoft/go-mssqldb v1.5.0 h1:CgENxkwtOBNj3Jg6T1X209y2blCfTTcwuOlznd2k9fk= +github.com/microsoft/go-mssqldb v1.5.0/go.mod h1:lmWsjHD8XX/Txr0f8ZqgbEZSC+BZjmEQy/Ms+rLrvho= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -1944,8 +1945,8 @@ github.com/urfave/cli/v2 v2.11.0/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhA github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.21.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= -github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c= -github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= +github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= @@ -2152,8 +2153,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2289,8 +2290,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2980,8 +2981,8 @@ k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 h1:xMMXJlJbsU8w3V5N2FLDQ8YgU8s1EoULdbQBcAeNJkY= -k8s.io/utils v0.0.0-20230313181309-38a27ef9d749/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= diff --git a/internal/authentication/oauth2/clientcredentials.go b/internal/authentication/oauth2/clientcredentials.go new file mode 100644 index 0000000000..8122984839 --- /dev/null +++ b/internal/authentication/oauth2/clientcredentials.go @@ -0,0 +1,165 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oauth2 + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net/http" + "net/url" + "sync" + "time" + + "golang.org/x/oauth2" + ccreds "golang.org/x/oauth2/clientcredentials" + + "github.com/dapr/kit/logger" +) + +// ClientCredentialsMetadata is the metadata fields which can be used by a +// component to configure an OIDC client_credentials token source. +type ClientCredentialsMetadata struct { + TokenCAPEM string `mapstructure:"oauth2TokenCAPEM"` + TokenURL string `mapstructure:"oauth2TokenURL"` + ClientID string `mapstructure:"oauth2ClientID"` + ClientSecret string `mapstructure:"oauth2ClientSecret"` + Audiences []string `mapstructure:"oauth2Audiences"` + Scopes []string `mapstructure:"oauth2Scopes"` +} + +type ClientCredentialsOptions struct { + Logger logger.Logger + TokenURL string + ClientID string + ClientSecret string + Scopes []string + Audiences []string + CAPEM []byte +} + +// ClientCredentials is an OAuth2 Token Source that uses the client_credentials +// grant type to fetch a token. +type ClientCredentials struct { + log logger.Logger + currentToken *oauth2.Token + httpClient *http.Client + fetchTokenFn func(context.Context) (*oauth2.Token, error) + + lock sync.RWMutex +} + +func NewClientCredentials(ctx context.Context, opts ClientCredentialsOptions) (*ClientCredentials, error) { + conf, httpClient, err := opts.toConfig() + if err != nil { + return nil, err + } + + token, err := conf.Token(context.WithValue(ctx, oauth2.HTTPClient, httpClient)) + if err != nil { + return nil, fmt.Errorf("error fetching initial oauth2 client_credentials token: %w", err) + } + + opts.Logger.Info("Fetched initial oauth2 client_credentials token") + + return &ClientCredentials{ + log: opts.Logger, + currentToken: token, + httpClient: httpClient, + fetchTokenFn: conf.Token, + }, nil +} + +func (c *ClientCredentialsOptions) toConfig() (*ccreds.Config, *http.Client, error) { + if len(c.Scopes) == 0 { + return nil, nil, errors.New("oauth2 client_credentials token source requires at least one scope") + } + + if len(c.Audiences) == 0 { + return nil, nil, errors.New("oauth2 client_credentials token source requires at least one audience") + } + + _, err := url.Parse(c.TokenURL) + if err != nil { + return nil, nil, fmt.Errorf("error parsing token URL: %w", err) + } + + conf := &ccreds.Config{ + ClientID: c.ClientID, + ClientSecret: c.ClientSecret, + TokenURL: c.TokenURL, + Scopes: c.Scopes, + EndpointParams: url.Values{"audience": c.Audiences}, + } + + // If caPool is nil, then the Go TLS library will use the system's root CA. + var caPool *x509.CertPool + if len(c.CAPEM) > 0 { + caPool = x509.NewCertPool() + if !caPool.AppendCertsFromPEM(c.CAPEM) { + return nil, nil, errors.New("failed to parse CA PEM") + } + } + + return conf, &http.Client{ + Timeout: time.Second * 30, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: caPool, + }, + }, + }, nil +} + +func (c *ClientCredentials) Token() (string, error) { + c.lock.RLock() + defer c.lock.RUnlock() + + if !c.currentToken.Valid() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + if err := c.renewToken(ctx); err != nil { + return "", err + } + } + + return c.currentToken.AccessToken, nil +} + +func (c *ClientCredentials) renewToken(ctx context.Context) error { + c.lock.Lock() + defer c.lock.Unlock() + + // We need to check if the current token is valid because we might have lost + // the mutex lock race from the caller and we don't want to double-fetch a + // token unnecessarily! + if c.currentToken.Valid() { + return nil + } + + token, err := c.fetchTokenFn(context.WithValue(ctx, oauth2.HTTPClient, c.httpClient)) + if err != nil { + return err + } + + if !c.currentToken.Valid() { + return errors.New("oauth2 client_credentials token source returned an invalid token") + } + + c.currentToken = token + return nil +} diff --git a/internal/authentication/oauth2/clientcredentials_test.go b/internal/authentication/oauth2/clientcredentials_test.go new file mode 100644 index 0000000000..ba643b2129 --- /dev/null +++ b/internal/authentication/oauth2/clientcredentials_test.go @@ -0,0 +1,95 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oauth2 + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + ccreds "golang.org/x/oauth2/clientcredentials" +) + +func Test_toConfig(t *testing.T) { + tests := map[string]struct { + opts ClientCredentialsOptions + expConfig *ccreds.Config + expErr bool + }{ + "no scopes should error": { + opts: ClientCredentialsOptions{ + TokenURL: "https://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Audiences: []string{"audience"}, + }, + expErr: true, + }, + "bad URL endpoint should error": { + opts: ClientCredentialsOptions{ + TokenURL: "&&htp:/f url", + ClientID: "client-id", + ClientSecret: "client-secret", + Audiences: []string{"audience"}, + Scopes: []string{"foo"}, + }, + expErr: true, + }, + "bad CA certificate should error": { + opts: ClientCredentialsOptions{ + TokenURL: "http://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Audiences: []string{"audience"}, + Scopes: []string{"foo"}, + CAPEM: []byte("ca-pem"), + }, + expErr: true, + }, + "no audiences should error": { + opts: ClientCredentialsOptions{ + TokenURL: "http://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Scopes: []string{"foo"}, + }, + expErr: true, + }, + "should default scope": { + opts: ClientCredentialsOptions{ + TokenURL: "http://localhost:8080", + ClientID: "client-id", + ClientSecret: "client-secret", + Audiences: []string{"audience"}, + Scopes: []string{"foo", "bar"}, + }, + expConfig: &ccreds.Config{ + ClientID: "client-id", + ClientSecret: "client-secret", + TokenURL: "http://localhost:8080", + Scopes: []string{"foo", "bar"}, + EndpointParams: url.Values{"audience": []string{"audience"}}, + }, + expErr: false, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + config, _, err := test.opts.toConfig() + assert.Equalf(t, test.expErr, err != nil, "%v", err) + assert.Equal(t, test.expConfig, config) + }) + } +} diff --git a/internal/component/postgresql/postgresql.go b/internal/component/postgresql/postgresql.go index 76af6bb568..d7d0c3375f 100644 --- a/internal/component/postgresql/postgresql.go +++ b/internal/component/postgresql/postgresql.go @@ -72,7 +72,12 @@ func (p *PostgreSQL) Init(ctx context.Context, metadata state.Metadata) error { // Features returns the features available in this state store. func (p *PostgreSQL) Features() []state.Feature { - return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureQueryAPI} + return []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureQueryAPI, + state.FeatureTTL, + } } // Delete removes an entity from the store. diff --git a/internal/wasm/wasm_test.go b/internal/wasm/wasm_test.go index 57379e7c72..be0276ab30 100644 --- a/internal/wasm/wasm_test.go +++ b/internal/wasm/wasm_test.go @@ -172,9 +172,14 @@ func TestNewModuleConfig(t *testing.T) { maxDuration: 50 * time.Millisecond * 5, }, { - name: "strictSandbox = true", - metadata: &InitMetadata{StrictSandbox: true, Guest: binStrict}, - minDuration: 10 * time.Microsecond, + name: "strictSandbox = true", + metadata: &InitMetadata{StrictSandbox: true, Guest: binStrict}, + // In strict mode, nanosleep is implemented by an incrementing + // number. The resolution of the real clock timing the wasm + // invocation is lower resolution in Windows, so we can't verify a + // lower bound. In any case, the important part is that we aren't + // actually sleeping 50ms, which is what wasm thinks is happening. + minDuration: 0, maxDuration: 1 * time.Millisecond, }, } @@ -211,7 +216,8 @@ func TestNewModuleConfig(t *testing.T) { } else { require.NotEqual(t, deterministicOut, out.String()) } - require.True(t, duration > tc.minDuration && duration < tc.maxDuration, duration) + require.GreaterOrEqual(t, duration, tc.minDuration) + require.LessOrEqual(t, duration, tc.maxDuration) }) } } diff --git a/metadata/utils.go b/metadata/utils.go index 1c9248c9ab..44c3165473 100644 --- a/metadata/utils.go +++ b/metadata/utils.go @@ -23,6 +23,7 @@ import ( "time" "github.com/mitchellh/mapstructure" + "github.com/spf13/cast" "github.com/dapr/components-contrib/internal/utils" "github.com/dapr/kit/ptr" @@ -142,16 +143,27 @@ func GetMetadataProperty(props map[string]string, keys ...string) (val string, o // This is an extension of mitchellh/mapstructure which also supports decoding durations func DecodeMetadata(input any, result any) error { // avoids a common mistake of passing the metadata struct, instead of the properties map - // if input is of type struct, case it to metadata.Base and access the Properties instead + // if input is of type struct, cast it to metadata.Base and access the Properties instead v := reflect.ValueOf(input) if v.Kind() == reflect.Struct { f := v.FieldByName("Properties") if f.IsValid() && f.Kind() == reflect.Map { - properties := f.Interface().(map[string]string) - input = properties + input = f.Interface().(map[string]string) } } + inputMap, err := cast.ToStringMapStringE(input) + if err != nil { + return fmt.Errorf("input object cannot be cast to map[string]string: %w", err) + } + + // Handle aliases + err = resolveAliases(inputMap, result) + if err != nil { + return fmt.Errorf("failed to resolve aliases: %w", err) + } + + // Finally, decode the metadata using mapstructure decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ DecodeHook: mapstructure.ComposeDecodeHookFunc( toTimeDurationArrayHookFunc(), @@ -166,10 +178,77 @@ func DecodeMetadata(input any, result any) error { if err != nil { return err } - err = decoder.Decode(input) + err = decoder.Decode(inputMap) return err } +func resolveAliases(md map[string]string, result any) error { + // Get the list of all keys in the map + keys := make(map[string]string, len(md)) + for k := range md { + lk := strings.ToLower(k) + + // Check if there are duplicate keys after lowercasing + _, ok := keys[lk] + if ok { + return fmt.Errorf("key %s is duplicate in the metadata", lk) + } + + keys[lk] = k + } + + // Error if result is not pointer to struct, or pointer to pointer to struct + t := reflect.TypeOf(result) + if t.Kind() != reflect.Pointer { + return fmt.Errorf("not a pointer: %s", t.Kind().String()) + } + t = t.Elem() + if t.Kind() == reflect.Pointer { + t = t.Elem() + } + if t.Kind() != reflect.Struct { + return fmt.Errorf("not a struct: %s", t.Kind().String()) + } + + // Iterate through all the properties of result to see if anyone has the "mapstructurealiases" property + for i := 0; i < t.NumField(); i++ { + currentField := t.Field(i) + + // Ignored fields that are not exported or that don't have a "mapstructure" tag + mapstructureTag := currentField.Tag.Get("mapstructure") + if !currentField.IsExported() || mapstructureTag == "" { + continue + } + + // If the current property has a value in the metadata, then we don't need to handle aliases + _, ok := keys[strings.ToLower(mapstructureTag)] + if ok { + continue + } + + // Check if there's a "mapstructurealiases" tag + aliasesTag := strings.ToLower(currentField.Tag.Get("mapstructurealiases")) + if aliasesTag == "" { + continue + } + + // Look for the first alias that has a value + var mdKey string + for _, alias := range strings.Split(aliasesTag, ",") { + mdKey, ok = keys[alias] + if !ok { + continue + } + + // We found an alias + md[mapstructureTag] = md[mdKey] + break + } + } + + return nil +} + func toTruthyBoolHookFunc() mapstructure.DecodeHookFunc { return func( f reflect.Type, diff --git a/metadata/utils_test.go b/metadata/utils_test.go index 33a4af23c5..13cfc559a6 100644 --- a/metadata/utils_test.go +++ b/metadata/utils_test.go @@ -19,6 +19,8 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/exp/maps" ) func TestIsRawPayload(t *testing.T) { @@ -111,6 +113,9 @@ func TestMetadataDecode(t *testing.T) { MyRegularDurationDefaultValueUnset time.Duration `mapstructure:"myregulardurationdefaultvalueunset"` MyRegularDurationDefaultValueEmpty time.Duration `mapstructure:"myregulardurationdefaultvalueempty"` + + AliasedFieldA string `mapstructure:"aliasA1" mapstructurealiases:"aliasA2"` + AliasedFieldB string `mapstructure:"aliasB1" mapstructurealiases:"aliasB2"` } var m testMetadata @@ -131,6 +136,9 @@ func TestMetadataDecode(t *testing.T) { "mydurationarray": "1s,2s,3s,10", "mydurationarraypointer": "1s,10,2s,20,3s,30", "mydurationarraypointerempty": ",", + "aliasA2": "hello", + "aliasB1": "ciao", + "aliasB2": "bonjour", } err := DecodeMetadata(testData, &m) @@ -149,6 +157,8 @@ func TestMetadataDecode(t *testing.T) { assert.Equal(t, []time.Duration{time.Second, time.Second * 2, time.Second * 3, time.Second * 10}, m.MyDurationArray) assert.Equal(t, []time.Duration{time.Second, time.Second * 10, time.Second * 2, time.Second * 20, time.Second * 3, time.Second * 30}, *m.MyDurationArrayPointer) assert.Equal(t, []time.Duration{}, *m.MyDurationArrayPointerEmpty) + assert.Equal(t, "hello", m.AliasedFieldA) + assert.Equal(t, "ciao", m.AliasedFieldB) }) t.Run("Test metadata decode hook for truthy values", func(t *testing.T) { @@ -303,3 +313,172 @@ func TestMetadataStructToStringMap(t *testing.T) { assert.Empty(t, metadatainfo["ignored"].Aliases) }) } + +func TestResolveAliases(t *testing.T) { + tests := []struct { + name string + md map[string]string + result any + wantErr bool + wantMd map[string]string + }{ + { + name: "no aliases", + md: map[string]string{ + "hello": "world", + "ciao": "mondo", + }, + result: &struct { + Hello string `mapstructure:"hello"` + Ciao string `mapstructure:"ciao"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "world", + "ciao": "mondo", + }, + }, + { + name: "set with aliased field", + md: map[string]string{ + "ciao": "mondo", + }, + result: &struct { + Hello string `mapstructure:"hello" mapstructurealiases:"ciao"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "mondo", + "ciao": "mondo", + }, + }, + { + name: "do not overwrite existing fields with aliases", + md: map[string]string{ + "hello": "world", + "ciao": "mondo", + }, + result: &struct { + Hello string `mapstructure:"hello" mapstructurealiases:"ciao"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "world", + "ciao": "mondo", + }, + }, + { + name: "no fields with aliased value", + md: map[string]string{ + "bonjour": "monde", + }, + result: &struct { + Hello string `mapstructure:"hello" mapstructurealiases:"ciao"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "bonjour": "monde", + }, + }, + { + name: "multiple aliases", + md: map[string]string{ + "bonjour": "monde", + }, + result: &struct { + Hello string `mapstructure:"hello" mapstructurealiases:"ciao,bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "monde", + "bonjour": "monde", + }, + }, + { + name: "first alias wins", + md: map[string]string{ + "ciao": "mondo", + "bonjour": "monde", + }, + result: &struct { + Hello string `mapstructure:"hello" mapstructurealiases:"ciao,bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "mondo", + "ciao": "mondo", + "bonjour": "monde", + }, + }, + { + name: "no aliases with mixed case", + md: map[string]string{ + "hello": "world", + "CIAO": "mondo", + }, + result: &struct { + Hello string `mapstructure:"Hello"` + Ciao string `mapstructure:"ciao"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "hello": "world", + "CIAO": "mondo", + }, + }, + { + name: "set with aliased field with mixed case", + md: map[string]string{ + "ciao": "mondo", + }, + result: &struct { + Hello string `mapstructure:"Hello" mapstructurealiases:"CIAO"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "Hello": "mondo", + "ciao": "mondo", + }, + }, + { + name: "do not overwrite existing fields with aliases with mixed cases", + md: map[string]string{ + "HELLO": "world", + "CIAO": "mondo", + }, + result: &struct { + Hello string `mapstructure:"hELLo" mapstructurealiases:"cIAo"` + Bonjour string `mapstructure:"bonjour"` + }{}, + wantMd: map[string]string{ + "HELLO": "world", + "CIAO": "mondo", + }, + }, + { + name: "multiple aliases with mixed cases", + md: map[string]string{ + "bonjour": "monde", + }, + result: &struct { + Hello string `mapstructure:"HELLO" mapstructurealiases:"CIAO,BONJOUR"` + }{}, + wantMd: map[string]string{ + "HELLO": "monde", + "bonjour": "monde", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + md := maps.Clone(tt.md) + err := resolveAliases(md, tt.result) + + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + require.Equal(t, tt.wantMd, md) + }) + } +} diff --git a/middleware/http/bearer/bearer_middleware.go b/middleware/http/bearer/bearer_middleware.go index 030403716a..e3e7359e82 100644 --- a/middleware/http/bearer/bearer_middleware.go +++ b/middleware/http/bearer/bearer_middleware.go @@ -23,6 +23,7 @@ import ( "github.com/lestrrat-go/httprc" "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/lestrrat-go/jwx/v2/jws" "github.com/lestrrat-go/jwx/v2/jwt" "github.com/dapr/components-contrib/internal/httputils" @@ -112,7 +113,7 @@ func (m *Middleware) GetHandler(ctx context.Context, metadata middleware.Metadat _, err = jwt.Parse([]byte(rawToken), jwt.WithContext(r.Context()), jwt.WithAcceptableSkew(allowedClockSkew), - jwt.WithKeySet(keyset), + jwt.WithKeySet(keyset, jws.WithInferAlgorithmFromKey(true)), jwt.WithAudience(meta.Audience), jwt.WithIssuer(meta.Issuer), ) diff --git a/middleware/http/bearer/metadata.go b/middleware/http/bearer/metadata.go index 7bf831201f..54f45fa945 100644 --- a/middleware/http/bearer/metadata.go +++ b/middleware/http/bearer/metadata.go @@ -29,16 +29,12 @@ import ( type bearerMiddlewareMetadata struct { // Issuer authority. - Issuer string `json:"issuer" mapstructure:"issuer"` + Issuer string `json:"issuer" mapstructure:"issuer" mapstructurealiases:"issuerURL"` // Audience to expect in the token (usually, a client ID). - Audience string `json:"audience" mapstructure:"audience"` + Audience string `json:"audience" mapstructure:"audience" mapstructurealiases:"clientID"` // Optional address of the JKWS file. // If missing, will try to fetch the URL set in the OpenID Configuration document `/.well-known/openid-configuration`. JWKSURL string `json:"jwksURL" mapstructure:"jwksURL"` - // Deprecated - use "issuer" instead. - IssuerURL string `json:"issuerURL" mapstructure:"issuerURL"` - // Deprecated - use "audience" instead. - ClientID string `json:"clientID" mapstructure:"clientID"` // Internal properties logger logger.Logger `json:"-" mapstructure:"-"` @@ -52,18 +48,6 @@ func (md *bearerMiddlewareMetadata) fromMetadata(metadata middleware.Metadata) e return err } - // Support IssuerURL as deprecated alias for Issuer - if md.Issuer == "" && md.IssuerURL != "" { - md.Issuer = md.IssuerURL - md.logger.Warnf("Metadata property 'issuerURL' is deprecated and will be removed in the future. Please use 'issuer' instead.") - } - - // Support ClientID as deprecated alias for Audience - if md.Audience == "" && md.ClientID != "" { - md.Audience = md.ClientID - md.logger.Warnf("Metadata property 'clientID' is deprecated and will be removed in the future. Please use 'audience' instead.") - } - // Validate properties if md.Issuer == "" { return errors.New("metadata property 'issuer' is required") diff --git a/middleware/http/wasm/benchmark_test.go b/middleware/http/wasm/benchmark_test.go index c74d7c2938..57c54994f0 100644 --- a/middleware/http/wasm/benchmark_test.go +++ b/middleware/http/wasm/benchmark_test.go @@ -1,3 +1,16 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package wasm import ( @@ -28,8 +41,8 @@ func BenchmarkNative(b *testing.B) { } func BenchmarkTinygo(b *testing.B) { - path := "file://internal/e2e-guests/rewrite/main.wasm" - benchmarkMiddleware(b, path) + url := "file://internal/e2e-guests/rewrite/main.wasm" + benchmarkMiddleware(b, url) } // BenchmarkWat gives baseline performance for the same handler by @@ -39,8 +52,8 @@ func BenchmarkWat(b *testing.B) { benchmarkMiddleware(b, url) } -func benchmarkMiddleware(b *testing.B, path string) { - md := metadata.Base{Properties: map[string]string{"url": path}} +func benchmarkMiddleware(b *testing.B, url string) { + md := metadata.Base{Properties: map[string]string{"url": url}} l := logger.NewLogger(b.Name()) l.SetOutput(io.Discard) diff --git a/middleware/http/wasm/example/router.go b/middleware/http/wasm/example/router.go index a0d5c5ff8e..311808f3a8 100644 --- a/middleware/http/wasm/example/router.go +++ b/middleware/http/wasm/example/router.go @@ -1,3 +1,16 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( diff --git a/middleware/http/wasm/httpwasm.go b/middleware/http/wasm/httpwasm.go index df57241ff0..9fef5850d1 100644 --- a/middleware/http/wasm/httpwasm.go +++ b/middleware/http/wasm/httpwasm.go @@ -1,3 +1,16 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package wasm import ( @@ -22,6 +35,12 @@ type middleware struct { logger logger.Logger } +type Metadata struct { + // GuestConfig is an optional configuration passed to WASM guests. + // Users can pass an arbitrary string to be parsed by the guest code. + GuestConfig string `mapstructure:"guestConfig"` +} + func NewMiddleware(logger logger.Logger) dapr.Middleware { return &middleware{logger: logger} } @@ -36,18 +55,27 @@ func (m *middleware) GetHandler(ctx context.Context, metadata dapr.Metadata) (fu // getHandler is extracted for unit testing. func (m *middleware) getHandler(ctx context.Context, metadata dapr.Metadata) (*requestHandler, error) { + // parse common wasm metadata configuration meta, err := wasm.GetInitMetadata(ctx, metadata.Base) if err != nil { return nil, fmt.Errorf("wasm: failed to parse metadata: %w", err) } + // parse wasm middleware specific metadata + var middlewareMeta Metadata + err = mdutils.DecodeMetadata(metadata.Base, &middlewareMeta) + if err != nil { + return nil, fmt.Errorf("wasm: failed to parse wasm middleware metadata: %w", err) + } + var stdout, stderr bytes.Buffer mw, err := wasmnethttp.NewMiddleware(ctx, meta.Guest, handler.Logger(m), handler.ModuleConfig(wasm.NewModuleConfig(meta). WithName(meta.GuestName). WithStdout(&stdout). // reset per request - WithStderr(&stderr))) // reset per request + WithStderr(&stderr)), // reset per request + handler.GuestConfig([]byte(middlewareMeta.GuestConfig))) if err != nil { return nil, err } diff --git a/middleware/http/wasm/httpwasm_test.go b/middleware/http/wasm/httpwasm_test.go index 347c70fc46..670896ce0f 100644 --- a/middleware/http/wasm/httpwasm_test.go +++ b/middleware/http/wasm/httpwasm_test.go @@ -1,3 +1,16 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package wasm import ( diff --git a/middleware/http/wasm/internal/e2e-guests/config/main.go b/middleware/http/wasm/internal/e2e-guests/config/main.go new file mode 100644 index 0000000000..c0694f58e3 --- /dev/null +++ b/middleware/http/wasm/internal/e2e-guests/config/main.go @@ -0,0 +1,25 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package main ensures tests can prove logging or stdio isn't missed, both +// during initialization (main) and request (rewrite). +package main + +import ( + "github.com/http-wasm/http-wasm-guest-tinygo/handler" + "github.com/http-wasm/http-wasm-guest-tinygo/handler/api" +) + +func main() { + handler.Host.Log(api.LogLevelInfo, string(handler.Host.GetConfig())) +} diff --git a/middleware/http/wasm/internal/e2e-guests/config/main.wasm b/middleware/http/wasm/internal/e2e-guests/config/main.wasm new file mode 100755 index 0000000000..74ac0c5e53 Binary files /dev/null and b/middleware/http/wasm/internal/e2e-guests/config/main.wasm differ diff --git a/middleware/http/wasm/internal/e2e-guests/output/main.go b/middleware/http/wasm/internal/e2e-guests/output/main.go index 759dd10c12..08162512b3 100644 --- a/middleware/http/wasm/internal/e2e-guests/output/main.go +++ b/middleware/http/wasm/internal/e2e-guests/output/main.go @@ -1,3 +1,16 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + // Package main ensures tests can prove logging or stdio isn't missed, both // during initialization (main) and request (rewrite). package main diff --git a/middleware/http/wasm/internal/e2e-guests/rewrite/main.go b/middleware/http/wasm/internal/e2e-guests/rewrite/main.go index 0709daf91c..9325f9600d 100644 --- a/middleware/http/wasm/internal/e2e-guests/rewrite/main.go +++ b/middleware/http/wasm/internal/e2e-guests/rewrite/main.go @@ -1,3 +1,16 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( diff --git a/middleware/http/wasm/internal/e2e_test.go b/middleware/http/wasm/internal/e2e_test.go index efbcf58003..21c91ce34f 100644 --- a/middleware/http/wasm/internal/e2e_test.go +++ b/middleware/http/wasm/internal/e2e_test.go @@ -1,3 +1,16 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package internal_test import ( @@ -25,11 +38,12 @@ var guestWasm map[string][]byte const ( guestWasmOutput = "output" guestWasmRewrite = "rewrite" + guestWasmConfig = "config" ) // TestMain ensures we can read the test wasm prior to running e2e tests. func TestMain(m *testing.M) { - wasms := []string{guestWasmOutput, guestWasmRewrite} + wasms := []string{guestWasmOutput, guestWasmRewrite, guestWasmConfig} guestWasm = make(map[string][]byte, len(wasms)) for _, name := range wasms { if wasm, err := os.ReadFile(path.Join("e2e-guests", name, "main.wasm")); err != nil { @@ -48,9 +62,10 @@ func Test_EndToEnd(t *testing.T) { l.SetOutput(&buf) type testCase struct { - name string - guest []byte - test func(t *testing.T, handler http.Handler, log *bytes.Buffer) + name string + guest []byte + property map[string]string + test func(t *testing.T, handler http.Handler, log *bytes.Buffer) } tests := []testCase{ @@ -127,6 +142,14 @@ func Test_EndToEnd(t *testing.T) { require.Equal(t, "/v1.0/hello?name=teddy", r.URL.RequestURI()) }, }, + { + name: "log config to console", + guest: guestWasm[guestWasmConfig], + property: map[string]string{"guestConfig": "config bytes in any format"}, + test: func(t *testing.T, handler http.Handler, log *bytes.Buffer) { + require.Contains(t, log.String(), "config bytes in any format") + }, + }, } t.Run("local", func(t *testing.T) { @@ -139,6 +162,11 @@ func Test_EndToEnd(t *testing.T) { require.NoError(t, os.WriteFile(wasmPath, tc.guest, 0o600)) meta := metadata.Base{Properties: map[string]string{"url": "file://" + wasmPath}} + if len(tc.property) > 0 { + for k, v := range tc.property { + meta.Properties[k] = v + } + } handlerFn, err := wasm.NewMiddleware(l).GetHandler(context.Background(), middleware.Metadata{Base: meta}) require.NoError(t, err) handler := handlerFn(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) @@ -182,6 +210,11 @@ func Test_EndToEnd(t *testing.T) { defer ts.Close() meta := metadata.Base{Properties: map[string]string{"url": ts.URL + "/guest.wasm"}} + if len(tc.property) > 0 { + for k, v := range tc.property { + meta.Properties[k] = v + } + } handlerFn, err := wasm.NewMiddleware(l).GetHandler(context.Background(), middleware.Metadata{Base: meta}) require.NoError(t, err) handler := handlerFn(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) diff --git a/pubsub/pulsar/metadata.go b/pubsub/pulsar/metadata.go index 5bddedb6d6..719f75a69d 100644 --- a/pubsub/pulsar/metadata.go +++ b/pubsub/pulsar/metadata.go @@ -13,7 +13,11 @@ limitations under the License. package pulsar -import "time" +import ( + "time" + + "github.com/dapr/components-contrib/internal/authentication/oauth2" +) type pulsarMetadata struct { Host string `mapstructure:"host"` @@ -26,12 +30,14 @@ type pulsarMetadata struct { Tenant string `mapstructure:"tenant"` Namespace string `mapstructure:"namespace"` Persistent bool `mapstructure:"persistent"` - Token string `mapstructure:"token"` RedeliveryDelay time.Duration `mapstructure:"redeliveryDelay"` internalTopicSchemas map[string]schemaMetadata `mapstructure:"-"` PublicKey string `mapstructure:"publicKey"` PrivateKey string `mapstructure:"privateKey"` Keys string `mapstructure:"keys"` + + Token string `mapstructure:"token"` + oauth2.ClientCredentialsMetadata `mapstructure:",squash"` } type schemaMetadata struct { diff --git a/pubsub/pulsar/pulsar.go b/pubsub/pulsar/pulsar.go index 8846f2dc50..336b525fad 100644 --- a/pubsub/pulsar/pulsar.go +++ b/pubsub/pulsar/pulsar.go @@ -25,12 +25,12 @@ import ( "sync/atomic" "time" - "github.com/hamba/avro/v2" - "github.com/apache/pulsar-client-go/pulsar" "github.com/apache/pulsar-client-go/pulsar/crypto" + "github.com/hamba/avro/v2" lru "github.com/hashicorp/golang-lru/v2" + "github.com/dapr/components-contrib/internal/authentication/oauth2" "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/pubsub" "github.com/dapr/kit/logger" @@ -157,7 +157,7 @@ func parsePulsarMetadata(meta pubsub.Metadata) (*pulsarMetadata, error) { return &m, nil } -func (p *Pulsar) Init(_ context.Context, metadata pubsub.Metadata) error { +func (p *Pulsar) Init(ctx context.Context, metadata pubsub.Metadata) error { m, err := parsePulsarMetadata(metadata) if err != nil { return err @@ -173,9 +173,28 @@ func (p *Pulsar) Init(_ context.Context, metadata pubsub.Metadata) error { ConnectionTimeout: 30 * time.Second, TLSAllowInsecureConnection: !m.EnableTLS, } - if m.Token != "" { + + switch { + case len(m.Token) > 0: options.Authentication = pulsar.NewAuthenticationToken(m.Token) + case len(m.ClientCredentialsMetadata.TokenURL) > 0: + var cc *oauth2.ClientCredentials + cc, err = oauth2.NewClientCredentials(ctx, oauth2.ClientCredentialsOptions{ + Logger: p.logger, + TokenURL: m.ClientCredentialsMetadata.TokenURL, + CAPEM: []byte(m.ClientCredentialsMetadata.TokenCAPEM), + ClientID: m.ClientCredentialsMetadata.ClientID, + ClientSecret: m.ClientCredentialsMetadata.ClientSecret, + Scopes: m.ClientCredentialsMetadata.Scopes, + Audiences: m.ClientCredentialsMetadata.Audiences, + }) + if err != nil { + return fmt.Errorf("could not instantiate oauth2 token provider: %w", err) + } + + options.Authentication = pulsar.NewAuthenticationTokenFromSupplier(cc.Token) } + client, err := pulsar.NewClient(options) if err != nil { return fmt.Errorf("could not instantiate pulsar client: %v", err) diff --git a/secretstores/aws/parameterstore/metadata.yaml b/secretstores/aws/parameterstore/metadata.yaml new file mode 100644 index 0000000000..e8b40d6441 --- /dev/null +++ b/secretstores/aws/parameterstore/metadata.yaml @@ -0,0 +1,34 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: secretstores +name: aws.parameterstore +version: v1 +status: alpha +title: "AWS SSM Parameter Store" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-secret-stores/aws-parameter-store/ +builtinAuthenticationProfiles: + - name: "aws" +metadata: + - name: region + required: true + description: | + The specific AWS region the AWS SSM Parameter Store instance is deployed in. + example: '"us-east-1"' + type: string + - name: sessionToken + required: false + sensitive: true + description: | + AWS session token to use. A session token is only required if you are using + temporary security credentials. + example: '"TOKEN"' + type: string + - name: prefix + required: false + description: | + The SSM Parameter Store prefix to be specified. If specified, it will be + used as the 'BeginsWith' as part of the 'ParameterStringFilter'. + example: '"myprefix"' + type: string \ No newline at end of file diff --git a/secretstores/aws/parameterstore/parameterstore.go b/secretstores/aws/parameterstore/parameterstore.go index fef4294f32..5a62804b01 100644 --- a/secretstores/aws/parameterstore/parameterstore.go +++ b/secretstores/aws/parameterstore/parameterstore.go @@ -42,8 +42,8 @@ func NewParameterStore(logger logger.Logger) secretstores.SecretStore { type ParameterStoreMetaData struct { Region string `json:"region"` - AccessKey string `json:"accessKey"` - SecretKey string `json:"secretKey"` + AccessKey string `json:"accessKey" mapstructure:"accessKey" mdignore:"true"` + SecretKey string `json:"secretKey" mapstructure:"secretKey" mdignore:"true"` SessionToken string `json:"sessionToken"` Prefix string `json:"prefix"` } diff --git a/state/aws/dynamodb/dynamodb.go b/state/aws/dynamodb/dynamodb.go index ba7c3ab96b..86d6c79c48 100644 --- a/state/aws/dynamodb/dynamodb.go +++ b/state/aws/dynamodb/dynamodb.go @@ -46,11 +46,13 @@ type StateStore struct { } type dynamoDBMetadata struct { + // Ignored by metadata parser because included in built-in authentication profile + AccessKey string `json:"accessKey" mapstructure:"accessKey" mdignore:"true"` + SecretKey string `json:"secretKey" mapstructure:"secretKey" mdignore:"true"` + SessionToken string `json:"sessionToken" mapstructure:"sessionToken" mdignore:"true"` + Region string `json:"region"` Endpoint string `json:"endpoint"` - AccessKey string `json:"accessKey"` - SecretKey string `json:"secretKey"` - SessionToken string `json:"sessionToken"` Table string `json:"table"` TTLAttributeName string `json:"ttlAttributeName"` PartitionKey string `json:"partitionKey"` @@ -92,7 +94,19 @@ func (d *StateStore) Init(_ context.Context, metadata state.Metadata) error { // Features returns the features available in this state store. func (d *StateStore) Features() []state.Feature { - return []state.Feature{state.FeatureETag, state.FeatureTransactional} + // TTLs are enabled only if ttlAttributeName is set + if d.ttlAttributeName == "" { + return []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + } + } + + return []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureTTL, + } } // Get retrieves a dynamoDB item. diff --git a/state/aws/dynamodb/metadata.yaml b/state/aws/dynamodb/metadata.yaml new file mode 100644 index 0000000000..ed0c9a3fbc --- /dev/null +++ b/state/aws/dynamodb/metadata.yaml @@ -0,0 +1,59 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: state +name: aws.dynamodb +version: v1 +status: stable +title: "AWS DynamoDB" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-dynamodb/ +capabilities: + - crud + - transactional + - etag + - ttl + - actorStateStore +builtinAuthenticationProfiles: + - name: "aws" +metadata: + - name: table + required: true + description: | + The name of the DynamoDB table to use. + example: '"Contracts"' + type: string + - name: region + required: false + description: | + The AWS region to use. Ensure that DynamoDB is available in that region. + See the `Amazon DynamoDB endpoints and quotas` documentation. + url: + title: Amazon DynamoDB endpoints and quotas + url: https://docs.aws.amazon.com/general/latest/gr/ddb.html + example: '"us-east-1"' + type: string + - name: endpoint + required: false + description: | + AWS endpoint for the component to use. Only used for local development. + The endpoint is not necessary when running against production AWS. + example: '"http://localhost:4566"' + type: string + - name: ttlAttributeName + required: false + description: | + The table attribute name which should be used for TTL. + example: '"expiresAt"' + type: string + - name: partitionKey + required: false + description: | + The table primary key or partition key attribute name. + This field is used to replace the default primary key attribute name "key". + url: + title: More details + url: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-dynamodb/#partition-keys + example: '"ContractID"' + type: string + \ No newline at end of file diff --git a/state/azure/cosmosdb/cosmosdb.go b/state/azure/cosmosdb/cosmosdb.go index 65a34e0fcb..c56352c38e 100644 --- a/state/azure/cosmosdb/cosmosdb.go +++ b/state/azure/cosmosdb/cosmosdb.go @@ -204,6 +204,7 @@ func (c *StateStore) Features() []state.Feature { state.FeatureETag, state.FeatureTransactional, state.FeatureQueryAPI, + state.FeatureTTL, } } diff --git a/state/cassandra/cassandra.go b/state/cassandra/cassandra.go index 3ac0cda60a..6bbc00b431 100644 --- a/state/cassandra/cassandra.go +++ b/state/cassandra/cassandra.go @@ -117,7 +117,9 @@ func (c *Cassandra) Init(_ context.Context, metadata state.Metadata) error { // Features returns the features available in this state store. func (c *Cassandra) Features() []state.Feature { - return nil + return []state.Feature{ + state.FeatureTTL, + } } func (c *Cassandra) tryCreateKeyspace(keyspace string, replicationFactor int) error { diff --git a/state/cloudflare/workerskv/workerskv.go b/state/cloudflare/workerskv/workerskv.go index c1a8609f6b..7a5ee03bb1 100644 --- a/state/cloudflare/workerskv/workerskv.go +++ b/state/cloudflare/workerskv/workerskv.go @@ -90,7 +90,9 @@ func (q *CFWorkersKV) GetComponentMetadata() (metadataInfo metadata.MetadataMap) // Features returns the features supported by this state store. func (q CFWorkersKV) Features() []state.Feature { - return []state.Feature{} + return []state.Feature{ + state.FeatureTTL, + } } func (q *CFWorkersKV) Delete(parentCtx context.Context, stateReq *state.DeleteRequest) error { diff --git a/state/couchbase/couchbase.go b/state/couchbase/couchbase.go index 3da43f1567..b1fe9ea7ac 100644 --- a/state/couchbase/couchbase.go +++ b/state/couchbase/couchbase.go @@ -66,9 +66,11 @@ type couchbaseMetadata struct { // NewCouchbaseStateStore returns a new couchbase state store. func NewCouchbaseStateStore(logger logger.Logger) state.Store { s := &Couchbase{ - json: jsoniter.ConfigFastest, - features: []state.Feature{state.FeatureETag}, - logger: logger, + json: jsoniter.ConfigFastest, + features: []state.Feature{ + state.FeatureETag, + }, + logger: logger, } s.BulkStore = state.NewDefaultBulkStore(s) return s @@ -271,3 +273,10 @@ func (cbs *Couchbase) GetComponentMetadata() (metadataInfo metadata.MetadataMap) metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) return } + +func (cbs *Couchbase) Close() error { + if cbs.bucket == nil { + return nil + } + return cbs.bucket.Close() +} diff --git a/state/etcd/etcd.go b/state/etcd/etcd.go index 4bac16a911..4d9360545a 100644 --- a/state/etcd/etcd.go +++ b/state/etcd/etcd.go @@ -67,9 +67,13 @@ func NewEtcdStateStoreV2(logger logger.Logger) state.Store { func newETCD(logger logger.Logger, schema schemaMarshaller) state.Store { s := &Etcd{ - schema: schema, - logger: logger, - features: []state.Feature{state.FeatureETag, state.FeatureTransactional}, + schema: schema, + logger: logger, + features: []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureTTL, + }, } s.BulkStore = state.NewDefaultBulkStore(s) return s @@ -188,38 +192,29 @@ func (e *Etcd) doSet(ctx context.Context, key string, val any, etag *string, ttl return err } + var leaseID clientv3.LeaseID if ttlInSeconds != nil { - resp, err := e.client.Grant(ctx, *ttlInSeconds) + var resp *clientv3.LeaseGrantResponse + resp, err = e.client.Grant(ctx, *ttlInSeconds) if err != nil { return fmt.Errorf("couldn't grant lease %s: %w", key, err) } - if etag != nil { - etag, _ := strconv.ParseInt(*etag, 10, 64) - _, err = e.client.Txn(ctx). - If(clientv3.Compare(clientv3.ModRevision(key), "=", etag)). - Then(clientv3.OpPut(key, reqVal, clientv3.WithLease(resp.ID))). - Commit() - } else { - _, err = e.client.Put(ctx, key, reqVal, clientv3.WithLease(resp.ID)) - } - if err != nil { - return fmt.Errorf("couldn't set key %s: %w", key, err) - } + leaseID = resp.ID + } + + if etag != nil { + etag, _ := strconv.ParseInt(*etag, 10, 64) + _, err = e.client.Txn(ctx). + If(clientv3.Compare(clientv3.ModRevision(key), "=", etag)). + Then(clientv3.OpPut(key, reqVal, clientv3.WithLease(leaseID))). + Commit() } else { - var err error - if etag != nil { - etag, _ := strconv.ParseInt(*etag, 10, 64) - _, err = e.client.Txn(ctx). - If(clientv3.Compare(clientv3.ModRevision(key), "=", etag)). - Then(clientv3.OpPut(key, reqVal)). - Commit() - } else { - _, err = e.client.Put(ctx, key, reqVal) - } - if err != nil { - return fmt.Errorf("couldn't set key %s: %w", key, err) - } + _, err = e.client.Put(ctx, key, reqVal, clientv3.WithLease(leaseID)) + } + if err != nil { + return fmt.Errorf("couldn't set key %s: %w", key, err) } + return nil } diff --git a/state/feature.go b/state/feature.go index e42d3fdf5a..9a53370076 100644 --- a/state/feature.go +++ b/state/feature.go @@ -24,9 +24,11 @@ const ( FeatureTransactional Feature = "TRANSACTIONAL" // FeatureQueryAPI is the feature that performs query operations. FeatureQueryAPI Feature = "QUERY_API" + // FeatureTTL is the feature that supports TTLs. + FeatureTTL Feature = "TTL" ) -// Feature names a feature that can be implemented by PubSub components. +// Feature names a feature that can be implemented by state store components. type Feature string // IsPresent checks if a given feature is present in the list. diff --git a/state/in-memory/in_memory.go b/state/in-memory/in_memory.go index a05f2bd94e..f0bc73f3f7 100644 --- a/state/in-memory/in_memory.go +++ b/state/in-memory/in_memory.go @@ -91,6 +91,7 @@ func (store *inMemoryStore) Features() []state.Feature { return []state.Feature{ state.FeatureETag, state.FeatureTransactional, + state.FeatureTTL, } } diff --git a/state/memcached/memcached.go b/state/memcached/memcached.go index d4171e13ff..2bb0a99b2e 100644 --- a/state/memcached/memcached.go +++ b/state/memcached/memcached.go @@ -93,7 +93,9 @@ func (m *Memcached) Init(_ context.Context, metadata state.Metadata) error { // Features returns the features available in this state store. func (m *Memcached) Features() []state.Feature { - return nil + return []state.Feature{ + state.FeatureTTL, + } } func getMemcachedMetadata(meta state.Metadata) (*memcachedMetadata, error) { diff --git a/state/mongodb/metadata.yaml b/state/mongodb/metadata.yaml index 2b76b3d04f..7537dc9169 100644 --- a/state/mongodb/metadata.yaml +++ b/state/mongodb/metadata.yaml @@ -17,6 +17,7 @@ capabilities: - transactional - etag - query + - ttl authenticationProfiles: - title: "Connection string" description: | diff --git a/state/mongodb/mongodb.go b/state/mongodb/mongodb.go index 49075586db..f9a0f6fe36 100644 --- a/state/mongodb/mongodb.go +++ b/state/mongodb/mongodb.go @@ -112,8 +112,13 @@ type Item struct { // NewMongoDB returns a new MongoDB state store. func NewMongoDB(logger logger.Logger) state.Store { s := &MongoDB{ - features: []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureQueryAPI}, - logger: logger, + features: []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureQueryAPI, + state.FeatureTTL, + }, + logger: logger, } s.BulkStore = state.NewDefaultBulkStore(s) return s diff --git a/state/mysql/mysql.go b/state/mysql/mysql.go index 6e5f6414b9..4554a5415e 100644 --- a/state/mysql/mysql.go +++ b/state/mysql/mysql.go @@ -225,7 +225,11 @@ func (m *MySQL) parseMetadata(md map[string]string) error { // Features returns the features available in this state store. func (m *MySQL) Features() []state.Feature { - return []state.Feature{state.FeatureETag, state.FeatureTransactional} + return []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureTTL, + } } // Ping the database. diff --git a/state/oci/objectstorage/objectstorage.go b/state/oci/objectstorage/objectstorage.go index 7ed5748eca..471ee88145 100644 --- a/state/oci/objectstorage/objectstorage.go +++ b/state/oci/objectstorage/objectstorage.go @@ -168,10 +168,13 @@ func (r *StateStore) Ping(ctx context.Context) error { func NewOCIObjectStorageStore(logger logger.Logger) state.Store { s := &StateStore{ - json: jsoniter.ConfigFastest, - features: []state.Feature{state.FeatureETag}, - logger: logger, - client: nil, + json: jsoniter.ConfigFastest, + features: []state.Feature{ + state.FeatureETag, + state.FeatureTTL, + }, + logger: logger, + client: nil, } s.BulkStore = state.NewDefaultBulkStore(s) diff --git a/state/oracledatabase/oracledatabase.go b/state/oracledatabase/oracledatabase.go index 32e883287b..890e9e7606 100644 --- a/state/oracledatabase/oracledatabase.go +++ b/state/oracledatabase/oracledatabase.go @@ -44,7 +44,11 @@ func NewOracleDatabaseStateStore(logger logger.Logger) state.Store { // This unexported constructor allows injecting a dbAccess instance for unit testing. func newOracleDatabaseStateStore(logger logger.Logger, dba dbAccess) *OracleDatabase { return &OracleDatabase{ - features: []state.Feature{state.FeatureETag, state.FeatureTransactional}, + features: []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureTTL, + }, logger: logger, dbaccess: dba, } diff --git a/state/redis/redis.go b/state/redis/redis.go index d5e1c33ee4..a3f65af26b 100644 --- a/state/redis/redis.go +++ b/state/redis/redis.go @@ -161,9 +161,9 @@ func (r *StateStore) Init(ctx context.Context, metadata state.Metadata) error { // Features returns the features available in this state store. func (r *StateStore) Features() []state.Feature { if r.clientHasJSON { - return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureQueryAPI} + return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureTTL, state.FeatureQueryAPI} } else { - return []state.Feature{state.FeatureETag, state.FeatureTransactional} + return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureTTL} } } diff --git a/state/rethinkdb/rethinkdb.go b/state/rethinkdb/rethinkdb.go index 481ef740e7..dd33c5e09a 100644 --- a/state/rethinkdb/rethinkdb.go +++ b/state/rethinkdb/rethinkdb.go @@ -37,7 +37,7 @@ const ( stateArchiveTablePKName = "key" ) -// RethinkDB is a state store implementation with transactional support for RethinkDB. +// RethinkDB is a state store implementation for RethinkDB. type RethinkDB struct { session *r.Session config *stateConfig diff --git a/state/sqlite/sqlite.go b/state/sqlite/sqlite.go index 218c16508f..a9ff2ee41a 100644 --- a/state/sqlite/sqlite.go +++ b/state/sqlite/sqlite.go @@ -48,6 +48,7 @@ func newSQLiteStateStore(logger logger.Logger, dba DBAccess) *SQLiteStore { features: []state.Feature{ state.FeatureETag, state.FeatureTransactional, + state.FeatureTTL, }, dbaccess: dba, } diff --git a/state/sqlserver/metadata.yaml b/state/sqlserver/metadata.yaml index f5129bbb25..8b62a1ec26 100644 --- a/state/sqlserver/metadata.yaml +++ b/state/sqlserver/metadata.yaml @@ -15,6 +15,7 @@ capabilities: - "crud" - "transactional" - "etag" + - "ttl" authenticationProfiles: - title: "Connection string" description: | diff --git a/state/sqlserver/sqlserver.go b/state/sqlserver/sqlserver.go index c87cafa491..77623d6152 100644 --- a/state/sqlserver/sqlserver.go +++ b/state/sqlserver/sqlserver.go @@ -65,7 +65,11 @@ const ( // New creates a new instance of a SQL Server transaction store. func New(logger logger.Logger) state.Store { s := &SQLServer{ - features: []state.Feature{state.FeatureETag, state.FeatureTransactional}, + features: []state.Feature{ + state.FeatureETag, + state.FeatureTransactional, + state.FeatureTTL, + }, logger: logger, migratorFactory: newMigration, } @@ -238,12 +242,6 @@ func (s *SQLServer) executeDelete(ctx context.Context, db dbExecutor, req *state return nil } -// TvpDeleteTableStringKey defines a table type with string key. -type TvpDeleteTableStringKey struct { - ID string - RowVersion []byte -} - // Get returns an entity from store. func (s *SQLServer) Get(ctx context.Context, req *state.GetRequest) (*state.GetResponse, error) { rows, err := s.db.QueryContext(ctx, s.getCommand, sql.Named(keyColumnName, req.Key)) diff --git a/tests/certification/bindings/azure/servicebusqueues/servicebusqueue_test.go b/tests/certification/bindings/azure/servicebusqueues/servicebusqueue_test.go index ad7d2af960..40a7e50021 100644 --- a/tests/certification/bindings/azure/servicebusqueues/servicebusqueue_test.go +++ b/tests/certification/bindings/azure/servicebusqueues/servicebusqueue_test.go @@ -14,8 +14,12 @@ limitations under the License. package servicebusqueue_test import ( + "bytes" "context" + "crypto/rand" + "encoding/base64" "fmt" + "io" "testing" "time" @@ -49,6 +53,15 @@ const ( numMessages = 100 ) +var testprefix string + +func init() { + // Generate a random test prefix + rnd := make([]byte, 7) + io.ReadFull(rand.Reader, rnd) + testprefix = base64.RawURLEncoding.EncodeToString(rnd) +} + func TestServiceBusQueue(t *testing.T) { messagesFor1 := watcher.NewOrdered() messagesFor2 := watcher.NewOrdered() @@ -67,11 +80,11 @@ func TestServiceBusQueue(t *testing.T) { msgsFor1 := make([]string, numMessages/2) msgsFor2 := make([]string, numMessages/2) for i := 0; i < numMessages/2; i++ { - msgsFor1[i] = fmt.Sprintf("sb-binding-1: Message %03d", i) + msgsFor1[i] = fmt.Sprintf("%s: sb-binding-1: Message %03d", testprefix, i) } for i := numMessages / 2; i < numMessages; i++ { - msgsFor2[i-(numMessages/2)] = fmt.Sprintf("sb-binding-2: Message %03d", i) + msgsFor2[i-(numMessages/2)] = fmt.Sprintf("%s: sb-binding-2: Message %03d", testprefix, i) } messagesFor1.ExpectStrings(msgsFor1...) @@ -108,11 +121,19 @@ func TestServiceBusQueue(t *testing.T) { // Setup the input binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("sb-binding-1", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + messagesFor1.Observe(string(in.Data)) ctx.Logf("Got message: %s", string(in.Data)) return []byte("{}"), nil }), s.AddBindingInvocationHandler("sb-binding-2", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + messagesFor2.Observe(string(in.Data)) ctx.Logf("Got message: %s", string(in.Data)) return []byte("{}"), nil @@ -128,7 +149,7 @@ func TestServiceBusQueue(t *testing.T) { embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(grpcPort), embedded.WithDaprHTTPPort(httpPort), - embedded.WithComponentsPath("./components/standard"), + embedded.WithResourcesPath("./components/standard"), componentRuntimeOptions(), )). // Block the standard AMPQ ports. @@ -151,23 +172,38 @@ func TestAzureServiceBusQueuesTTLs(t *testing.T) { ctx.Logf("Sending messages for expiration.") for i := 0; i < numMessages; i++ { - msg := fmt.Sprintf("Expiring message %d", i) + msg := fmt.Sprintf("%s: Expiring message %d", testprefix, i) metadata := make(map[string]string) // Send to the queue with TTL. - queueTTLReq := &daprClient.InvokeBindingRequest{Name: "queuettl", Operation: "create", Data: []byte(msg), Metadata: metadata} + queueTTLReq := &daprClient.InvokeBindingRequest{ + Name: "queuettl", + Operation: "create", + Data: []byte(msg), + Metadata: metadata, + } err := client.InvokeOutputBinding(ctx, queueTTLReq) require.NoError(ctx, err, "error publishing message") // Send message with TTL. - messageTTLReq := &daprClient.InvokeBindingRequest{Name: "messagettl", Operation: "create", Data: []byte(msg), Metadata: metadata} + messageTTLReq := &daprClient.InvokeBindingRequest{ + Name: "messagettl", + Operation: "create", + Data: []byte(msg), + Metadata: metadata, + } messageTTLReq.Metadata["ttlInSeconds"] = "10" err = client.InvokeOutputBinding(ctx, messageTTLReq) require.NoError(ctx, err, "error publishing message") // Send message with TTL to ensure it overwrites Queue TTL. - mixedTTLReq := &daprClient.InvokeBindingRequest{Name: "mixedttl", Operation: "create", Data: []byte(msg), Metadata: metadata} + mixedTTLReq := &daprClient.InvokeBindingRequest{ + Name: "mixedttl", + Operation: "create", + Data: []byte(msg), + Metadata: metadata, + } mixedTTLReq.Metadata["ttlInSeconds"] = "10" err = client.InvokeOutputBinding(ctx, mixedTTLReq) require.NoError(ctx, err, "error publishing message") @@ -182,16 +218,28 @@ func TestAzureServiceBusQueuesTTLs(t *testing.T) { // Setup the input binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("queuettl", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + ctx.Logf("Oh no! Got message: %s", string(in.Data)) ttlMessages.FailIfNotExpected(t, string(in.Data)) return []byte("{}"), nil }), s.AddBindingInvocationHandler("messagettl", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + ctx.Logf("Oh no! Got message: %s", string(in.Data)) ttlMessages.FailIfNotExpected(t, string(in.Data)) return []byte("{}"), nil }), s.AddBindingInvocationHandler("mixedttl", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + ctx.Logf("Oh no! Got message: %s", string(in.Data)) ttlMessages.FailIfNotExpected(t, string(in.Data)) return []byte("{}"), nil @@ -207,7 +255,7 @@ func TestAzureServiceBusQueuesTTLs(t *testing.T) { embedded.WithoutApp(), embedded.WithDaprGRPCPort(grpcPort), embedded.WithDaprHTTPPort(httpPort), - embedded.WithComponentsPath("./components/ttl"), + embedded.WithResourcesPath("./components/ttl"), componentRuntimeOptions(), )). Step("send ttl messages", sendTTLMessages). @@ -242,7 +290,7 @@ func TestAzureServiceBusQueueRetriesOnError(t *testing.T) { // that will satisfy the test. msgs := make([]string, numMessages/2) for i := 0; i < numMessages/2; i++ { - msgs[i] = fmt.Sprintf("Message %03d", i) + msgs[i] = fmt.Sprintf("%s: Message %03d", testprefix, i) } messages.ExpectStrings(msgs...) @@ -252,7 +300,11 @@ func TestAzureServiceBusQueueRetriesOnError(t *testing.T) { for _, msg := range msgs { ctx.Logf("Sending: %q", msg) - req := &daprClient.InvokeBindingRequest{Name: "retry-binding", Operation: "create", Data: []byte(msg)} + req := &daprClient.InvokeBindingRequest{ + Name: "retry-binding", + Operation: "create", + Data: []byte(msg), + } err := client.InvokeOutputBinding(ctx, req) require.NoError(ctx, err, "error publishing message") } @@ -271,6 +323,10 @@ func TestAzureServiceBusQueueRetriesOnError(t *testing.T) { // Setup the input binding endpoint err = multierr.Combine(err, s.AddBindingInvocationHandler("retry-binding", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + if err := sim(); err != nil { ctx.Logf("Failing message: %s", string(in.Data)) return nil, err @@ -291,7 +347,7 @@ func TestAzureServiceBusQueueRetriesOnError(t *testing.T) { embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(grpcPort), embedded.WithDaprHTTPPort(httpPort), - embedded.WithComponentsPath("./components/retry"), + embedded.WithResourcesPath("./components/retry"), componentRuntimeOptions(), )). Step("send and wait", test). @@ -312,10 +368,17 @@ func TestServiceBusQueueMetadata(t *testing.T) { // Send events that the application above will observe. ctx.Log("Invoking binding!") - req := &daprClient.InvokeBindingRequest{Name: "sb-binding-1", Operation: "create", Data: []byte("test msg"), Metadata: map[string]string{"Testmetadata": "Some Metadata"}} + req := &daprClient.InvokeBindingRequest{ + Name: "sb-binding-1", + Operation: "create", + Data: []byte(testprefix + ": test msg"), + Metadata: map[string]string{"Testmetadata": "Some Metadata"}, + } err = client.InvokeOutputBinding(ctx, req) require.NoError(ctx, err, "error publishing message") + messages.ExpectStrings(string(req.Data)) + // Do the messages we observed match what we expect? messages.Assert(ctx, time.Minute) @@ -327,11 +390,15 @@ func TestServiceBusQueueMetadata(t *testing.T) { // Setup the input binding endpoints err = multierr.Combine(err, s.AddBindingInvocationHandler("sb-binding-1", func(_ context.Context, in *common.BindingEvent) ([]byte, error) { + if !bytes.HasPrefix(in.Data, []byte(testprefix)) { + return []byte("{}"), nil + } + messages.Observe(string(in.Data)) - ctx.Logf("Got message: %s - %+v", string(in.Data), in.Metadata) - require.NotEmpty(t, in.Metadata) - require.Contains(t, in.Metadata, "Testmetadata") - require.Equal(t, "Some Metadata", in.Metadata["Testmetadata"]) + ctx.Logf("Got message: %s - %#v", string(in.Data), in.Metadata) + require.NotEmptyf(t, in.Metadata, "Data: %s - Metadata: %#v", in.Data, in.Metadata) + require.Containsf(t, in.Metadata, "Testmetadata", "Data: %s - Metadata: %#v", in.Data, in.Metadata) + require.Equalf(t, "Some+Metadata", in.Metadata["Testmetadata"], "Data: %s - Metadata: %#v", in.Data, in.Metadata) // + because the message is encoded for HTTP headers return []byte("{}"), nil })) @@ -346,7 +413,7 @@ func TestServiceBusQueueMetadata(t *testing.T) { embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(grpcPort), embedded.WithDaprHTTPPort(httpPort), - embedded.WithComponentsPath("./components/standard"), + embedded.WithResourcesPath("./components/standard"), componentRuntimeOptions(), )). Step("send and wait", test). @@ -364,7 +431,12 @@ func TestServiceBusQueueDisableEntityManagement(t *testing.T) { // Send events that the application above will observe. ctx.Log("Invoking binding!") - req := &daprClient.InvokeBindingRequest{Name: "mgmt-binding", Operation: "create", Data: []byte("test msg"), Metadata: map[string]string{"TestMetadata": "Some Metadata"}} + req := &daprClient.InvokeBindingRequest{ + Name: "mgmt-binding", + Operation: "create", + Data: []byte(testprefix + ": test msg"), + Metadata: map[string]string{"TestMetadata": "Some Metadata"}, + } err = client.InvokeOutputBinding(ctx, req) require.Error(ctx, err, "error publishing message") return nil @@ -376,7 +448,7 @@ func TestServiceBusQueueDisableEntityManagement(t *testing.T) { embedded.WithoutApp(), embedded.WithDaprGRPCPort(grpcPort), embedded.WithDaprHTTPPort(httpPort), - embedded.WithComponentsPath("./components/disable_entity_mgmt"), + embedded.WithResourcesPath("./components/disable_entity_mgmt"), componentRuntimeOptions(), )). Step("send and wait", testWithExpectedFailure). diff --git a/tests/certification/go.mod b/tests/certification/go.mod index 04177a6292..3fcac4c78c 100644 --- a/tests/certification/go.mod +++ b/tests/certification/go.mod @@ -3,15 +3,15 @@ module github.com/dapr/components-contrib/tests/certification go 1.20 require ( - cloud.google.com/go/pubsub v1.32.0 + cloud.google.com/go/pubsub v1.33.0 dubbo.apache.org/dubbo-go/v3 v3.0.3-0.20230118042253-4f159a2b38f3 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/Shopify/sarama v1.38.1 github.com/a8m/documentdb v1.3.0 github.com/apache/dubbo-go-hessian2 v1.11.5 github.com/apache/pulsar-client-go v0.11.0 github.com/apache/thrift v0.13.0 - github.com/aws/aws-sdk-go v1.44.299 + github.com/aws/aws-sdk-go v1.44.315 github.com/benbjohnson/clock v1.3.5 github.com/cenkalti/backoff/v4 v4.2.1 github.com/cloudwego/kitex v0.5.0 @@ -36,7 +36,7 @@ require ( go.mongodb.org/mongo-driver v1.12.0 go.uber.org/multierr v1.11.0 go.uber.org/ratelimit v0.3.0 - golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb + golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b modernc.org/sqlite v1.24.0 ) @@ -44,7 +44,7 @@ require ( cloud.google.com/go v0.110.2 // indirect cloud.google.com/go/compute v1.20.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/datastore v1.12.1 // indirect + cloud.google.com/go/datastore v1.13.0 // indirect cloud.google.com/go/iam v1.1.0 // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect @@ -55,12 +55,12 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5 // indirect - github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.0.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.4.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0 // indirect github.com/Azure/go-amqp v1.0.1 // indirect @@ -197,7 +197,7 @@ require ( github.com/mattn/go-isatty v0.0.18 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/microsoft/durabletask-go v0.2.4 // indirect - github.com/microsoft/go-mssqldb v1.3.0 // indirect + github.com/microsoft/go-mssqldb v1.5.0 // indirect github.com/miekg/dns v1.1.50 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -248,7 +248,7 @@ require ( github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.47.0 // indirect + github.com/valyala/fasthttp v1.48.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect @@ -269,7 +269,7 @@ require ( golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.11.0 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/net v0.13.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.10.0 // indirect @@ -298,7 +298,7 @@ require ( k8s.io/component-base v0.26.3 // indirect k8s.io/klog/v2 v2.80.1 // indirect k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect lukechampine.com/uint128 v1.3.0 // indirect modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect diff --git a/tests/certification/go.sum b/tests/certification/go.sum index 1075e827a0..488a687fb0 100644 --- a/tests/certification/go.sum +++ b/tests/certification/go.sum @@ -27,8 +27,8 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.12.1 h1:i8HMKsqg/Sl3ZlOTGl471Z8j2uKtbRDT9VXJUIVlMik= -cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.13.0 h1:ktbC66bOQB3HJPQe8qNI1/aiQ77PMu7hD4mzE6uxe3w= +cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94= cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= @@ -37,8 +37,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.32.0 h1:JOEkgEYBuUTHSyHS4TcqOFuWr+vD6qO/imsFqShUCp4= -cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -69,8 +69,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybI github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5 h1:qS0Bp4do0cIvnuQgSGeO6ZCu/q/HlRKl4NPfv1eJ2p0= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.5/go.mod h1:Beh5cHIXJ0oWEDWk9lNFtuklCojLLQ5hl+LqSNTTs0I= -github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 h1:bFa9IcjvrCber6gGgDAUZ+I2bO8J7s8JxXmu9fhi2ss= -github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1/go.mod h1:l3wvZkG9oW07GLBW5Cd0WwG5asOfJ8aqE8raUvNzLpk= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2 h1:iXFUCl7NK2DPVKfixcYDPGj3uLV7yf5eolBsoWD8Sc4= +github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2/go.mod h1:E1WPwLx0wZyus7NBHjhrHE1QgWwKJPE81fnUbT+FxqI= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.0.1 h1:7G4EhZbWFwfgkNfJkNoZmFL8FfWT6P96YVwG71uhNxY= @@ -81,12 +81,13 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1.1/go.mod h1:7fQVOnRA11ScLE8dOCWanXHQa2NMFOM2i0u/1VRICXA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0 h1:upXr9dsOnTJk3eHQ3ldyvIXAIGggHtkrfrgbcas6DXU= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.14.0/go.mod h1:w2K61Z8eppIuGbQRx1SKYld2Lrr5vrGvnUwWAhF4nso= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0 h1:qvCB+Za4z8dtU3R5CC7zhlxTLlT3eaEMugglVvjUWtk= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.0.0/go.mod h1:w2K61Z8eppIuGbQRx1SKYld2Lrr5vrGvnUwWAhF4nso= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 h1:T028gtTPiYt/RMUfs8nVsAL7FDQrfLlrm/NnRG/zcC4= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0/go.mod h1:cw4zVQgBby0Z5f2v0itn6se2dDP17nTjbZFXW5uPyHA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0 h1:lJwNFV+xYjHREUTHJKx/ZF6CJSt9znxmLw9DqSTvyRU= github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.0/go.mod h1:GfT0aGew8Qj5yiQVqOO5v7N8fanbJGyUoHqXg56qcVY= github.com/Azure/go-amqp v1.0.1 h1:Jf8OQCKzRDMZ3pCiH4onM7yrhl5curkRSGkRLTyP35o= @@ -171,8 +172,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.44.299 h1:HVD9lU4CAFHGxleMJp95FV/sRhtg7P4miHD1v88JAQk= -github.com/aws/aws-sdk-go v1.44.299/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.315 h1:kYTC+Y/bJ9M7QQRvkI/LN5OWvhkIOL/YuFFRhS5QAOo= +github.com/aws/aws-sdk-go v1.44.315/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= @@ -913,8 +914,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microsoft/durabletask-go v0.2.4 h1:jeTz559GSXHmOzp5iTbeIq35YYxKSaDHkJcnl8F9wX4= github.com/microsoft/durabletask-go v0.2.4/go.mod h1:UtJXHmKalksdccRiN9Y16cHJYYtZN0bqmqOSiy56V8g= -github.com/microsoft/go-mssqldb v1.3.0 h1:JcPVl+acL8Z/cQcJc9zP0OkjQ+l20bco/cCDpMbmGJk= -github.com/microsoft/go-mssqldb v1.3.0/go.mod h1:lmWsjHD8XX/Txr0f8ZqgbEZSC+BZjmEQy/Ms+rLrvho= +github.com/microsoft/go-mssqldb v1.5.0 h1:CgENxkwtOBNj3Jg6T1X209y2blCfTTcwuOlznd2k9fk= +github.com/microsoft/go-mssqldb v1.5.0/go.mod h1:lmWsjHD8XX/Txr0f8ZqgbEZSC+BZjmEQy/Ms+rLrvho= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -1244,8 +1245,8 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c= -github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= +github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -1412,8 +1413,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1515,8 +1516,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1964,8 +1965,8 @@ k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= diff --git a/tests/certification/pubsub/pulsar/components/consumer_five/pulsar.yaml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_five/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_five/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_five/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/consumer_four/pulsar.yaml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_four/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_four/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_four/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/consumer_one/pulsar.yml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_one/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_one/pulsar.yml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_one/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/components/consumer_six/pulsar.yaml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_six/pulsar.yaml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_six/pulsar.yaml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_six/pulsar.yaml diff --git a/tests/certification/pubsub/pulsar/components/consumer_three/pulsar.yml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_three/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_three/pulsar.yml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_three/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/components/consumer_two/pulsar.yml b/tests/certification/pubsub/pulsar/components/auth-none/consumer_two/pulsar.yml similarity index 100% rename from tests/certification/pubsub/pulsar/components/consumer_two/pulsar.yml rename to tests/certification/pubsub/pulsar/components/auth-none/consumer_two/pulsar.yml diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_five/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_five/pulsar.yml.tmpl new file mode 100644 index 0000000000..9251380ebb --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_five/pulsar.yml.tmpl @@ -0,0 +1,32 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification5 + - name: redeliveryDelay + value: 200ms + - name: publicKey + value: public.key + - name: privateKey + value: private.key + - name: keys + value: myapp.key + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_four/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_four/pulsar.yml.tmpl new file mode 100644 index 0000000000..3078529e2e --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_four/pulsar.yml.tmpl @@ -0,0 +1,28 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification4 + - name: redeliveryDelay + value: 200ms + - name: certification-pubsub-topic-active.jsonschema + value: "{\"type\":\"record\",\"name\":\"Example\",\"namespace\":\"test\",\"fields\":[{\"name\":\"ID\",\"type\":\"int\"},{\"name\":\"Name\",\"type\":\"string\"}]}" + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_one/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_one/pulsar.yml.tmpl new file mode 100644 index 0000000000..d5258c3a1c --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_one/pulsar.yml.tmpl @@ -0,0 +1,26 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification1 + - name: redeliveryDelay + value: 200ms + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_six/pulsar.yaml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_six/pulsar.yaml.tmpl new file mode 100644 index 0000000000..1f7bf98510 --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_six/pulsar.yaml.tmpl @@ -0,0 +1,32 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification5 + - name: redeliveryDelay + value: 200ms + - name: publicKey + value: "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1KDAM4L8RtJ+nLaXBrBh\nzVpvTemsKVZoAct8A+ShepOHT9lgHOCGLFGWNla6K6j+b3AV/P/fAAhwj82vwTDd\nruXSflvSdmYeFAw3Ypphc1A5oM53wSRWhg63potBNWqdDzj8ApYgqjpmjYSQdL5/\na3golb36GYFrY0MLFTv7wZ87pmMIPsOgGIcPbCHker2fRZ34WXYLb1hkeUpwx4eK\njpwcg35gccvR6o/UhbKAuc60V1J9Wof2sNgtlRaQej45wnpjWYzZrIyk5qUbn0Qi\nCdpIrXvYtANq0Id6gP8zJvUEdPIgNuYxEmVCl9jI+8eGI6peD0qIt8U80hf9axhJ\n3QIDAQAB\n-----END PUBLIC KEY-----\n" + - name: privateKey + value: "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA1KDAM4L8RtJ+nLaXBrBhzVpvTemsKVZoAct8A+ShepOHT9lg\nHOCGLFGWNla6K6j+b3AV/P/fAAhwj82vwTDdruXSflvSdmYeFAw3Ypphc1A5oM53\nwSRWhg63potBNWqdDzj8ApYgqjpmjYSQdL5/a3golb36GYFrY0MLFTv7wZ87pmMI\nPsOgGIcPbCHker2fRZ34WXYLb1hkeUpwx4eKjpwcg35gccvR6o/UhbKAuc60V1J9\nWof2sNgtlRaQej45wnpjWYzZrIyk5qUbn0QiCdpIrXvYtANq0Id6gP8zJvUEdPIg\nNuYxEmVCl9jI+8eGI6peD0qIt8U80hf9axhJ3QIDAQABAoIBAQCKuHnM4ac/eXM7\nQPDVX1vfgyHc3hgBPCtNCHnXfGFRvFBqavKGxIElBvGOcBS0CWQ+Rg1Ca5kMx3TQ\njSweSYhH5A7pe3Sa5FK5V6MGxJvRhMSkQi/lJZUBjzaIBJA9jln7pXzdHx8ekE16\nBMPONr6g2dr4nuI9o67xKrtfViwRDGaG6eh7jIMlEqMMc6WqyhvI67rlVDSTHFKX\njlMcozJ3IT8BtTzKg2Tpy7ReVuJEpehum8yn1ZVdAnotBDJxI07DC1cbOP4M2fHM\ngfgPYWmchauZuTeTFu4hrlY5jg0/WLs6by8r/81+vX3QTNvejX9UdTHMSIfQdX82\nAfkCKUVhAoGBAOvGv+YXeTlPRcYC642x5iOyLQm+BiSX4jKtnyJiTU2s/qvvKkIu\nxAOk3OtniT9NaUAHEZE9tI71dDN6IgTLQlAcPCzkVh6Sc5eG0MObqOO7WOMCWBkI\nlaAKKBbd6cGDJkwGCJKnx0pxC9f8R4dw3fmXWgWAr8ENiekMuvjSfjZ5AoGBAObd\ns2L5uiUPTtpyh8WZ7rEvrun3djBhzi+d7rgxEGdditeiLQGKyZbDPMSMBuus/5wH\nwfi0xUq50RtYDbzQQdC3T/C20oHmZbjWK5mDaLRVzWS89YG/NT2Q8eZLBstKqxkx\ngoT77zoUDfRy+CWs1xvXzgxagD5Yg8/OrCuXOqWFAoGAPIw3r6ELknoXEvihASxU\nS4pwInZYIYGXpygLG8teyrnIVOMAWSqlT8JAsXtPNaBtjPHDwyazfZrvEmEk51JD\nX0tA8M5ah1NYt+r5JaKNxp3P/8wUT6lyszyoeubWJsnFRfSusuq/NRC+1+KDg/aq\nKnSBu7QGbm9JoT2RrmBv5RECgYBRn8Lj1I1muvHTNDkiuRj2VniOSirkUkA2/6y+\nPMKi+SS0tqcY63v4rNCYYTW1L7Yz8V44U5mJoQb4lvpMbolGhPljjxAAU3hVkItb\nvGVRlSCIZHKczADD4rJUDOS7DYxO3P1bjUN4kkyYx+lKUMDBHFzCa2D6Kgt4dobS\n5qYajQKBgQC7u7MFPkkEMqNqNGu5erytQkBq1v1Ipmf9rCi3iIj4XJLopxMgw0fx\n6jwcwNInl72KzoUBLnGQ9PKGVeBcgEgdI+a+tq+1TJo6Ta+hZSx+4AYiKY18eRKG\neNuER9NOcSVJ7Eqkcw4viCGyYDm2vgNV9HJ0VlAo3RDh8x5spEN+mg==\n-----END RSA PRIVATE KEY-----\n" + - name: keys + value: myapp.key + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_three/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_three/pulsar.yml.tmpl new file mode 100644 index 0000000000..c61e7d0a37 --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_three/pulsar.yml.tmpl @@ -0,0 +1,26 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification3 + - name: redeliveryDelay + value: 200ms + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_two/pulsar.yml.tmpl b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_two/pulsar.yml.tmpl new file mode 100644 index 0000000000..06262d2620 --- /dev/null +++ b/tests/certification/pubsub/pulsar/components/auth-oauth2/consumer_two/pulsar.yml.tmpl @@ -0,0 +1,26 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: messagebus +spec: + type: pubsub.pulsar + version: v1 + metadata: + - name: host + value: "localhost:6650" + - name: consumerID + value: certification2 + - name: redeliveryDelay + value: 200ms + - name: oauth2TokenURL + value: https://localhost:8085/issuer1/token + - name: oauth2ClientID + value: foo + - name: oauth2ClientSecret + value: bar + - name: oauth2Scopes + value: openid + - name: oauth2Audiences + value: pulsar + - name: oauth2TokenCAPEM + value: "{{ .OAuth2CAPEM }}" diff --git a/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oauth2-server.yaml b/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oauth2-server.yaml new file mode 100644 index 0000000000..220011d949 --- /dev/null +++ b/tests/certification/pubsub/pulsar/config/docker-compose_auth-mock-oauth2-server.yaml @@ -0,0 +1,14 @@ +# We run in network_mode: "host" so `localhost` is the same for both the host +# and containers. This is required as the mock server uses the SNI hostname to +# build the issuer URL. +version: '3' +services: + mock-oauth2-server: + image: ghcr.io/navikt/mock-oauth2-server:1.0.0 + container_name: mock-oauth2-server + restart: on-failure + network_mode: "host" + environment: + - PORT=8085 + - LOG_LEVEL=DEBUG + - 'JSON_CONFIG={"interactiveLogin":false,"httpServer":{"type":"NettyWrapper","ssl":{}},"tokenCallbacks":[{"issuerId":"issuer1","tokenExpiry":120,"requestMappings":[{"requestParam":"scope","match":"openid","claims":{"sub":"foo","aud":["pulsar"]}}]}]}' diff --git a/tests/certification/pubsub/pulsar/docker-compose.yml b/tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml similarity index 97% rename from tests/certification/pubsub/pulsar/docker-compose.yml rename to tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml index 5ae0535596..7857b77961 100644 --- a/tests/certification/pubsub/pulsar/docker-compose.yml +++ b/tests/certification/pubsub/pulsar/config/docker-compose_auth-none.yaml @@ -16,4 +16,4 @@ services: - pulsarconf:/pulsar/conf volumes: pulsardata: - pulsarconf: \ No newline at end of file + pulsarconf: diff --git a/tests/certification/pubsub/pulsar/config/docker-compose_auth-oauth2.yaml.tmpl b/tests/certification/pubsub/pulsar/config/docker-compose_auth-oauth2.yaml.tmpl new file mode 100644 index 0000000000..55ae595957 --- /dev/null +++ b/tests/certification/pubsub/pulsar/config/docker-compose_auth-oauth2.yaml.tmpl @@ -0,0 +1,96 @@ +# We run the pulsar services individually as OAuth2 doesn't seem to work in +# standalone mode. OAuth2 is also only available from pulsar v3 onwards. We use +# host networking as the mock OAuth server uses the SNI host name to determine +# the host name of the OAuth2 issuer URL, so we need to have the mock server +# reachable by localhost from both the pulsar services and the host network. +version: '3' +services: + # Start zookeeper + zookeeper: + image: apachepulsar/pulsar:3.0.0 + container_name: zookeeper + restart: on-failure + network_mode: "host" + environment: + - metadataStoreUrl=zk:localhost:2181 + - metricsProvider.httpPort=7000 + - PULSAR_MEM=-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m + command: > + bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \ + bin/generate-zookeeper-config.sh conf/zookeeper.conf && \ + exec bin/pulsar zookeeper" + healthcheck: + test: ["CMD", "bin/pulsar-zookeeper-ruok.sh"] + interval: 1s + timeout: 5s + retries: 300 + + # Init cluster metadata + pulsar-init: + container_name: pulsar-init + image: apachepulsar/pulsar:3.0.0 + network_mode: "host" + env_file: + - ./pulsar_auth-oauth2.conf + command: > + bin/pulsar initialize-cluster-metadata \ + --cluster cluster-a \ + --zookeeper localhost:2181 \ + --configuration-store localhost:2181 \ + --web-service-url http://localhost:8080 \ + --broker-service-url pulsar://localhost:6650 + depends_on: + zookeeper: + condition: service_healthy + + # Start bookie + bookie: + image: apachepulsar/pulsar:3.0.0 + container_name: bookie + restart: on-failure + network_mode: "host" + environment: + - clusterName=cluster-a + - zkServers=localhost:2181 + - metadataServiceUri=metadata-store:zk:localhost:2181 + # otherwise every time we run docker compose uo or down we fail to start due to Cookie + # See: https://github.com/apache/bookkeeper/blob/405e72acf42bb1104296447ea8840d805094c787/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Cookie.java#L57-68 + - advertisedAddress=localhost + - BOOKIE_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m + env_file: + - ./pulsar_auth-oauth2.conf + volumes: + - "{{ .TmpDir }}:/pulsar/conf/dapr" + depends_on: + zookeeper: + condition: service_healthy + pulsar-init: + condition: service_completed_successfully + command: bash -c "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + + # Start broker + broker: + image: apachepulsar/pulsar:3.0.0 + container_name: broker + restart: on-failure + network_mode: "host" + env_file: + - ./pulsar_auth-oauth2.conf + volumes: + - "{{ .TmpDir }}:/pulsar/conf/dapr" + environment: + - metadataStoreUrl=zk:localhost:2181 + - zookeeperServers=localhost:2181 + - clusterName=cluster-a + - managedLedgerDefaultEnsembleSize=1 + - managedLedgerDefaultWriteQuorum=1 + - managedLedgerDefaultAckQuorum=1 + - advertisedAddress=localhost + - advertisedListeners=external:pulsar://127.0.0.1:6650 + - PULSAR_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m + depends_on: + zookeeper: + condition: service_healthy + bookie: + condition: service_started + command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" diff --git a/tests/certification/pubsub/pulsar/config/pulsar_auth-oauth2.conf b/tests/certification/pubsub/pulsar/config/pulsar_auth-oauth2.conf new file mode 100644 index 0000000000..a9f35e4a7e --- /dev/null +++ b/tests/certification/pubsub/pulsar/config/pulsar_auth-oauth2.conf @@ -0,0 +1,39 @@ +# Configuration to enable authentication +authenticationEnabled=true +authenticationProviders=org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID + +# Required settings for AuthenticationProviderOpenID +# A comma separated list of allowed, or trusted, token issuers. The token issuer is the URL of the token issuer. +PULSAR_PREFIX_openIDAllowedTokenIssuers=https://localhost:8085/issuer1 +# The list of allowed audiences for the token. The audience is the intended recipient of the token. A token with +# at least one of these audience claims will pass the audience validation check. +PULSAR_PREFIX_openIDAllowedAudiences=pulsar + +# Optional settings (values shown are the defaults) +# The path to the file containing the trusted certificate(s) of the token issuer(s). If not set, uses the default +# trust store of the JVM. Note: in version 3.0.0, the default only applies when this setting is not an environment +# variable and is not in the configuration file. +PULSAR_PREFIX_openIDTokenIssuerTrustCertsFilePath=/pulsar/conf/dapr/ca.pem +# The JWT's claim to use for the role/principal during authorization. +PULSAR_PREFIX_openIDRoleClaim=sub +# The leeway, in seconds, to use when validating the token's expiration time. +PULSAR_PREFIX_openIDAcceptedTimeLeewaySeconds=0 + +# Cache settings +PULSAR_PREFIX_openIDCacheSize=5 +PULSAR_PREFIX_openIDCacheRefreshAfterWriteSeconds=64800 +PULSAR_PREFIX_openIDCacheExpirationSeconds=86400 +PULSAR_PREFIX_openIDHttpConnectionTimeoutMillis=10000 +PULSAR_PREFIX_openIDHttpReadTimeoutMillis=10000 + +# The number of seconds to wait before refreshing the JWKS when a token presents a key ID (kid claim) that is not +# in the cache. This setting is available from Pulsar 3.0.1 and is documented below. +PULSAR_PREFIX_openIDKeyIdCacheMissRefreshSeconds=300 + +# Whether to require that issuers use HTTPS. It is part of the OAuth2 spec to use HTTPS, so the default is true. +# This setting is for testing purposes and is not recommended for any production environment. +#PULSAR_PREFIX_openIDRequireIssuersUseHttps=false + +# A setting describing how to handle discovery of the OpenID Connect configuration document when the issuer is not +# in the list of allowed issuers. This setting is documented below. +PULSAR_PREFIX_openIDFallbackDiscoveryMode=DISABLED diff --git a/tests/certification/pubsub/pulsar/pulsar_test.go b/tests/certification/pubsub/pulsar/pulsar_test.go index 9c7fd3e405..df170e5678 100644 --- a/tests/certification/pubsub/pulsar/pulsar_test.go +++ b/tests/certification/pubsub/pulsar/pulsar_test.go @@ -16,17 +16,28 @@ package pulsar_test import ( "bytes" "context" + "crypto/tls" "encoding/json" + "encoding/pem" "fmt" + "io" + "io/fs" "io/ioutil" "net/http" + "os" + "os/exec" + "path/filepath" + "strings" "testing" + "text/template" "time" "github.com/google/uuid" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "go.uber.org/multierr" + "github.com/dapr/components-contrib/internal/authentication/oauth2" pubsub_pulsar "github.com/dapr/components-contrib/pubsub/pulsar" pubsub_loader "github.com/dapr/dapr/pkg/components/pubsub" @@ -54,21 +65,23 @@ const ( appID1 = "app-1" appID2 = "app-2" - numMessages = 10 - appPort = 8000 - portOffset = 2 - messageKey = "partitionKey" - pubsubName = "messagebus" - topicActiveName = "certification-pubsub-topic-active" - topicPassiveName = "certification-pubsub-topic-passive" - topicToBeCreated = "certification-topic-per-test-run" - topicDefaultName = "certification-topic-default" - topicMultiPartitionName = "certification-topic-multi-partition8" - partition0 = "partition-0" - partition1 = "partition-1" - clusterName = "pulsarcertification" - dockerComposeYAML = "docker-compose.yml" - pulsarURL = "localhost:6650" + numMessages = 10 + appPort = 8001 + portOffset = 2 + messageKey = "partitionKey" + pubsubName = "messagebus" + topicActiveName = "certification-pubsub-topic-active" + topicPassiveName = "certification-pubsub-topic-passive" + topicToBeCreated = "certification-topic-per-test-run" + topicDefaultName = "certification-topic-default" + topicMultiPartitionName = "certification-topic-multi-partition8" + partition0 = "partition-0" + partition1 = "partition-1" + clusterName = "pulsarcertification" + dockerComposeAuthNoneYAML = "./config/docker-compose_auth-none.yaml" + dockerComposeAuthOAuth2YAML = "./config/docker-compose_auth-oauth2.yaml.tmpl" + dockerComposeMockOAuth2YAML = "./config/docker-compose_auth-mock-oauth2-server.yaml" + pulsarURL = "localhost:6650" subscribeTypeKey = "subscribeType" @@ -82,6 +95,103 @@ const ( processModeSync = "sync" ) +type pulsarSuite struct { + suite.Suite + + authType string + oauth2CAPEM []byte + dockerComposeYAML string + componentsPath string + services []string +} + +func TestPulsar(t *testing.T) { + t.Run("Auth:None", func(t *testing.T) { + suite.Run(t, &pulsarSuite{ + authType: "none", + dockerComposeYAML: dockerComposeAuthNoneYAML, + componentsPath: "./components/auth-none", + services: []string{"standalone"}, + }) + }) + + t.Run("Auth:OAuth2", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.Chmod(dir, 0o777)) + + t.Log("Starting OAuth2 server...") + out, err := exec.Command( + "docker-compose", + "-p", "oauth2", + "-f", dockerComposeMockOAuth2YAML, + "up", "-d").CombinedOutput() + require.NoError(t, err, string(out)) + t.Log(string(out)) + + t.Cleanup(func() { + t.Log("Stopping OAuth2 server...") + out, err = exec.Command( + "docker-compose", + "-p", "oauth2", + "-f", dockerComposeMockOAuth2YAML, + "down", "-v", + "--remove-orphans").CombinedOutput() + require.NoError(t, err, string(out)) + t.Log(string(out)) + }) + + t.Log("Waiting for OAuth server to be ready...") + oauth2CA := peerCertificate(t, "localhost:8085") + t.Log("OAuth server is ready") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.pem"), oauth2CA, 0o644)) + outf, err := os.OpenFile("./config/pulsar_auth-oauth2.conf", os.O_RDONLY, 0o644) + require.NoError(t, err) + inf, err := os.OpenFile(filepath.Join(dir, "pulsar_auth-oauth2.conf"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + require.NoError(t, err) + _, err = io.Copy(inf, outf) + require.NoError(t, err) + outf.Close() + inf.Close() + + td := struct { + TmpDir string + OAuth2CAPEM string + }{ + TmpDir: dir, + OAuth2CAPEM: strings.ReplaceAll(string(oauth2CA), "\n", "\\n"), + } + + tmpl, err := template.New("").ParseFiles(dockerComposeAuthOAuth2YAML) + require.NoError(t, err) + f, err := os.OpenFile(filepath.Join(dir, "docker-compose.yaml"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + require.NoError(t, err) + require.NoError(t, tmpl.ExecuteTemplate(f, "docker-compose_auth-oauth2.yaml.tmpl", td)) + + require.NoError(t, filepath.Walk("./components/auth-oauth2", func(path string, info fs.FileInfo, err error) error { + if info.IsDir() { + return nil + } + tmpl, err := template.New("").ParseFiles(path) + require.NoError(t, err) + path = strings.TrimSuffix(path, ".tmpl") + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, path)), 0o755)) + f, err := os.OpenFile(filepath.Join(dir, path), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + require.NoError(t, err) + require.NoError(t, tmpl.ExecuteTemplate(f, filepath.Base(path)+".tmpl", td)) + return nil + })) + + suite.Run(t, &pulsarSuite{ + oauth2CAPEM: oauth2CA, + authType: "oauth2", + dockerComposeYAML: filepath.Join(dir, "docker-compose.yaml"), + componentsPath: filepath.Join(dir, "components/auth-oauth2"), + services: []string{"zookeeper", "pulsar-init", "bookie", "broker"}, + }) + }) +} + func subscriberApplication(appID string, topicName string, messagesWatcher *watcher.Watcher) app.SetupFn { return func(ctx flow.Context, s common.Service) error { // Simulate periodic errors. @@ -196,7 +306,8 @@ func assertMessages(timeout time.Duration, messageWatchers ...*watcher.Watcher) } } -func TestPulsar(t *testing.T) { +func (p *pulsarSuite) TestPulsar() { + t := p.T() consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -244,10 +355,10 @@ func TestPulsar(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -267,7 +378,7 @@ func TestPulsar(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -280,7 +391,7 @@ func TestPulsar(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -295,7 +406,8 @@ func TestPulsar(t *testing.T) { Run() } -func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { +func (p *pulsarSuite) TestPulsarMultipleSubsSameConsumerIDs() { + t := p.T() consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -312,10 +424,10 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -335,7 +447,7 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -348,7 +460,7 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -362,7 +474,9 @@ func TestPulsarMultipleSubsSameConsumerIDs(t *testing.T) { Run() } -func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { +func (p *pulsarSuite) TestPulsarMultipleSubsDifferentConsumerIDs() { + t := p.T() + consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -376,10 +490,10 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -401,7 +515,7 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -414,7 +528,7 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -427,7 +541,8 @@ func TestPulsarMultipleSubsDifferentConsumerIDs(t *testing.T) { Run() } -func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { +func (p *pulsarSuite) TestPulsarMultiplePubSubsDifferentConsumerIDs() { + t := p.T() consumerGroup1 := watcher.NewUnordered() consumerGroup2 := watcher.NewUnordered() @@ -445,10 +560,10 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -470,7 +585,7 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -483,7 +598,7 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -498,7 +613,8 @@ func TestPulsarMultiplePubSubsDifferentConsumerIDs(t *testing.T) { Run() } -func TestPulsarNonexistingTopic(t *testing.T) { +func (p *pulsarSuite) TestPulsarNonexistingTopic() { + t := p.T() consumerGroup1 := watcher.NewUnordered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -511,10 +627,10 @@ func TestPulsarNonexistingTopic(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset*3), subscriberApplication(appID1, topicToBeCreated, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -537,7 +653,7 @@ func TestPulsarNonexistingTopic(t *testing.T) { })). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset*3), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*3), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*3), @@ -550,7 +666,8 @@ func TestPulsarNonexistingTopic(t *testing.T) { Run() } -func TestPulsarNetworkInterruption(t *testing.T) { +func (p *pulsarSuite) TestPulsarNetworkInterruption() { + t := p.T() consumerGroup1 := watcher.NewUnordered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -563,10 +680,10 @@ func TestPulsarNetworkInterruption(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -589,7 +706,7 @@ func TestPulsarNetworkInterruption(t *testing.T) { })). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -603,7 +720,8 @@ func TestPulsarNetworkInterruption(t *testing.T) { Run() } -func TestPulsarPersitant(t *testing.T) { +func (p *pulsarSuite) TestPulsarPersitant() { + t := p.T() consumerGroup1 := watcher.NewUnordered() flow.New(t, "pulsar certification persistant test"). @@ -611,10 +729,10 @@ func TestPulsarPersitant(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -636,23 +754,24 @@ func TestPulsarPersitant(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), componentRuntimeOptions(), )). Step("publish messages to topic1", publishMessages(nil, sidecarName1, topicActiveName, consumerGroup1)). - Step("stop pulsar server", dockercompose.Stop(clusterName, dockerComposeYAML, "standalone")). + Step("stop pulsar server", dockercompose.Stop(clusterName, p.dockerComposeYAML, p.services...)). Step("wait", flow.Sleep(5*time.Second)). - Step("start pulsar server", dockercompose.Start(clusterName, dockerComposeYAML, "standalone")). + Step("start pulsar server", dockercompose.Start(clusterName, p.dockerComposeYAML, p.services...)). Step("wait", flow.Sleep(10*time.Second)). Step("verify if app1 has received messages published to active topic", assertMessages(10*time.Second, consumerGroup1)). Step("reset", flow.Reset(consumerGroup1)). Run() } -func TestPulsarDelay(t *testing.T) { +func (p *pulsarSuite) TestPulsarDelay() { + t := p.T() consumerGroup1 := watcher.NewUnordered() date := time.Now() @@ -682,10 +801,10 @@ func TestPulsarDelay(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -707,7 +826,7 @@ func TestPulsarDelay(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_three"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_three")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -730,7 +849,8 @@ type schemaTest struct { Name string `json:"name"` } -func TestPulsarSchema(t *testing.T) { +func (p *pulsarSuite) TestPulsarSchema() { + t := p.T() consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -772,10 +892,10 @@ func TestPulsarSchema(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -795,7 +915,7 @@ func TestPulsarSchema(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_four"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_four")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -818,7 +938,7 @@ func componentRuntimeOptions() []runtime.Option { } } -func createMultiPartitionTopic(tenant, namespace, topic string, partition int) flow.Runnable { +func (p *pulsarSuite) createMultiPartitionTopic(tenant, namespace, topic string, partition int) flow.Runnable { return func(ctx flow.Context) error { reqURL := fmt.Sprintf("http://localhost:8080/admin/v2/persistent/%s/%s/%s/partitions", tenant, namespace, topic) @@ -838,6 +958,19 @@ func createMultiPartitionTopic(tenant, namespace, topic string, partition int) f req.Header.Set("Content-Type", "application/json") + if p.authType == "oauth2" { + cc, err := p.oauth2ClientCredentials() + if err != nil { + return err + } + token, err := cc.Token() + if err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + } + rsp, err := http.DefaultClient.Do(req) if err != nil { @@ -858,7 +991,8 @@ func createMultiPartitionTopic(tenant, namespace, topic string, partition int) f } } -func TestPulsarPartitionedOrderingProcess(t *testing.T) { +func (p *pulsarSuite) TestPulsarPartitionedOrderingProcess() { + t := p.T() consumerGroup1 := watcher.NewOrdered() // Set the partition key on all messages so they are written to the same partition. This allows for checking of ordered messages. @@ -867,14 +1001,14 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { } flow.New(t, "pulsar certification - process message in order with partitioned-topic"). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort+portOffset), subscriberApplicationWithoutError(appID1, topicMultiPartitionName, consumerGroup1))). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -897,10 +1031,10 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { return err })). Step("create multi-partition topic explicitly", retry.Do(10*time.Second, 30, - createMultiPartitionTopic("public", "default", topicMultiPartitionName, 4))). + p.createMultiPartitionTopic("public", "default", topicMultiPartitionName, 4))). // Run the Dapr sidecar with the component entitymanagement Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_one"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_one")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset), @@ -913,7 +1047,7 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { // Run the Dapr sidecar with the component 2. Step(sidecar.Run(sidecarName2, - embedded.WithComponentsPath("./components/consumer_two"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_two")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort+portOffset*3), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort+portOffset*3), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort+portOffset*3), @@ -927,7 +1061,8 @@ func TestPulsarPartitionedOrderingProcess(t *testing.T) { Run() } -func TestPulsarEncryptionFromFile(t *testing.T) { +func (p *pulsarSuite) TestPulsarEncryptionFromFile() { + t := p.T() consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -969,10 +1104,10 @@ func TestPulsarEncryptionFromFile(t *testing.T) { // Run subscriberApplication app1 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -992,7 +1127,7 @@ func TestPulsarEncryptionFromFile(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_five"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_five")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -1004,7 +1139,8 @@ func TestPulsarEncryptionFromFile(t *testing.T) { Run() } -func TestPulsarEncryptionFromData(t *testing.T) { +func (p *pulsarSuite) TestPulsarEncryptionFromData() { + t := p.T() consumerGroup1 := watcher.NewUnordered() publishMessages := func(sidecarName string, topicName string, messageWatchers ...*watcher.Watcher) flow.Runnable { @@ -1046,10 +1182,10 @@ func TestPulsarEncryptionFromData(t *testing.T) { // Run subscriberApplication app2 Step(app.Run(appID1, fmt.Sprintf(":%d", appPort), subscriberSchemaApplication(appID1, topicActiveName, consumerGroup1))). - Step(dockercompose.Run(clusterName, dockerComposeYAML)). + Step(dockercompose.Run(clusterName, p.dockerComposeYAML)). Step("wait", flow.Sleep(10*time.Second)). Step("wait for pulsar readiness", retry.Do(10*time.Second, 30, func(ctx flow.Context) error { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) + client, err := p.client(t) if err != nil { return fmt.Errorf("could not create pulsar client: %v", err) } @@ -1069,7 +1205,7 @@ func TestPulsarEncryptionFromData(t *testing.T) { return err })). Step(sidecar.Run(sidecarName1, - embedded.WithComponentsPath("./components/consumer_six"), + embedded.WithComponentsPath(filepath.Join(p.componentsPath, "consumer_six")), embedded.WithAppProtocol(runtime.HTTPProtocol, appPort), embedded.WithDaprGRPCPort(runtime.DefaultDaprAPIGRPCPort), embedded.WithDaprHTTPPort(runtime.DefaultDaprHTTPPort), @@ -1080,3 +1216,57 @@ func TestPulsarEncryptionFromData(t *testing.T) { Step("reset", flow.Reset(consumerGroup1)). Run() } + +func (p *pulsarSuite) client(t *testing.T) (pulsar.Client, error) { + t.Helper() + + opts := pulsar.ClientOptions{ + URL: "pulsar://localhost:6650", + } + switch p.authType { + case "oauth2": + cc, err := p.oauth2ClientCredentials() + require.NoError(t, err) + opts.Authentication = pulsar.NewAuthenticationTokenFromSupplier(cc.Token) + default: + } + + return pulsar.NewClient(opts) +} + +func (p *pulsarSuite) oauth2ClientCredentials() (*oauth2.ClientCredentials, error) { + cc, err := oauth2.NewClientCredentials(context.Background(), oauth2.ClientCredentialsOptions{ + Logger: logger.NewLogger("dapr.test.readiness"), + TokenURL: "https://localhost:8085/issuer1/token", + ClientID: "foo", + ClientSecret: "bar", + Scopes: []string{"openid"}, + Audiences: []string{"pulsar"}, + CAPEM: p.oauth2CAPEM, + }) + if err != nil { + return nil, err + } + + return cc, nil +} + +func peerCertificate(t *testing.T, hostport string) []byte { + conf := &tls.Config{InsecureSkipVerify: true} + + for { + time.Sleep(1 * time.Second) + + conn, err := tls.Dial("tcp", hostport, conf) + if err != nil { + t.Log(err) + continue + } + + defer conn.Close() + + certs := conn.ConnectionState().PeerCertificates + require.Len(t, certs, 1, "expected 1 peer certificate") + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certs[0].Raw}) + } +} diff --git a/tests/config/state/aws/dynamodb/terraform/statestore.yml b/tests/config/state/aws/dynamodb/terraform/statestore.yml index 5f708ad74e..ea04cda14e 100644 --- a/tests/config/state/aws/dynamodb/terraform/statestore.yml +++ b/tests/config/state/aws/dynamodb/terraform/statestore.yml @@ -13,4 +13,6 @@ spec: - name: region value: "us-east-1" - name: table - value: ${{STATE_AWS_DYNAMODB_TABLE_1}} \ No newline at end of file + value: ${{STATE_AWS_DYNAMODB_TABLE_1}} + - name: ttlAttributeName + value: "expiresAt" \ No newline at end of file diff --git a/tests/config/state/tests.yml b/tests/config/state/tests.yml index 199a540c88..6228d34db4 100644 --- a/tests/config/state/tests.yml +++ b/tests/config/state/tests.yml @@ -75,9 +75,10 @@ components: - component: in-memory operations: [ "transaction", "etag", "first-write", "ttl" ] - component: aws.dynamodb.docker + # In the Docker variant, we do not set ttlAttributeName in the metadata, so TTLs are not enabled operations: [ "transaction", "etag", "first-write" ] - component: aws.dynamodb.terraform - operations: [ "transaction", "etag", "first-write" ] + operations: [ "transaction", "etag", "first-write", "ttl" ] - component: etcd.v1 operations: [ "transaction", "etag", "first-write", "ttl" ] - component: etcd.v2 diff --git a/tests/conformance/standalone_loader.go b/tests/conformance/standalone_loader.go index a32cde4edd..ef87dd9e4e 100644 --- a/tests/conformance/standalone_loader.go +++ b/tests/conformance/standalone_loader.go @@ -16,13 +16,14 @@ package conformance import ( "bufio" "bytes" + "errors" "io" "log" "os" "path/filepath" "strings" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) const ( @@ -63,7 +64,7 @@ func (s *StandaloneComponents) LoadComponents() ([]Component, error) { continue } - components, _ := s.decodeYaml(path, b) + components := s.decodeYaml(path, b) list = append(list, components...) } } @@ -82,17 +83,15 @@ func (s *StandaloneComponents) isYaml(fileName string) bool { } // decodeYaml decodes the yaml document. -func (s *StandaloneComponents) decodeYaml(filename string, b []byte) ([]Component, []error) { +func (s *StandaloneComponents) decodeYaml(filename string, b []byte) []Component { list := []Component{} - errors := []error{} scanner := bufio.NewScanner(bytes.NewReader(b)) scanner.Split(s.splitYamlDoc) for { var comp Component - comp.Spec = ComponentSpec{} err := s.decode(scanner, &comp) - if err == io.EOF { + if errors.Is(err, io.EOF) { break } @@ -103,13 +102,13 @@ func (s *StandaloneComponents) decodeYaml(filename string, b []byte) ([]Componen list = append(list, comp) } - return list, errors + return list } // decode reads the YAML resource in document. -func (s *StandaloneComponents) decode(scanner *bufio.Scanner, c interface{}) error { +func (s *StandaloneComponents) decode(scanner *bufio.Scanner, c any) error { if scanner.Scan() { - return yaml.Unmarshal(scanner.Bytes(), &c) + return yaml.Unmarshal(scanner.Bytes(), c) } err := scanner.Err() diff --git a/tests/conformance/standalone_loader_test.go b/tests/conformance/standalone_loader_test.go index 61c0e1f76a..18b3df8b66 100644 --- a/tests/conformance/standalone_loader_test.go +++ b/tests/conformance/standalone_loader_test.go @@ -17,6 +17,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestStandaloneIsYaml(t *testing.T) { @@ -48,12 +49,11 @@ spec: - name: prop2 value: value2 ` - components, errs := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) - assert.Len(t, components, 1) - assert.Empty(t, errs) + components := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) + require.Len(t, components, 1) assert.Equal(t, "statestore", components[0].Name) assert.Equal(t, "state.couchbase", components[0].Spec.Type) - assert.Len(t, components[0].Spec.Metadata, 2) + require.Len(t, components[0].Spec.Metadata, 2) assert.Equal(t, "prop1", components[0].Spec.Metadata[0].Name) assert.Equal(t, "value1", components[0].Spec.Metadata[0].Value.String()) } @@ -72,17 +72,15 @@ spec: - name: prop2 value: value2 ` - components, errs := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) + components := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) assert.Len(t, components, 0) - assert.Len(t, errs, 0) } func TestStandaloneDecodeUnsuspectingFile(t *testing.T) { request := NewStandaloneComponents("test_component_path") - components, errs := request.decodeYaml("components/messagebus.yaml", []byte("hey there")) + components := request.decodeYaml("components/messagebus.yaml", []byte("hey there")) assert.Len(t, components, 0) - assert.Len(t, errs, 0) } func TestStandaloneDecodeInvalidYaml(t *testing.T) { @@ -93,9 +91,8 @@ apiVersion: dapr.io/v1alpha1 kind: Component metadata: name: statestore` - components, errs := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) + components := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) assert.Len(t, components, 0) - assert.Len(t, errs, 0) } func TestStandaloneDecodeValidMultiYaml(t *testing.T) { @@ -123,9 +120,8 @@ spec: - name: prop3 value: value3 ` - components, errs := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) + components := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) assert.Len(t, components, 2) - assert.Empty(t, errs) assert.Equal(t, "statestore1", components[0].Name) assert.Equal(t, "state.couchbase", components[0].Spec.Type) assert.Len(t, components[0].Spec.Metadata, 2) @@ -172,9 +168,8 @@ spec: - name: prop3 value: value3 ` - components, errs := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) + components := request.decodeYaml("components/messagebus.yaml", []byte(yaml)) assert.Len(t, components, 2) - assert.Len(t, errs, 0) assert.Equal(t, "statestore1", components[0].Name) assert.Equal(t, "state.couchbase", components[0].Spec.Type) diff --git a/tests/conformance/state/state.go b/tests/conformance/state/state.go index bbc2f78d0c..93c0a25729 100644 --- a/tests/conformance/state/state.go +++ b/tests/conformance/state/state.go @@ -990,7 +990,21 @@ func ConformanceTests(t *testing.T, props map[string]string, statestore state.St } if config.HasOperation("ttl") { + t.Run("set ttl with bad value should error", func(t *testing.T) { + require.Error(t, statestore.Set(context.Background(), &state.SetRequest{ + Key: key + "-ttl", + Value: "⏱️", + Metadata: map[string]string{ + "ttlInSeconds": "foo", + }, + })) + }) + t.Run("set and get with TTL", func(t *testing.T) { + // Check if ttl feature is listed + features := statestore.Features() + require.True(t, state.FeatureTTL.IsPresent(features)) + err := statestore.Set(context.Background(), &state.SetRequest{ Key: key + "-ttl", Value: "⏱️", @@ -1016,6 +1030,194 @@ func ConformanceTests(t *testing.T, props map[string]string, statestore state.St return res.Data == nil }, time.Second*3, 200*time.Millisecond, "expected object to have been deleted in time") }) + } else { + t.Run("ttl feature not present", func(t *testing.T) { + // We skip this check for Cloudflare Workers KV + // Even though the component supports TTLs, it's not tested in the conformance tests because the minimum TTL for the component is 1 minute, and the state store doesn't have strong consistency + if config.ComponentName == "cloudflare.workerskv" { + t.Skip() + } + + features := statestore.Features() + require.False(t, state.FeatureTTL.IsPresent(features)) + }) + + t.Run("no TTL should not return any expire time", func(t *testing.T) { + err := statestore.Set(context.Background(), &state.SetRequest{ + Key: key + "-no-ttl", + Value: "⏱️", + Metadata: map[string]string{}, + }) + require.NoError(t, err) + + // Request immediately + res, err := statestore.Get(context.Background(), &state.GetRequest{Key: key + "-no-ttl"}) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + + assert.NotContains(t, res.Metadata, "ttlExpireTime") + }) + + t.Run("ttlExpireTime", func(t *testing.T) { + if !config.HasOperation("transaction") { + // This test is only for state stores that support transactions + return + } + + unsupported := []string{ + "redis.v6", + "redis.v7", + "etcd.v1", + } + + for _, noSup := range unsupported { + if strings.Contains(config.ComponentName, noSup) { + t.Skipf("skipping test for unsupported state store %s", noSup) + } + } + + t.Run("set and get expire time", func(t *testing.T) { + now := time.Now() + err := statestore.Set(context.Background(), &state.SetRequest{ + Key: key + "-ttl-expire-time", + Value: "⏱️", + Metadata: map[string]string{ + // Expire in an hour. + "ttlInSeconds": "3600", + }, + }) + require.NoError(t, err) + + // Request immediately + res, err := statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + + require.Containsf(t, res.Metadata, "ttlExpireTime", "expected metadata to contain ttlExpireTime") + expireTime, err := time.Parse(time.RFC3339, res.Metadata["ttlExpireTime"]) + require.NoError(t, err) + assert.InDelta(t, now.Add(time.Hour).UnixMilli(), expireTime.UnixMilli(), float64(time.Minute*10)) + }) + + t.Run("ttl set to -1 should remove the TTL of a state store key", func(t *testing.T) { + req := func(meta map[string]string) *state.SetRequest { + return &state.SetRequest{ + Key: key + "-ttl-expire-time-minus-1", + Value: "⏱️", + Metadata: meta, + } + } + + require.NoError(t, statestore.Set(context.Background(), req(map[string]string{ + // Expire in 2 seconds. + "ttlInSeconds": "2", + }))) + + // Request immediately + res, err := statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + assert.Contains(t, res.Metadata, "ttlExpireTime") + + // Remove TTL by setting a value of -1. + require.NoError(t, statestore.Set(context.Background(), req(map[string]string{ + "ttlInSeconds": "-1", + }))) + res, err = statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + assert.NotContains(t, res.Metadata, "ttlExpireTime") + + // Ensure that the key is not expired after previous TTL. + time.Sleep(3 * time.Second) + + res, err = statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + + // Set a new TTL. + require.NoError(t, statestore.Set(context.Background(), req(map[string]string{ + "ttlInSeconds": "2", + }))) + res, err = statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + assert.Contains(t, res.Metadata, "ttlExpireTime") + + // Remove TTL by omitting the ttlInSeconds field. + require.NoError(t, statestore.Set(context.Background(), req(map[string]string{}))) + res, err = statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + assert.NotContains(t, res.Metadata, "ttlExpireTime") + + // Ensure key is not expired after previous TTL. + time.Sleep(3 * time.Second) + res, err = statestore.Get(context.Background(), &state.GetRequest{ + Key: key + "-ttl-expire-time-minus-1", + }) + require.NoError(t, err) + assertEquals(t, "⏱️", res) + assert.NotContains(t, res.Metadata, "ttlExpireTime") + }) + + t.Run("set and get expire time bulkGet", func(t *testing.T) { + now := time.Now() + require.NoError(t, statestore.Set(context.Background(), &state.SetRequest{ + Key: key + "-ttl-expire-time-bulk-1", + Value: "123", + Metadata: map[string]string{"ttlInSeconds": "3600"}, + })) + + require.NoError(t, statestore.Set(context.Background(), &state.SetRequest{ + Key: key + "-ttl-expire-time-bulk-2", + Value: "234", + Metadata: map[string]string{"ttlInSeconds": "3600"}, + })) + + // Request immediately + res, err := statestore.BulkGet(context.Background(), []state.GetRequest{ + {Key: key + "-ttl-expire-time-bulk-1"}, + {Key: key + "-ttl-expire-time-bulk-2"}, + }, state.BulkGetOpts{}) + require.NoError(t, err) + + require.Len(t, res, 2) + sort.Slice(res, func(i, j int) bool { + return res[i].Key < res[j].Key + }) + + assert.Equal(t, key+"-ttl-expire-time-bulk-1", res[0].Key) + assert.Equal(t, key+"-ttl-expire-time-bulk-2", res[1].Key) + assert.Equal(t, []byte(`"123"`), res[0].Data) + assert.Equal(t, []byte(`"234"`), res[1].Data) + + for i := range res { + if config.HasOperation("transaction") { + require.Containsf(t, res[i].Metadata, "ttlExpireTime", "expected metadata to contain ttlExpireTime") + expireTime, err := time.Parse(time.RFC3339, res[i].Metadata["ttlExpireTime"]) + require.NoError(t, err) + // Check the expire time is returned and is in a 10 minute window. This + // window should be _more_ than enough. + assert.InDelta(t, now.Add(time.Hour).UnixMilli(), expireTime.UnixMilli(), float64(time.Minute*10)) + } else { + assert.NotContains(t, res[i].Metadata, "ttlExpireTime") + } + } + }) + }) } } diff --git a/tests/e2e/pubsub/jetstream/go.mod b/tests/e2e/pubsub/jetstream/go.mod index 90a6ffa8b7..9e1f8d9c97 100644 --- a/tests/e2e/pubsub/jetstream/go.mod +++ b/tests/e2e/pubsub/jetstream/go.mod @@ -24,7 +24,7 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cast v1.5.1 // indirect golang.org/x/crypto v0.11.0 // indirect - golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb // indirect + golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b // indirect golang.org/x/sys v0.10.0 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/tests/e2e/pubsub/jetstream/go.sum b/tests/e2e/pubsub/jetstream/go.sum index c760387b14..7c0a3cb5c4 100644 --- a/tests/e2e/pubsub/jetstream/go.sum +++ b/tests/e2e/pubsub/jetstream/go.sum @@ -56,8 +56,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=