diff --git a/.build-tools/pkg/metadataschema/builtin-authentication-profiles.go b/.build-tools/pkg/metadataschema/builtin-authentication-profiles.go index b49dd3b8a8..d2fb6983b8 100644 --- a/.build-tools/pkg/metadataschema/builtin-authentication-profiles.go +++ b/.build-tools/pkg/metadataschema/builtin-authentication-profiles.go @@ -20,6 +20,34 @@ import ( // ParseBuiltinAuthenticationProfile returns an AuthenticationProfile(s) from a given BuiltinAuthenticationProfile. func ParseBuiltinAuthenticationProfile(bi BuiltinAuthenticationProfile) ([]AuthenticationProfile, error) { switch bi.Name { + case "aws": + return []AuthenticationProfile{ + { + Title: "AWS: Access Key ID and Secret Access Key", + Description: "Authenticate using an Access Key ID and Secret Access Key included in the metadata", + Metadata: []Metadata{ + { + Name: "accessKey", + Required: true, + Sensitive: true, + Description: "AWS access key associated with an IAM account", + Example: `"AKIAIOSFODNN7EXAMPLE"`, + }, + { + Name: "secretKey", + Required: true, + Sensitive: true, + Description: "The secret key associated with the access key", + Example: `"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"`, + }, + }, + }, + { + Title: "AWS: Credentials from Environment Variables", + Description: "Use AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY from the environment", + Metadata: []Metadata{}, + }, + }, nil case "azuread": azureEnvironmentMetadata := Metadata{ Name: "azureEnvironment", diff --git a/.github/infrastructure/conformance/azure/conf-test-azure-postgres.bicep b/.github/infrastructure/conformance/azure/conf-test-azure-postgres.bicep new file mode 100644 index 0000000000..9d92fdd952 --- /dev/null +++ b/.github/infrastructure/conformance/azure/conf-test-azure-postgres.bicep @@ -0,0 +1,68 @@ +// ------------------------------------------------------------------------ +// Copyright 2021 The Dapr Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------ + +param postgresServerName string +param sdkAuthSpId string +param sdkAuthSpName string +param rgLocation string = resourceGroup().location +param confTestTags object = {} +param postgresqlVersion string = '14' +param tenantId string = subscription().tenantId + +resource postgresServer 'Microsoft.DBforPostgreSQL/flexibleServers@2023-03-01-preview' = { + name: postgresServerName + location: rgLocation + tags: confTestTags + sku: { + name: 'Standard_B1ms' + tier: 'Burstable' + } + properties: { + storage: { + storageSizeGB: 32 + autoGrow: 'Disabled' + } + authConfig: { + activeDirectoryAuth: 'Enabled' + passwordAuth: 'Disabled' + tenantId: tenantId + } + network: {} + version: postgresqlVersion + } + + resource daprTestDB 'databases@2023-03-01-preview' = { + name: 'dapr_test' + properties: { + charset: 'UTF8' + collation: 'en_US.utf8' + } + } + + resource fwRules 'firewallRules@2023-03-01-preview' = { + name: 'allowall' + properties: { + startIpAddress: '0.0.0.0' + endIpAddress: '255.255.255.255' + } + } + + resource azureAdAdmin 'administrators@2023-03-01-preview' = { + name: sdkAuthSpId + properties: { + principalType: 'ServicePrincipal' + principalName: sdkAuthSpName + tenantId: tenantId + } + } +} diff --git a/.github/infrastructure/conformance/azure/conf-test-azure-sqlserver.bicep b/.github/infrastructure/conformance/azure/conf-test-azure-sqlserver.bicep index 1433bba525..f00798aac6 100644 --- a/.github/infrastructure/conformance/azure/conf-test-azure-sqlserver.bicep +++ b/.github/infrastructure/conformance/azure/conf-test-azure-sqlserver.bicep @@ -14,6 +14,7 @@ param sqlServerName string param rgLocation string = resourceGroup().location param confTestTags object = {} +@secure() param sqlServerAdminPassword string var sqlServerAdminName = '${sqlServerName}-admin' diff --git a/.github/infrastructure/conformance/azure/conf-test-azure-storage.bicep b/.github/infrastructure/conformance/azure/conf-test-azure-storage.bicep index 92fd9d10b9..762c71cee3 100644 --- a/.github/infrastructure/conformance/azure/conf-test-azure-storage.bicep +++ b/.github/infrastructure/conformance/azure/conf-test-azure-storage.bicep @@ -15,7 +15,7 @@ param storageName string param rgLocation string = resourceGroup().location param confTestTags object = {} -resource storageAccount 'Microsoft.Storage/storageAccounts@2021-02-01' = { +resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { name: storageName sku: { name: 'Standard_RAGRS' @@ -23,27 +23,23 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2021-02-01' = { kind: 'StorageV2' location: rgLocation tags: confTestTags -} -resource blobServices 'Microsoft.Storage/storageAccounts/blobServices@2021-02-01' = { - parent: storageAccount - name: 'default' - properties: { - deleteRetentionPolicy: { - enabled: true - days: 1 + resource blobServices 'blobServices@2022-09-01' = { + name: 'default' + properties: { + deleteRetentionPolicy: { + enabled: true + days: 1 + } } } -} -resource tableServices 'Microsoft.Storage/storageAccounts/tableServices@2021-09-01' = { - parent: storageAccount - name: 'default' - properties: {} -} + resource tableServices 'tableServices@2022-09-01' = { + name: 'default' + properties: {} -resource certificationTable 'Microsoft.Storage/storageAccounts/tableServices/tables@2021-09-01' = { - name: 'certificationTable' - parent: tableServices - properties: {} + resource certificationTable 'tables@2022-09-01' = { + name: 'certificationTable' + } + } } diff --git a/.github/infrastructure/conformance/azure/conf-test-azure.bicep b/.github/infrastructure/conformance/azure/conf-test-azure.bicep index 7c2e9a16cd..5ffe55a280 100644 --- a/.github/infrastructure/conformance/azure/conf-test-azure.bicep +++ b/.github/infrastructure/conformance/azure/conf-test-azure.bicep @@ -33,9 +33,12 @@ param adminId string @minLength(36) @maxLength(36) -@description('Provide the objectId of the Service Principal using secret auth with get access to secrets in Azure Key Vault.') +@description('Provide the objectId of the Service Principal using secret auth with get access to secrets in Azure Key Vault and access Azure PostgreSQL') param sdkAuthSpId string +@description('Provide the name of the Service Principal using secret auth with get access to secrets in Azure Key Vault and access Azure PostgreSQL') +param sdkAuthSpName string + @minLength(36) @maxLength(36) @description('Provide the objectId of the Service Principal using cert auth with get and list access to all assets in Azure Key Vault.') @@ -43,6 +46,7 @@ param certAuthSpId string @minLength(16) @description('Provide the SQL server admin password of at least 16 characters.') +@secure() param sqlServerAdminPassword string var confTestRgName = '${toLower(namePrefix)}-conf-test-rg' @@ -54,6 +58,7 @@ var iotHubName = '${toLower(namePrefix)}-conf-test-iothub' var keyVaultName = '${toLower(namePrefix)}-conf-test-kv' var serviceBusName = '${toLower(namePrefix)}-conf-test-servicebus' var sqlServerName = '${toLower(namePrefix)}-conf-test-sql' +var postgresServerName = '${toLower(namePrefix)}-conf-test-pg' var storageName = '${toLower(namePrefix)}ctstorage' resource confTestRg 'Microsoft.Resources/resourceGroups@2021-04-01' = { @@ -72,6 +77,7 @@ module cosmosDb 'conf-test-azure-cosmosdb.bicep' = { params: { confTestTags: confTestTags cosmosDbName: cosmosDbName + rgLocation: rgLocation } } @@ -81,6 +87,7 @@ module cosmosDbTable 'conf-test-azure-cosmosdb-table.bicep' = { params: { confTestTags: confTestTags cosmosDbTableAPIName: cosmosDbTableAPIName + rgLocation: rgLocation } } @@ -90,6 +97,7 @@ module eventGridTopic 'conf-test-azure-eventgrid.bicep' = { params: { confTestTags: confTestTags eventGridTopicName: eventGridTopicName + rgLocation: rgLocation } } @@ -99,6 +107,7 @@ module eventHubsNamespace 'conf-test-azure-eventhubs.bicep' = { params: { confTestTags: confTestTags eventHubsNamespaceName: eventHubsNamespaceName + rgLocation: rgLocation } } @@ -108,6 +117,7 @@ module iotHub 'conf-test-azure-iothub.bicep' = { params: { confTestTags: confTestTags iotHubName: iotHubName + rgLocation: rgLocation } } @@ -120,6 +130,7 @@ module keyVault 'conf-test-azure-keyvault.bicep' = { certAuthSpId: certAuthSpId keyVaultName: keyVaultName sdkAuthSpId: sdkAuthSpId + rgLocation: rgLocation } } @@ -129,6 +140,7 @@ module serviceBus 'conf-test-azure-servicebus.bicep' = { params: { confTestTags: confTestTags serviceBusName: serviceBusName + rgLocation: rgLocation } } @@ -139,6 +151,7 @@ module sqlServer 'conf-test-azure-sqlserver.bicep' = { confTestTags: confTestTags sqlServerName: sqlServerName sqlServerAdminPassword: sqlServerAdminPassword + rgLocation: rgLocation } } @@ -148,6 +161,19 @@ module storage 'conf-test-azure-storage.bicep' = { params: { confTestTags: confTestTags storageName: storageName + rgLocation: rgLocation + } +} + +module postgres 'conf-test-azure-postgres.bicep' = { + name: postgresServerName + scope: resourceGroup(confTestRg.name) + params: { + confTestTags: confTestTags + postgresServerName: postgresServerName + sdkAuthSpId: sdkAuthSpId + sdkAuthSpName: sdkAuthSpName + rgLocation: rgLocation } } @@ -176,4 +202,5 @@ output keyVaultName string = keyVault.name output serviceBusName string = serviceBus.name output sqlServerName string = sqlServer.name output sqlServerAdminName string = sqlServer.outputs.sqlServerAdminName +output postgresServerName string = postgres.name output storageName string = storage.name diff --git a/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh b/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh index 88b4285832..8220f57fcd 100755 --- a/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh +++ b/.github/infrastructure/conformance/azure/setup-azure-conf-test.sh @@ -229,6 +229,8 @@ SQL_SERVER_NAME_VAR_NAME="AzureSqlServerName" SQL_SERVER_DB_NAME_VAR_NAME="AzureSqlServerDbName" SQL_SERVER_CONNECTION_STRING_VAR_NAME="AzureSqlServerConnectionString" +AZURE_DB_POSTGRES_CONNSTRING_VAR_NAME="AzureDBPostgresConnectionString" + STORAGE_ACCESS_KEY_VAR_NAME="AzureBlobStorageAccessKey" STORAGE_ACCOUNT_VAR_NAME="AzureBlobStorageAccount" STORAGE_CONTAINER_VAR_NAME="AzureBlobStorageContainer" @@ -269,7 +271,7 @@ if [[ -n ${CREDENTIALS_PATH} ]]; then fi SDK_AUTH_SP_NAME="$(az ad sp show --id "${SDK_AUTH_SP_APPID}" --query "appDisplayName" --output tsv)" SDK_AUTH_SP_ID="$(az ad sp show --id "${SDK_AUTH_SP_APPID}" --query "id" --output tsv)" - echo "Using Service Principal from ${CREDENTIALS_PATH} for SDK Auth: ${SDK_AUTH_SP_NAME}" + echo "Using Service Principal from ${CREDENTIALS_PATH} for SDK Auth: ${SDK_AUTH_SP_NAME} (ID: ${SDK_AUTH_SP_ID})" else SDK_AUTH_SP_NAME="${PREFIX}-conf-test-runner-sp" SDK_AUTH_SP_INFO="$(az ad sp create-for-rbac --name "${SDK_AUTH_SP_NAME}" --sdk-auth --years 1)" @@ -277,7 +279,7 @@ else SDK_AUTH_SP_CLIENT_SECRET="$(echo "${SDK_AUTH_SP_INFO}" | jq -r '.clientSecret')" SDK_AUTH_SP_ID="$(az ad sp list --display-name "${SDK_AUTH_SP_NAME}" --query "[].id" --output tsv)" echo "${SDK_AUTH_SP_INFO}" - echo "Created Service Principal for SDK Auth: ${SDK_AUTH_SP_NAME}" + echo "Created Service Principal for SDK Auth: ${SDK_AUTH_SP_NAME} (ID: ${SDK_AUTH_SP_ID})" AZURE_CREDENTIALS_FILENAME="${OUTPUT_PATH}/AZURE_CREDENTIALS" echo "${SDK_AUTH_SP_INFO}" > "${AZURE_CREDENTIALS_FILENAME}" fi @@ -292,7 +294,17 @@ echo "Building conf-test-azure.bicep to ${ARM_TEMPLATE_FILE} ..." az bicep build --file conf-test-azure.bicep --outfile "${ARM_TEMPLATE_FILE}" echo "Creating azure deployment ${DEPLOY_NAME} in ${DEPLOY_LOCATION} and resource prefix ${PREFIX}-* ..." -az deployment sub create --name "${DEPLOY_NAME}" --location "${DEPLOY_LOCATION}" --template-file "${ARM_TEMPLATE_FILE}" -p namePrefix="${PREFIX}" -p adminId="${ADMIN_ID}" -p certAuthSpId="${CERT_AUTH_SP_ID}" -p sdkAuthSpId="${SDK_AUTH_SP_ID}" -p rgLocation="${DEPLOY_LOCATION}" -p sqlServerAdminPassword="${SQL_SERVER_ADMIN_PASSWORD}" +az deployment sub create \ + --name "${DEPLOY_NAME}" \ + --location "${DEPLOY_LOCATION}" \ + --template-file "${ARM_TEMPLATE_FILE}" \ + -p namePrefix="${PREFIX}" \ + -p adminId="${ADMIN_ID}" \ + -p certAuthSpId="${CERT_AUTH_SP_ID}" \ + -p sdkAuthSpId="${SDK_AUTH_SP_ID}" \ + -p sdkAuthSpName="${SDK_AUTH_SP_NAME}" \ + -p rgLocation="${DEPLOY_LOCATION}" \ + -p sqlServerAdminPassword="${SQL_SERVER_ADMIN_PASSWORD}" echo "Sleeping for 5s to allow created ARM deployment info to propagate to query endpoints ..." sleep 5 @@ -546,6 +558,7 @@ az keyvault secret set --name "${KEYVAULT_SERVICE_PRINCIPAL_CLIENT_ID_VAR_NAME}" KEYVAULT_SERVICE_PRINCIPAL_CLIENT_SECRET=${AKV_SPAUTH_SP_CLIENT_SECRET} echo export ${KEYVAULT_SERVICE_PRINCIPAL_CLIENT_SECRET_VAR_NAME}=\"${KEYVAULT_SERVICE_PRINCIPAL_CLIENT_SECRET}\" >> "${ENV_CONFIG_FILENAME}" az keyvault secret set --name "${KEYVAULT_SERVICE_PRINCIPAL_CLIENT_SECRET_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${KEYVAULT_SERVICE_PRINCIPAL_CLIENT_SECRET}" + # ------------------------------------ # Populate Blob Storage test settings # ------------------------------------ @@ -671,6 +684,15 @@ SQL_SERVER_CONNECTION_STRING="Server=${SQL_SERVER_NAME}.database.windows.net;por echo export ${SQL_SERVER_CONNECTION_STRING_VAR_NAME}=\"${SQL_SERVER_CONNECTION_STRING}\" >> "${ENV_CONFIG_FILENAME}" az keyvault secret set --name "${SQL_SERVER_CONNECTION_STRING_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${SQL_SERVER_CONNECTION_STRING}" +# ---------------------------------- +# Populate Azure Database for PostgreSQL test settings +# ---------------------------------- +echo "Configuring Azure Database for PostgreSQL test settings ..." + +AZURE_DB_POSTGRES_CONNSTRING="host=${PREFIX}-conf-test-pg.postgres.database.azure.com user=${SDK_AUTH_SP_NAME} port=5432 connect_timeout=30 database=dapr_test" +echo export ${AZURE_DB_POSTGRES_CONNSTRING_VAR_NAME}=\"${AZURE_DB_POSTGRES_CONNSTRING}\" >> "${ENV_CONFIG_FILENAME}" +az keyvault secret set --name "${AZURE_DB_POSTGRES_CONNSTRING_VAR_NAME}" --vault-name "${KEYVAULT_NAME}" --value "${AZURE_DB_POSTGRES_CONNSTRING}" + # ---------------------------------- # Populate Event Hubs test settings # ---------------------------------- diff --git a/.github/scripts/test-info.mjs b/.github/scripts/test-info.mjs index d0d315f316..8bfeb0187f 100644 --- a/.github/scripts/test-info.mjs +++ b/.github/scripts/test-info.mjs @@ -542,7 +542,11 @@ const components = { 'internal/component/sql', ], }, - 'state.etcd': { + 'state.etcd.v1': { + conformance: true, + conformanceSetup: 'docker-compose.sh etcd', + }, + 'state.etcd.v2': { conformance: true, conformanceSetup: 'docker-compose.sh etcd', }, @@ -578,8 +582,15 @@ const components = { conformanceSetup: 'docker-compose.sh oracledatabase', }, 'state.postgresql': { - conformance: true, certification: true, + sourcePkg: [ + 'state/postgresql', + 'internal/component/postgresql', + 'internal/component/sql', + ], + }, + 'state.postgresql.docker': { + conformance: true, conformanceSetup: 'docker-compose.sh postgresql', sourcePkg: [ 'state/postgresql', @@ -587,6 +598,15 @@ const components = { 'internal/component/sql', ], }, + 'state.postgresql.azure': { + conformance: true, + requiredSecrets: ['AzureDBPostgresConnectionString'], + sourcePkg: [ + 'state/postgresql', + 'internal/component/postgresql', + 'internal/component/sql', + ], + }, 'state.redis': { certification: true, sourcePkg: ['state/redis', 'internal/component/redis'], diff --git a/bindings/azure/blobstorage/metadata.yaml b/bindings/azure/blobstorage/metadata.yaml index 500c273b7b..047d4fa0bb 100644 --- a/bindings/azure/blobstorage/metadata.yaml +++ b/bindings/azure/blobstorage/metadata.yaml @@ -12,13 +12,13 @@ binding: output: true operations: - name: create - description: "Create blob." + description: "Create blob" - name: get - description: "Get blob." + description: "Get blob" - name: delete - description: "Delete blob." + description: "Delete blob" - name: list - description: "List blob." + description: "List blob" capabilities: [] builtinAuthenticationProfiles: - name: "azuread" diff --git a/bindings/azure/cosmosdb/cosmosdb.go b/bindings/azure/cosmosdb/cosmosdb.go index c0c39c2857..785514f74f 100644 --- a/bindings/azure/cosmosdb/cosmosdb.go +++ b/bindings/azure/cosmosdb/cosmosdb.go @@ -174,8 +174,6 @@ func (c *CosmosDB) lookup(m map[string]interface{}, ks []string) (val interface{ return nil, fmt.Errorf("needs at least one key") } - c.logger.Infof("%s, %s", ks[0], m[ks[0]]) - if val, ok = m[ks[0]]; !ok { return nil, fmt.Errorf("key not found %v", ks[0]) } diff --git a/bindings/azure/cosmosdb/metadata.yaml b/bindings/azure/cosmosdb/metadata.yaml index 4ec191dc3f..ce33d40b60 100644 --- a/bindings/azure/cosmosdb/metadata.yaml +++ b/bindings/azure/cosmosdb/metadata.yaml @@ -12,7 +12,7 @@ binding: output: true operations: - name: create - description: "Create an item." + description: "Create an item" capabilities: [] builtinAuthenticationProfiles: - name: "azuread" diff --git a/bindings/azure/cosmosdbgremlinapi/metadata.yaml b/bindings/azure/cosmosdbgremlinapi/metadata.yaml index 3c2988f05f..609bd1aea9 100644 --- a/bindings/azure/cosmosdbgremlinapi/metadata.yaml +++ b/bindings/azure/cosmosdbgremlinapi/metadata.yaml @@ -12,7 +12,7 @@ binding: output: true operations: - name: query - description: "Perform a query." + description: "Perform a query" capabilities: [] authenticationProfiles: - title: "Master key" diff --git a/bindings/azure/eventgrid/metadata.yaml b/bindings/azure/eventgrid/metadata.yaml index bd2ee8b6dc..7c99fc44fa 100644 --- a/bindings/azure/eventgrid/metadata.yaml +++ b/bindings/azure/eventgrid/metadata.yaml @@ -13,7 +13,7 @@ binding: output: true operations: - name: create - description: "Create an event subscription." + description: "Create an event subscription" capabilities: [] builtinAuthenticationProfiles: - name: "azuread" diff --git a/bindings/azure/eventhubs/metadata.yaml b/bindings/azure/eventhubs/metadata.yaml index 40a7b83c48..77c221cfa2 100644 --- a/bindings/azure/eventhubs/metadata.yaml +++ b/bindings/azure/eventhubs/metadata.yaml @@ -13,7 +13,7 @@ binding: output: true operations: - name: create - description: "Create an event subscription." + description: "Create an event subscription" capabilities: [] authenticationProfiles: - title: "Connection string" @@ -24,12 +24,15 @@ authenticationProfiles: sensitive: true description: | Connection string for the Event Hub or the Event Hub namespace. - example: '"Endpoint=sb://{EventHubNamespace}.servicebus.windows.net/;SharedAccessKeyName={PolicyName};SharedAccessKey={Key};EntityPath={EventHub}"' + example: | + "Endpoint=sb://{EventHubNamespace}.servicebus.windows.net/;SharedAccessKeyName={PolicyName};SharedAccessKey={Key};EntityPath={EventHub}" - name: eventHub type: string description: | - The name of the Event Hubs hub (“topic”). Required if the connection string doesn’t contain an EntityPath value. + The name of the Event Hubs hub ("topic"). Required if the connection string doesn't contain an EntityPath value. required: false # Optional when a connectionString is provided + example: | + mytopic builtinAuthenticationProfiles: - name: "azuread" metadata: @@ -50,6 +53,7 @@ builtinAuthenticationProfiles: type: bool required: false default: "false" + example: "false" description: | Allow management of the Event Hub namespace and storage account. - name: resourceGroupName @@ -62,7 +66,7 @@ builtinAuthenticationProfiles: - name: subscriptionId type: string required: false - bindings: + binding: input: true output: false description: | @@ -92,12 +96,13 @@ metadata: description: | DEPRECATED. deprecated: true + example: "" # Input-only metadata # consumerGroup is an alias for consumerId, if both are defined consumerId takes precedence. - name: consumerId type: string required: true # consumerGroup is an alias for this field, let's promote this to default - bindings: + binding: input: true output: false description: | @@ -106,7 +111,7 @@ metadata: - name: consumerGroup type: string required: false - bindings: + binding: input: true output: false description: | @@ -117,7 +122,7 @@ metadata: - name: storageAccountKey type: string required: false - bindings: + binding: input: true output: false description: | @@ -129,17 +134,19 @@ metadata: - name: storageConnectionString type: string required: false - bindings: + binding: input: true output: false description: | Connection string for the checkpoint store, alternative to specifying storageAccountKey. Property "storageAccountKey" is ignored when "storageConnectionString" is present + example: | + "BlobEndpoint=https://storagesample.blob.core.windows.net;..." - name: storageAccountName type: string required: true - bindings: + binding: input: true output: false description: | @@ -148,7 +155,7 @@ metadata: - name: storageContainerName type: string required: true - bindings: + binding: input: true output: false description: | diff --git a/bindings/cloudflare/queues/cfqueues.go b/bindings/cloudflare/queues/cfqueues.go index ad32503bf9..1b3d04a106 100644 --- a/bindings/cloudflare/queues/cfqueues.go +++ b/bindings/cloudflare/queues/cfqueues.go @@ -31,7 +31,7 @@ import ( ) // Link to the documentation for the component -const componentDocsURL = "https://docs.dapr.io/reference/components-reference/supported-bindings/cfqueues/" +const componentDocsURL = "https://docs.dapr.io/reference/components-reference/supported-bindings/cloudflare-queues/" // CFQueues is a binding for publishing messages on Cloudflare Queues type CFQueues struct { diff --git a/bindings/http/metadata.yaml b/bindings/http/metadata.yaml index de0ac80f4b..23309ea0a7 100644 --- a/bindings/http/metadata.yaml +++ b/bindings/http/metadata.yaml @@ -13,23 +13,23 @@ binding: input: false operations: - name: create - description: "Alias for \"post\", for backwards-compatibility." + description: "Alias for \"post\", for backwards-compatibility" - name: get - description: "Read data/records." + description: "Read data/records" - name: head - description: "Identical to get except that the server does not return a response body." + description: "Identical to get except that the server does not return a response body" - name: post - description: "Typically used to create records or send commands." + description: "Typically used to create records or send commands" - name: put - description: "Update data/records." + description: "Update data/records" - name: patch - description: "Sometimes used to update a subset of fields of a record." + description: "Sometimes used to update a subset of fields of a record" - name: delete - description: "Delete a data/record." + description: "Delete a data/record" - name: options - description: "Requests for information about the communication options available (not commonly used)." + description: "Requests for information about the communication options available (not commonly used)" - name: trace - description: "Used to invoke a remote, application-layer loop-back of the request message (not commonly used)." + description: "Used to invoke a remote, application-layer loop-back of the request message (not commonly used)" capabilities: [] metadata: - name: url diff --git a/bindings/postgres/metadata.yaml b/bindings/postgres/metadata.yaml index 321744d039..1d771523a3 100644 --- a/bindings/postgres/metadata.yaml +++ b/bindings/postgres/metadata.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=../../../component-metadata-schema.json +# yaml-language-server: $schema=../../component-metadata-schema.json schemaVersion: v1 type: bindings name: postgres @@ -26,7 +26,7 @@ authenticationProfiles: - name: url required: true sensitive: true - bindings: + binding: input: false output: true description: "Connection string for PostgreSQL." diff --git a/bindings/redis/metadata.yaml b/bindings/redis/metadata.yaml index fd20b1191c..d26b0e695a 100644 --- a/bindings/redis/metadata.yaml +++ b/bindings/redis/metadata.yaml @@ -14,16 +14,16 @@ binding: input: false operations: - name: create - description: "Create item." + description: "Create item" - name: get - description: "Get item." + description: "Get item" - name: delete - description: "Delete item." + description: "Delete item" - name: increment description: "Increment a key" authenticationProfiles: - title: "Username and password" - description: "Authenticate using username and password." + description: "Authenticate using username and password" metadata: - name: redisUsername type: string @@ -40,7 +40,7 @@ authenticationProfiles: description: | Password for Redis host. Use secretKeyRef for secret reference - example: "KeFg23!" + example: "KeFg23!" default: "" metadata: - name: redisHost diff --git a/bindings/wasm/Makefile b/bindings/wasm/Makefile index 671c185e59..6bb0e7134c 100644 --- a/bindings/wasm/Makefile +++ b/bindings/wasm/Makefile @@ -1,3 +1,3 @@ .PHONY: build build: - @$(MAKE) -C example + @$(MAKE) -C testdata diff --git a/bindings/wasm/output.go b/bindings/wasm/output.go index f29e7a591f..9d80d5511e 100644 --- a/bindings/wasm/output.go +++ b/bindings/wasm/output.go @@ -16,7 +16,6 @@ package wasm import ( "bytes" "context" - "crypto/rand" "fmt" "io" "reflect" @@ -40,11 +39,10 @@ const ExecuteOperation bindings.OperationKind = "execute" type outputBinding struct { logger logger.Logger runtimeConfig wazero.RuntimeConfig - moduleConfig wazero.ModuleConfig - guestName string - runtime wazero.Runtime - module wazero.CompiledModule + meta *wasm.InitMetadata + runtime wazero.Runtime + module wazero.CompiledModule instanceCounter atomic.Uint64 } @@ -61,28 +59,19 @@ func NewWasmOutput(logger logger.Logger) bindings.OutputBinding { // The below ensures context cancels in-flight wasm functions. runtimeConfig: wazero.NewRuntimeConfig(). WithCloseOnContextDone(true), - - // The below violate sand-boxing, but allow code to behave as expected. - moduleConfig: wazero.NewModuleConfig(). - WithRandSource(rand.Reader). - WithSysWalltime(). - WithSysNanosleep(), } } func (out *outputBinding) Init(ctx context.Context, metadata bindings.Metadata) (err error) { - meta, err := wasm.GetInitMetadata(ctx, metadata.Base) - if err != nil { + if out.meta, err = wasm.GetInitMetadata(metadata.Base); err != nil { return fmt.Errorf("wasm: failed to parse metadata: %w", err) } - out.guestName = meta.GuestName - // Create the runtime, which when closed releases any resources associated with it. out.runtime = wazero.NewRuntimeWithConfig(ctx, out.runtimeConfig) // Compile the module, which reduces execution time of Invoke - out.module, err = out.runtime.CompileModule(ctx, meta.Guest) + out.module, err = out.runtime.CompileModule(ctx, out.meta.Guest) if err != nil { _ = out.runtime.Close(context.Background()) return fmt.Errorf("wasm: error compiling binary: %w", err) @@ -101,11 +90,16 @@ func (out *outputBinding) Init(ctx context.Context, metadata bindings.Metadata) } func (out *outputBinding) Invoke(ctx context.Context, req *bindings.InvokeRequest) (*bindings.InvokeResponse, error) { + guestName := out.meta.GuestName + if guestName == "" { + guestName = out.module.Name() + } + // Currently, concurrent modules can conflict on name. Make sure we have // a unique one. instanceNum := out.instanceCounter.Add(1) - instanceName := out.guestName + "-" + strconv.FormatUint(instanceNum, 10) - moduleConfig := out.moduleConfig.WithName(instanceName) + instanceName := guestName + "-" + strconv.FormatUint(instanceNum, 10) + moduleConfig := wasm.NewModuleConfig(out.meta).WithName(instanceName) // Only assign STDIN if it is present in the request. if len(req.Data) > 0 { @@ -117,7 +111,7 @@ func (out *outputBinding) Invoke(ctx context.Context, req *bindings.InvokeReques moduleConfig = moduleConfig.WithStdout(&stdout) // Set the program name to the binary name - argsSlice := []string{out.guestName} + argsSlice := []string{guestName} // Get any remaining args from configuration if args := req.Metadata["args"]; args != "" { diff --git a/bindings/wasm/testdata/args/main.wasm b/bindings/wasm/testdata/args/main.wasm index 1b0ad29652..4a164b6f0a 100755 Binary files a/bindings/wasm/testdata/args/main.wasm and b/bindings/wasm/testdata/args/main.wasm differ diff --git a/bindings/wasm/testdata/example/main.wasm b/bindings/wasm/testdata/example/main.wasm index e9ca99eb68..119a962f3d 100755 Binary files a/bindings/wasm/testdata/example/main.wasm and b/bindings/wasm/testdata/example/main.wasm differ diff --git a/bindings/wasm/testdata/loop/main.wasm b/bindings/wasm/testdata/loop/main.wasm index 58e8e550e6..e57b3f255b 100755 Binary files a/bindings/wasm/testdata/loop/main.wasm and b/bindings/wasm/testdata/loop/main.wasm differ diff --git a/configuration/azure/appconfig/metadata.yaml b/configuration/azure/appconfig/metadata.yaml index dc2e619e70..040b72fb5d 100644 --- a/configuration/azure/appconfig/metadata.yaml +++ b/configuration/azure/appconfig/metadata.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=../../component-metadata-schema.json +# yaml-language-server: $schema=../../../component-metadata-schema.json schemaVersion: v1 type: configuration name: azure.appconfig @@ -17,7 +17,7 @@ authenticationProfiles: required: true sensitive: true description: "The Azure App Configuration connection string." - example: 'Endpoint=https://foo.azconfig.io;Id=osOX-l9-s0:sig;Secret=00000000000000000000000000000000000000000000"' + example: 'Endpoint=https://foo.azconfig.io;Id=osOX-l9-s0:sig;Secret=xxx"' # If omitted, uses the same values as ".binding" binding: output: true @@ -35,22 +35,22 @@ metadata: default: '3' example: '10' - name: retryDelay - description: "Specifies the initial amount of delay to use before retrying an operation. The delay increases exponentially with each retry up to the maximum specified by MaxRetryDelay. Defaults to 4 seconds. -1 disables delay between retries." + description: "Specifies the initial amount of delay to use before retrying an operation, in nanoseconds. The delay increases exponentially with each retry up to the maximum specified by MaxRetryDelay. Defaults to 4 seconds. -1 disables delay between retries." type: number default: '4000000000' example: '5000000000' - name: maxRetryDelay - description: "Specifies the maximum delay allowed before retrying an operation. Typically the value is greater than or equal to the value specified in RetryDelay. Defaults to 120 seconds. -1 disables the limit." + description: "Specifies the maximum delay allowed before retrying an operation, in nanoseconds. Typically the value is greater than or equal to the value specified in RetryDelay. Defaults to 120 seconds. -1 disables the limit." type: number default: '120000000000' example: '180000000000' - name: subscribePollInterval - description: "Specifies the poll interval for polling the subscribed keys for any changes. Default polling interval is set to 24 hours." + description: "Specifies the poll interval for polling the subscribed keys for any changes, in nanoseconds. Default polling interval is set to 24 hours." type: number default: '86400000000000' example: '240000000000' - name: requesttimeout - description: "Specifies the time allowed to pass until a request is failed. Default timeout is set to 15 seconds." + description: "Specifies the time allowed to pass until a request is failed, in nanoseconds. Default timeout is set to 15 seconds." type: number default: '15000000000' example: '30000000000' \ No newline at end of file diff --git a/go.mod b/go.mod index c6cedb8d5a..937f84ebff 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/hashicorp/consul/api v1.13.0 github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/hazelcast/hazelcast-go-client v0.0.0-20190530123621-6cf767c2f31a - github.com/http-wasm/http-wasm-host-go v0.5.0 + github.com/http-wasm/http-wasm-host-go v0.5.1 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.22.11+incompatible github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.28 github.com/influxdata/influxdb-client-go v1.4.0 @@ -102,7 +102,7 @@ require ( github.com/supplyon/gremcos v0.1.40 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.608 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/ssm v1.0.608 - github.com/tetratelabs/wazero v1.1.0 + github.com/tetratelabs/wazero v1.3.0 github.com/valyala/fasthttp v1.47.0 github.com/vmware/vmware-go-kcl v1.5.0 github.com/xdg-go/scram v1.1.2 @@ -119,6 +119,7 @@ require ( golang.org/x/oauth2 v0.8.0 google.golang.org/api v0.115.0 google.golang.org/grpc v1.54.0 + google.golang.org/protobuf v1.30.0 gopkg.in/couchbase/gocb.v1 v1.6.7 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/yaml.v3 v3.0.1 @@ -371,7 +372,6 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect - google.golang.org/protobuf v1.30.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/couchbase/gocbcore.v7 v7.1.18 // indirect gopkg.in/couchbaselabs/gocbconnstr.v1 v1.0.4 // indirect diff --git a/go.sum b/go.sum index 404e04880a..33ff6e88ca 100644 --- a/go.sum +++ b/go.sum @@ -1292,8 +1292,8 @@ github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKe github.com/hazelcast/hazelcast-go-client v0.0.0-20190530123621-6cf767c2f31a h1:j6SSiw7fWemWfrJL801xiQ6xRT7ZImika50xvmPN+tg= github.com/hazelcast/hazelcast-go-client v0.0.0-20190530123621-6cf767c2f31a/go.mod h1:VhwtcZ7sg3xq7REqGzEy7ylSWGKz4jZd05eCJropNzI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/http-wasm/http-wasm-host-go v0.5.0 h1:gEivjxSoBFeTWerGk9amBL21c54zhr3iSQYhu0OmAE0= -github.com/http-wasm/http-wasm-host-go v0.5.0/go.mod h1:Z83VkMiDJRBoqTLWCiQoaRK8thbRBDaIVxMT9KSNaP8= +github.com/http-wasm/http-wasm-host-go v0.5.1 h1:pdr46nnh/ya5Nj0rmPKrxI/zx5781yG/tix8P17tcFI= +github.com/http-wasm/http-wasm-host-go v0.5.1/go.mod h1:GslHNHfjfM15UDrxEh/jp9JTreW6xt/zbxQzLkk9YMM= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.22.11+incompatible h1:bSww59mgbqFRGCRvlvfQutsptE3lRjNiU5C0YNT/bWw= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.22.11+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.28 h1:2w2khA5uopaLcZactRuQFGtUgMbN92aWZdOuT4b9HxU= @@ -1890,8 +1890,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.608 h1:yLiH github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.608/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/ssm v1.0.608 h1:Wi11Yw2E6W7ZE/B2whvLq5ILeZbrRukVCc1s6ptbOU8= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/ssm v1.0.608/go.mod h1:oOEV4UP04zUSkZCjjfZ32DUpbWSdWbOyaO5gDfC1IH0= -github.com/tetratelabs/wazero v1.1.0 h1:EByoAhC+QcYpwSZJSs/aV0uokxPwBgKxfiokSUwAknQ= -github.com/tetratelabs/wazero v1.1.0/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ= +github.com/tetratelabs/wazero v1.3.0 h1:nqw7zCldxE06B8zSZAY0ACrR9OH5QCcPwYmYlwtcwtE= +github.com/tetratelabs/wazero v1.3.0/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ= github.com/tevid/gohamcrest v1.1.1/go.mod h1:3UvtWlqm8j5JbwYZh80D/PVBt0mJ1eJiYgZMibh0H/k= github.com/tidwall/gjson v1.2.1/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= github.com/tidwall/gjson v1.9.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= diff --git a/internal/authentication/azure/services.go b/internal/authentication/azure/services.go index 1b19bf6ad4..62966f04bc 100644 --- a/internal/authentication/azure/services.go +++ b/internal/authentication/azure/services.go @@ -20,6 +20,8 @@ import ( const ( // Service configuration for Azure SQL. Namespaced with dapr.io ServiceAzureSQL cloud.ServiceName = "dapr.io/azuresql" + // Service configuration for OSS RDBMS (Azure Database for PostgreSQL and MySQL). Namespaced with dapr.io + ServiceOSSRDBMS cloud.ServiceName = "dapr.io/oss-rdbms" ) func init() { @@ -33,4 +35,13 @@ func init() { cloud.AzurePublic.Services[ServiceAzureSQL] = cloud.ServiceConfiguration{ Audience: "https://database.windows.net", } + cloud.AzureChina.Services[ServiceOSSRDBMS] = cloud.ServiceConfiguration{ + Audience: "https://ossrdbms-aad.database.chinacloudapi.cn", + } + cloud.AzureGovernment.Services[ServiceOSSRDBMS] = cloud.ServiceConfiguration{ + Audience: "https://ossrdbms-aad.database.usgovcloudapi.net", + } + cloud.AzurePublic.Services[ServiceOSSRDBMS] = cloud.ServiceConfiguration{ + Audience: "https://ossrdbms-aad.database.windows.net", + } } diff --git a/internal/component/azure/blobstorage/client.go b/internal/component/azure/blobstorage/client.go index 154be61d26..858cd2b7ff 100644 --- a/internal/component/azure/blobstorage/client.go +++ b/internal/component/azure/blobstorage/client.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "net/url" + "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -49,8 +50,10 @@ func CreateContainerStorageClient(parentCtx context.Context, log logger.Logger, return nil, nil, err } - if val, _ := mdutils.GetMetadataProperty(meta, azauth.MetadataKeys["StorageEndpoint"]...); val != "" { - m.customEndpoint = val + // Check if using a custom endpoint + err = m.setCustomEndpoint(log, meta, azEnvSettings) + if err != nil { + return nil, nil, err } // Get the container client @@ -74,21 +77,50 @@ func CreateContainerStorageClient(parentCtx context.Context, log logger.Logger, return client, m, nil } +// Sets the customEndpoint property if needed +func (opts *ContainerClientOpts) setCustomEndpoint(log logger.Logger, meta map[string]string, azEnvSettings azauth.EnvironmentSettings) error { + val, _ := mdutils.GetMetadataProperty(meta, azauth.MetadataKeys["StorageEndpoint"]...) + if val == "" { + return nil + } + + endpointURL, err := url.Parse(val) + if err != nil { + return fmt.Errorf("failed to parse custom endpoint %q: %w", val, err) + } + + // Check if the custom endpoint is set to an Azure Blob Storage public endpoint + azbURL := opts.getAzureBlobStorageContainerURL(azEnvSettings) + if endpointURL.Hostname() == azbURL.Hostname() && azbURL.Path == endpointURL.Path { + log.Warn("Metadata property endpoint is set to an Azure Blob Storage endpoint and will be ignored") + } else { + log.Info("Using custom endpoint for Azure Blob Storage") + opts.customEndpoint = strings.TrimSuffix(endpointURL.String(), "/") + } + + return nil +} + // GetContainerURL returns the URL of the container, needed by some auth methods. -func (opts ContainerClientOpts) GetContainerURL(azEnvSettings azauth.EnvironmentSettings) (u *url.URL, err error) { +func (opts *ContainerClientOpts) GetContainerURL(azEnvSettings azauth.EnvironmentSettings) (u *url.URL, err error) { if opts.customEndpoint != "" { u, err = url.Parse(fmt.Sprintf("%s/%s/%s", opts.customEndpoint, opts.AccountName, opts.ContainerName)) if err != nil { return nil, fmt.Errorf("failed to get container's URL with custom endpoint") } } else { - u, _ = url.Parse(fmt.Sprintf("https://%s.blob.%s/%s", opts.AccountName, azEnvSettings.EndpointSuffix(azauth.ServiceAzureStorage), opts.ContainerName)) + u = opts.getAzureBlobStorageContainerURL(azEnvSettings) } return u, nil } +func (opts *ContainerClientOpts) getAzureBlobStorageContainerURL(azEnvSettings azauth.EnvironmentSettings) *url.URL { + u, _ := url.Parse(fmt.Sprintf("https://%s.blob.%s/%s", opts.AccountName, azEnvSettings.EndpointSuffix(azauth.ServiceAzureStorage), opts.ContainerName)) + return u +} + // InitContainerClient returns a new container.Client object from the given options. -func (opts ContainerClientOpts) InitContainerClient(azEnvSettings azauth.EnvironmentSettings) (client *container.Client, err error) { +func (opts *ContainerClientOpts) InitContainerClient(azEnvSettings azauth.EnvironmentSettings) (client *container.Client, err error) { clientOpts := &container.ClientOptions{ ClientOptions: azcore.ClientOptions{ Retry: policy.RetryOptions{ @@ -149,7 +181,7 @@ func (opts ContainerClientOpts) InitContainerClient(azEnvSettings azauth.Environ // EnsureContainer creates the container if it doesn't already exist. // Property "accessLevel" indicates the public access level; nil-value means the container is private -func (opts ContainerClientOpts) EnsureContainer(ctx context.Context, client *container.Client, accessLevel *azblob.PublicAccessType) error { +func (opts *ContainerClientOpts) EnsureContainer(ctx context.Context, client *container.Client, accessLevel *azblob.PublicAccessType) error { // Create the container // This will return an error if it already exists _, err := client.Create(ctx, &container.CreateOptions{ diff --git a/internal/component/azure/blobstorage/client_test.go b/internal/component/azure/blobstorage/client_test.go index 85835fc4d5..0b6dc3bf0d 100644 --- a/internal/component/azure/blobstorage/client_test.go +++ b/internal/component/azure/blobstorage/client_test.go @@ -14,23 +14,25 @@ limitations under the License. package blobstorage import ( + "bytes" "context" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" azauth "github.com/dapr/components-contrib/internal/authentication/azure" "github.com/dapr/kit/logger" ) -type scenario struct { - metadata map[string]string - expectedFailureSubString string -} - func TestClientInitFailures(t *testing.T) { log := logger.NewLogger("test") + type scenario struct { + metadata map[string]string + expectedFailureSubString string + } + scenarios := map[string]scenario{ "missing accountName": { metadata: createTestMetadata(false, true, true), @@ -50,6 +52,94 @@ func TestClientInitFailures(t *testing.T) { } } +func TestSetCustomEndpoint(t *testing.T) { + logDest := &bytes.Buffer{} + log := logger.NewLogger("test") + log.SetOutput(logDest) + + t.Run("no custom endpoint", func(t *testing.T) { + meta := createTestMetadata(true, true, true) + m, err := parseMetadata(meta) + require.NoError(t, err) + + azEnvSettings, err := azauth.NewEnvironmentSettings(meta) + require.NoError(t, err) + + err = m.setCustomEndpoint(log, meta, azEnvSettings) + require.NoError(t, err) + + assert.Equal(t, "", m.customEndpoint) + + u, err := m.GetContainerURL(azEnvSettings) + require.NoError(t, err) + assert.Equal(t, "https://account.blob.core.windows.net/test", u.String()) + }) + + t.Run("custom endpoint set", func(t *testing.T) { + meta := createTestMetadata(true, true, true) + meta[azauth.MetadataKeys["StorageEndpoint"][0]] = "https://localhost:8080" + + m, err := parseMetadata(meta) + require.NoError(t, err) + + azEnvSettings, err := azauth.NewEnvironmentSettings(meta) + require.NoError(t, err) + + err = m.setCustomEndpoint(log, meta, azEnvSettings) + require.NoError(t, err) + + assert.Equal(t, "https://localhost:8080", m.customEndpoint) + + u, err := m.GetContainerURL(azEnvSettings) + require.NoError(t, err) + assert.Equal(t, "https://localhost:8080/account/test", u.String()) + }) + + t.Run("custom endpoint set with trailing slash removed", func(t *testing.T) { + meta := createTestMetadata(true, true, true) + meta[azauth.MetadataKeys["StorageEndpoint"][0]] = "https://localhost:8080/" + + m, err := parseMetadata(meta) + require.NoError(t, err) + + azEnvSettings, err := azauth.NewEnvironmentSettings(meta) + require.NoError(t, err) + + err = m.setCustomEndpoint(log, meta, azEnvSettings) + require.NoError(t, err) + + assert.Equal(t, "https://localhost:8080", m.customEndpoint) + + u, err := m.GetContainerURL(azEnvSettings) + require.NoError(t, err) + assert.Equal(t, "https://localhost:8080/account/test", u.String()) + }) + + t.Run("custom endpoint set to Azure Blob Storage endpoint", func(t *testing.T) { + logDest.Reset() + + meta := createTestMetadata(true, true, true) + meta[azauth.MetadataKeys["StorageEndpoint"][0]] = "https://account.blob.core.windows.net/test" + + m, err := parseMetadata(meta) + require.NoError(t, err) + + azEnvSettings, err := azauth.NewEnvironmentSettings(meta) + require.NoError(t, err) + + err = m.setCustomEndpoint(log, meta, azEnvSettings) + require.NoError(t, err) + + assert.Equal(t, "", m.customEndpoint) + + u, err := m.GetContainerURL(azEnvSettings) + require.NoError(t, err) + assert.Equal(t, "https://account.blob.core.windows.net/test", u.String()) + + assert.Contains(t, logDest.String(), "Metadata property endpoint is set to an Azure Blob Storage endpoint and will be ignored") + }) +} + func createTestMetadata(accountName bool, accountKey bool, container bool) map[string]string { m := map[string]string{} if accountName { diff --git a/internal/component/postgresql/metadata.go b/internal/component/postgresql/metadata.go index 8c4eb110e4..fb10bed6a0 100644 --- a/internal/component/postgresql/metadata.go +++ b/internal/component/postgresql/metadata.go @@ -14,9 +14,15 @@ limitations under the License. package postgresql import ( + "context" "fmt" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/dapr/components-contrib/internal/authentication/azure" "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/state" "github.com/dapr/kit/ptr" @@ -33,13 +39,19 @@ const ( ) type postgresMetadataStruct struct { - ConnectionString string - ConnectionMaxIdleTime time.Duration - TableName string // Could be in the format "schema.table" or just "table" - MetadataTableName string // Could be in the format "schema.table" or just "table" + ConnectionString string `mapstructure:"connectionString"` + ConnectionMaxIdleTime time.Duration `mapstructure:"connectionMaxIdleTime"` + TableName string `mapstructure:"tableName"` // Could be in the format "schema.table" or just "table" + MetadataTableName string `mapstructure:"metadataTableName"` // Could be in the format "schema.table" or just "table" + Timeout time.Duration `mapstructure:"timeoutInSeconds"` + CleanupInterval *time.Duration `mapstructure:"cleanupIntervalInSeconds"` + MaxConns int `mapstructure:"maxConns"` + UseAzureAD bool `mapstructure:"useAzureAD"` - Timeout time.Duration `mapstructure:"timeoutInSeconds"` - CleanupInterval *time.Duration `mapstructure:"cleanupIntervalInSeconds"` + // Set to true if the component can support authentication with Azure AD. + // This is different from the "useAzureAD" property above, which is provided by the user and instructs the component to authenticate using Azure AD. + azureADEnabled bool + azureEnv azure.EnvironmentSettings } func (m *postgresMetadataStruct) InitWithMetadata(meta state.Metadata) error { @@ -79,5 +91,67 @@ func (m *postgresMetadataStruct) InitWithMetadata(meta state.Metadata) error { } } + // Populate the Azure environment if using Azure AD + if m.azureADEnabled && m.UseAzureAD { + m.azureEnv, err = azure.NewEnvironmentSettings(meta.Properties) + if err != nil { + return err + } + } + return nil } + +// GetPgxPoolConfig returns the pgxpool.Config object that contains the credentials for connecting to Postgres. +func (m *postgresMetadataStruct) GetPgxPoolConfig() (*pgxpool.Config, error) { + // Get the config from the connection string + config, err := pgxpool.ParseConfig(m.ConnectionString) + if err != nil { + return nil, fmt.Errorf("failed to parse connection string: %w", err) + } + if m.ConnectionMaxIdleTime > 0 { + config.MaxConnIdleTime = m.ConnectionMaxIdleTime + } + if m.MaxConns > 1 { + config.MaxConns = int32(m.MaxConns) + } + + // Check if we should use Azure AD + if m.azureADEnabled && m.UseAzureAD { + tokenCred, errToken := m.azureEnv.GetTokenCredential() + if errToken != nil { + return nil, errToken + } + + // Reset the password + config.ConnConfig.Password = "" + + /*// For Azure AD, using SSL is required + // If not already enabled, configure TLS without certificate validation + if config.ConnConfig.TLSConfig == nil { + config.ConnConfig.TLSConfig = &tls.Config{ + //nolint:gosec + InsecureSkipVerify: true, + } + }*/ + + // We need to retrieve the token every time we attempt a new connection + // This is because tokens expire, and connections can drop and need to be re-established at any time + // Fortunately, we can do this with the "BeforeConnect" hook + config.BeforeConnect = func(ctx context.Context, cc *pgx.ConnConfig) error { + at, err := tokenCred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{ + m.azureEnv.Cloud.Services[azure.ServiceOSSRDBMS].Audience + "/.default", + }, + }) + if err != nil { + return err + } + + cc.Password = at.Token + return nil + } + } + + return config, nil +} diff --git a/internal/component/postgresql/postgresdbaccess.go b/internal/component/postgresql/postgresdbaccess.go index edf4e9b48b..5a5ec16e1b 100644 --- a/internal/component/postgresql/postgresdbaccess.go +++ b/internal/component/postgresql/postgresdbaccess.go @@ -67,7 +67,10 @@ func newPostgresDBAccess(logger logger.Logger, opts Options) *PostgresDBAccess { logger.Debug("Instantiating new Postgres state store") return &PostgresDBAccess{ - logger: logger, + logger: logger, + metadata: postgresMetadataStruct{ + azureADEnabled: opts.EnableAzureAD, + }, migrateFn: opts.MigrateFn, setQueryFn: opts.SetQueryFn, etagColumn: opts.ETagColumn, @@ -84,15 +87,11 @@ func (p *PostgresDBAccess) Init(ctx context.Context, meta state.Metadata) error return err } - config, err := pgxpool.ParseConfig(p.metadata.ConnectionString) + config, err := p.metadata.GetPgxPoolConfig() if err != nil { - err = fmt.Errorf("failed to parse connection string: %w", err) p.logger.Error(err) return err } - if p.metadata.ConnectionMaxIdleTime > 0 { - config.MaxConnIdleTime = p.metadata.ConnectionMaxIdleTime - } connCtx, connCancel := context.WithTimeout(ctx, p.metadata.Timeout) p.db, err = pgxpool.NewWithConfig(connCtx, config) diff --git a/internal/component/postgresql/postgresql.go b/internal/component/postgresql/postgresql.go index d5fc0f401d..f44d8abdfa 100644 --- a/internal/component/postgresql/postgresql.go +++ b/internal/component/postgresql/postgresql.go @@ -31,9 +31,10 @@ type PostgreSQL struct { } type Options struct { - MigrateFn func(context.Context, PGXPoolConn, MigrateOptions) error - SetQueryFn func(*state.SetRequest, SetQueryOptions) string - ETagColumn string + MigrateFn func(context.Context, PGXPoolConn, MigrateOptions) error + SetQueryFn func(*state.SetRequest, SetQueryOptions) string + ETagColumn string + EnableAzureAD bool } type MigrateOptions struct { diff --git a/internal/proto/state/etcd/v2/value.pb.go b/internal/proto/state/etcd/v2/value.pb.go new file mode 100644 index 0000000000..e411d91ab2 --- /dev/null +++ b/internal/proto/state/etcd/v2/value.pb.go @@ -0,0 +1,202 @@ +// +//Copyright 2021 The Dapr Authors +//Licensed under the Apache License, Version 2.0 (the "License"); +//you may not use this file except in compliance with the License. +//You may obtain a copy of the License at +//http://www.apache.org/licenses/LICENSE-2.0 +//Unless required by applicable law or agreed to in writing, software +//distributed under the License is distributed on an "AS IS" BASIS, +//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//See the License for the specific language governing permissions and +//limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.21.12 +// source: internal/proto/state/etcd/v2/value.proto + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Value is the value of the state key item. It contains the underlying data as +// well as necessary metadata. +type Value struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Required. The value of the state key item. + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + // Required. The creation time of the state key item. This is an + // approximation by the components-contrib instance since ETCD does not + // provide this information natively. + Ts *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=ts,proto3" json:"ts,omitempty"` + // Optional. The Time To Live of the state key item. The duration of the TTL + // is from the creation time of the key (`ts`). If not specified, the key has + // no TTL. + Ttl *durationpb.Duration `protobuf:"bytes,3,opt,name=ttl,proto3,oneof" json:"ttl,omitempty"` +} + +func (x *Value) Reset() { + *x = Value{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_state_etcd_v2_value_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Value) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Value) ProtoMessage() {} + +func (x *Value) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_state_etcd_v2_value_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Value.ProtoReflect.Descriptor instead. +func (*Value) Descriptor() ([]byte, []int) { + return file_internal_proto_state_etcd_v2_value_proto_rawDescGZIP(), []int{0} +} + +func (x *Value) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *Value) GetTs() *timestamppb.Timestamp { + if x != nil { + return x.Ts + } + return nil +} + +func (x *Value) GetTtl() *durationpb.Duration { + if x != nil { + return x.Ttl + } + return nil +} + +var File_internal_proto_state_etcd_v2_value_proto protoreflect.FileDescriptor + +var file_internal_proto_state_etcd_v2_value_proto_rawDesc = []byte{ + 0x0a, 0x28, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2f, 0x65, 0x74, 0x63, 0x64, 0x2f, 0x76, 0x32, 0x2f, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1e, 0x64, 0x61, 0x70, 0x72, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, + 0x73, 0x2e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x32, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x81, 0x01, 0x0a, 0x05, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x2a, 0x0a, 0x02, 0x74, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x02, 0x74, 0x73, 0x12, 0x30, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, + 0x03, 0x74, 0x74, 0x6c, 0x88, 0x01, 0x01, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x74, 0x74, 0x6c, 0x42, + 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, + 0x70, 0x72, 0x2f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x2d, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x69, 0x62, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2f, 0x65, 0x74, 0x63, 0x64, 0x2f, + 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_internal_proto_state_etcd_v2_value_proto_rawDescOnce sync.Once + file_internal_proto_state_etcd_v2_value_proto_rawDescData = file_internal_proto_state_etcd_v2_value_proto_rawDesc +) + +func file_internal_proto_state_etcd_v2_value_proto_rawDescGZIP() []byte { + file_internal_proto_state_etcd_v2_value_proto_rawDescOnce.Do(func() { + file_internal_proto_state_etcd_v2_value_proto_rawDescData = protoimpl.X.CompressGZIP(file_internal_proto_state_etcd_v2_value_proto_rawDescData) + }) + return file_internal_proto_state_etcd_v2_value_proto_rawDescData +} + +var file_internal_proto_state_etcd_v2_value_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_internal_proto_state_etcd_v2_value_proto_goTypes = []interface{}{ + (*Value)(nil), // 0: dapr.proto.components.state.v2.Value + (*timestamppb.Timestamp)(nil), // 1: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 2: google.protobuf.Duration +} +var file_internal_proto_state_etcd_v2_value_proto_depIdxs = []int32{ + 1, // 0: dapr.proto.components.state.v2.Value.ts:type_name -> google.protobuf.Timestamp + 2, // 1: dapr.proto.components.state.v2.Value.ttl:type_name -> google.protobuf.Duration + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_internal_proto_state_etcd_v2_value_proto_init() } +func file_internal_proto_state_etcd_v2_value_proto_init() { + if File_internal_proto_state_etcd_v2_value_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_internal_proto_state_etcd_v2_value_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Value); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_internal_proto_state_etcd_v2_value_proto_msgTypes[0].OneofWrappers = []interface{}{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_internal_proto_state_etcd_v2_value_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_internal_proto_state_etcd_v2_value_proto_goTypes, + DependencyIndexes: file_internal_proto_state_etcd_v2_value_proto_depIdxs, + MessageInfos: file_internal_proto_state_etcd_v2_value_proto_msgTypes, + }.Build() + File_internal_proto_state_etcd_v2_value_proto = out.File + file_internal_proto_state_etcd_v2_value_proto_rawDesc = nil + file_internal_proto_state_etcd_v2_value_proto_goTypes = nil + file_internal_proto_state_etcd_v2_value_proto_depIdxs = nil +} diff --git a/internal/proto/state/etcd/v2/value.proto b/internal/proto/state/etcd/v2/value.proto new file mode 100644 index 0000000000..7c1c6d6c3b --- /dev/null +++ b/internal/proto/state/etcd/v2/value.proto @@ -0,0 +1,39 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +syntax = "proto3"; + +package dapr.proto.components.state.v2; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; + +option go_package = "github.com/dapr/components-contrib/internal/proto/state/etcd/v2"; + + +// Value is the value of the state key item. It contains the underlying data as +// well as necessary metadata. +message Value { + // Required. The value of the state key item. + bytes data = 1; + + // Required. The creation time of the state key item. This is an + // approximation by the components-contrib instance since ETCD does not + // provide this information natively. + google.protobuf.Timestamp ts = 2; + + // Optional. The Time To Live of the state key item. The duration of the TTL + // is from the creation time of the key (`ts`). If not specified, the key has + // no TTL. + optional google.protobuf.Duration ttl = 3; +} diff --git a/internal/wasm/Makefile b/internal/wasm/Makefile new file mode 100644 index 0000000000..153969783c --- /dev/null +++ b/internal/wasm/Makefile @@ -0,0 +1,9 @@ +%/main.wasm: %/main.go + @(cd $(@D); tinygo build -o main.wasm -scheduler=none --no-debug -target=wasi main.go) + +.PHONY: build-tinygo +build-tinygo: testdata/args/main.wasm testdata/strict/main.wasm + +.PHONY: testdata +testdata: + @$(MAKE) build-tinygo diff --git a/internal/wasm/testdata/args/main.wasm b/internal/wasm/testdata/args/main.wasm index 1b0ad29652..4a164b6f0a 100755 Binary files a/internal/wasm/testdata/args/main.wasm and b/internal/wasm/testdata/args/main.wasm differ diff --git a/internal/wasm/testdata/strict/main.go b/internal/wasm/testdata/strict/main.go new file mode 100644 index 0000000000..b4fa2f4b8d --- /dev/null +++ b/internal/wasm/testdata/strict/main.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implieout. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/rand" + "encoding/hex" + "time" +) + +func main() { + t := time.Now() // should use walltime and nanotime + println(t.Nanosecond()) + println(time.Since(t)) // should use nanotime + time.Sleep(50 * time.Millisecond) // uses nanosleep + b := make([]byte, 5) + if _, err := rand.Read(b); err != nil { // uses randSource + panic(err) + } + println(hex.EncodeToString(b)) +} diff --git a/internal/wasm/testdata/strict/main.wasm b/internal/wasm/testdata/strict/main.wasm new file mode 100755 index 0000000000..c757fc6b83 Binary files /dev/null and b/internal/wasm/testdata/strict/main.wasm differ diff --git a/internal/wasm/wasm.go b/internal/wasm/wasm.go index 2ae1e04065..c008e5b056 100644 --- a/internal/wasm/wasm.go +++ b/internal/wasm/wasm.go @@ -1,13 +1,18 @@ package wasm import ( - "context" + "crypto/rand" "errors" "fmt" "net/url" "os" "path" "strings" + "sync/atomic" + "time" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/sys" "github.com/dapr/components-contrib/metadata" ) @@ -24,6 +29,19 @@ type InitMetadata struct { // retrieved via HTTP. In these cases, no filesystem will be mounted. URL string `mapstructure:"url"` + // StrictSandbox when true uses fake sources to avoid vulnerabilities such + // as timing attacks. + // + // # Affected configuration + // + // - sys.Walltime increments with a constant value when read, initially a + // second resolution of the current system time. + // - sys.Nanotime increments with a constant value when read, initially + // zero. + // - sys.Nanosleep returns immediately. + // - Random number generators are seeded with a deterministic source. + StrictSandbox bool `mapstructure:"strictSandbox"` + // Guest is WebAssembly binary implementing the guest, loaded from URL. Guest []byte `mapstructure:"-"` @@ -32,7 +50,7 @@ type InitMetadata struct { } // GetInitMetadata returns InitMetadata from the input metadata. -func GetInitMetadata(ctx context.Context, md metadata.Base) (*InitMetadata, error) { +func GetInitMetadata(md metadata.Base) (*InitMetadata, error) { // Note: the ctx will be used for other schemes such as HTTP and OCI. var m InitMetadata @@ -75,3 +93,31 @@ func GetInitMetadata(ctx context.Context, md metadata.Base) (*InitMetadata, erro return &m, nil } + +// NewModuleConfig returns a new module config appropriate for the initialized +// metadata. +func NewModuleConfig(m *InitMetadata) wazero.ModuleConfig { + if !m.StrictSandbox { + // The below violate sand-boxing, but allow code to behave as expected. + return wazero.NewModuleConfig(). + WithRandSource(rand.Reader). + WithSysNanotime(). + WithSysWalltime(). + WithSysNanosleep() + } + + // wazero's default is strict as defined here, except walltime. wazero + // does not return a real clock reading by default for performance and + // determinism reasons. + // See https://github.com/tetratelabs/wazero/blob/main/RATIONALE.md#syswalltime-and-nanotime + return wazero.NewModuleConfig(). + WithWalltime(newFakeWalltime(), sys.ClockResolution(time.Millisecond)) +} + +func newFakeWalltime() sys.Walltime { + t := time.Now().Unix() * int64(time.Second) + return func() (sec int64, nsec int32) { + wt := atomic.AddInt64(&t, int64(time.Millisecond)) + return wt / 1e9, int32(wt % 1e9) + } +} diff --git a/internal/wasm/wasm_test.go b/internal/wasm/wasm_test.go index ebfb657bcf..9c07dc0c84 100644 --- a/internal/wasm/wasm_test.go +++ b/internal/wasm/wasm_test.go @@ -1,11 +1,15 @@ package wasm import ( + "bytes" "context" _ "embed" "testing" + "time" "github.com/stretchr/testify/require" + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" "github.com/dapr/components-contrib/metadata" ) @@ -16,7 +20,7 @@ const ( ) //go:embed testdata/args/main.wasm -var urlArgsBin []byte +var binArgs []byte func TestGetInitMetadata(t *testing.T) { type testCase struct { @@ -34,10 +38,23 @@ func TestGetInitMetadata(t *testing.T) { }}, expected: &InitMetadata{ URL: urlArgsFile, - Guest: urlArgsBin, + Guest: binArgs, GuestName: "main", }, }, + { + name: "file valid - strictSandbox", + metadata: metadata.Base{Properties: map[string]string{ + "url": urlArgsFile, + "strictSandbox": "true", + }}, + expected: &InitMetadata{ + URL: urlArgsFile, + Guest: binArgs, + GuestName: "main", + StrictSandbox: true, + }, + }, { name: "empty url", metadata: metadata.Base{Properties: map[string]string{}}, @@ -105,8 +122,7 @@ func TestGetInitMetadata(t *testing.T) { for _, tt := range tests { tc := tt t.Run(tc.name, func(t *testing.T) { - ctx := context.Background() - md, err := GetInitMetadata(ctx, tc.metadata) + md, err := GetInitMetadata(tc.metadata) if tc.expectedErr == "" { require.NoError(t, err) require.Equal(t, tc.expected, md) @@ -117,3 +133,67 @@ func TestGetInitMetadata(t *testing.T) { }) } } + +//go:embed testdata/strict/main.wasm +var binStrict []byte + +func TestNewModuleConfig(t *testing.T) { + type testCase struct { + name string + metadata *InitMetadata + minDuration, maxDuration time.Duration + } + + tests := []testCase{ + { + name: "strictSandbox = false", + metadata: &InitMetadata{Guest: binStrict}, + // In CI, Nanosleep(50ms) returned after 197ms. + // As we can't control the platform clock, we have to be lenient + minDuration: 50 * time.Millisecond, + maxDuration: 50 * time.Millisecond * 5, + }, + { + name: "strictSandbox = true", + metadata: &InitMetadata{StrictSandbox: true, Guest: binStrict}, + minDuration: 10 * time.Microsecond, + maxDuration: 1 * time.Millisecond, + }, + } + + ctx := context.Background() + rt := wazero.NewRuntime(ctx) + defer rt.Close(ctx) + wasi_snapshot_preview1.MustInstantiate(ctx, rt) + + for _, tt := range tests { + tc := tt + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + + cfg := NewModuleConfig(tc.metadata). + WithStdout(&out).WithStderr(&out). + WithStartFunctions() // don't include instantiation in duration + mod, err := rt.InstantiateWithConfig(ctx, tc.metadata.Guest, cfg) + require.NoError(t, err) + + start := time.Now() + _, err = mod.ExportedFunction("_start").Call(ctx) + require.NoError(t, err) + duration := time.Since(start) + + // TODO: TinyGo doesn't seem to use monotonic time. Track below: + // https://github.com/tinygo-org/tinygo/issues/3776 + deterministicOut := `2000000 +1000000 +6393cff83a +` + if tc.metadata.StrictSandbox { + require.Equal(t, deterministicOut, out.String()) + } else { + require.NotEqual(t, deterministicOut, out.String()) + } + require.True(t, duration > tc.minDuration && duration < tc.maxDuration, duration) + }) + } +} diff --git a/middleware/http/wasm/example/go.mod b/middleware/http/wasm/example/go.mod index b8651c468e..ec15f4f775 100644 --- a/middleware/http/wasm/example/go.mod +++ b/middleware/http/wasm/example/go.mod @@ -2,4 +2,4 @@ module github.com/dapr/components-contrib/middleware/wasm/example go 1.20 -require github.com/http-wasm/http-wasm-guest-tinygo v0.1.0 +require github.com/http-wasm/http-wasm-guest-tinygo v0.3.0 diff --git a/middleware/http/wasm/example/go.sum b/middleware/http/wasm/example/go.sum index 6a27987f7e..cb03367741 100644 --- a/middleware/http/wasm/example/go.sum +++ b/middleware/http/wasm/example/go.sum @@ -1,2 +1,2 @@ -github.com/http-wasm/http-wasm-guest-tinygo v0.1.0 h1:vcYHJkbfQ2G0bD/zupIzHe/h1LZQJiVGdn5eZZTJM88= -github.com/http-wasm/http-wasm-guest-tinygo v0.1.0/go.mod h1:/3UO8OXP9nxe7d2qJ5ifTVkqM7KjaXxUZLoqBsDXpy0= +github.com/http-wasm/http-wasm-guest-tinygo v0.3.0 h1:J11RX1ajUC6fhVtv3ZU5k66SL4EB4DhThHmz4Ilwevw= +github.com/http-wasm/http-wasm-guest-tinygo v0.3.0/go.mod h1:zcKr7h/t5ha2ZWIMwV4iOqhfC/qno/tNPYgybVkn/MQ= diff --git a/middleware/http/wasm/example/router.wasm b/middleware/http/wasm/example/router.wasm index 212b339da3..da96de2079 100755 Binary files a/middleware/http/wasm/example/router.wasm and b/middleware/http/wasm/example/router.wasm differ diff --git a/middleware/http/wasm/httpwasm.go b/middleware/http/wasm/httpwasm.go index d8b110a70e..aa024f6a55 100644 --- a/middleware/http/wasm/httpwasm.go +++ b/middleware/http/wasm/httpwasm.go @@ -3,7 +3,6 @@ package wasm import ( "bytes" "context" - "crypto/rand" "fmt" "net/http" "reflect" @@ -12,7 +11,6 @@ import ( "github.com/http-wasm/http-wasm-host-go/api" "github.com/http-wasm/http-wasm-host-go/handler" wasmnethttp "github.com/http-wasm/http-wasm-host-go/handler/nethttp" - "github.com/tetratelabs/wazero" "github.com/dapr/components-contrib/internal/wasm" mdutils "github.com/dapr/components-contrib/metadata" @@ -38,7 +36,7 @@ func (m *middleware) GetHandler(ctx context.Context, metadata dapr.Metadata) (fu // getHandler is extracted for unit testing. func (m *middleware) getHandler(ctx context.Context, metadata dapr.Metadata) (*requestHandler, error) { - meta, err := wasm.GetInitMetadata(ctx, metadata.Base) + meta, err := wasm.GetInitMetadata(metadata.Base) if err != nil { return nil, fmt.Errorf("wasm: failed to parse metadata: %w", err) } @@ -46,15 +44,10 @@ func (m *middleware) getHandler(ctx context.Context, metadata dapr.Metadata) (*r var stdout, stderr bytes.Buffer mw, err := wasmnethttp.NewMiddleware(ctx, meta.Guest, handler.Logger(m), - handler.ModuleConfig(wazero.NewModuleConfig(). + handler.ModuleConfig(wasm.NewModuleConfig(meta). WithName(meta.GuestName). - WithStdout(&stdout). // reset per request - WithStderr(&stderr). // reset per request - // The below violate sand-boxing, but allow code to behave as expected. - WithRandSource(rand.Reader). - WithSysNanosleep(). - WithSysWalltime(). - WithSysNanosleep())) + WithStdout(&stdout). // reset per request + WithStderr(&stderr))) // reset per request if err != nil { return nil, err } diff --git a/middleware/http/wasm/internal/e2e-guests/go.mod b/middleware/http/wasm/internal/e2e-guests/go.mod index 42277c93d1..e24ba6af60 100644 --- a/middleware/http/wasm/internal/e2e-guests/go.mod +++ b/middleware/http/wasm/internal/e2e-guests/go.mod @@ -2,4 +2,4 @@ module github.com/dapr/components-contrib/middleware/wasm/internal go 1.20 -require github.com/http-wasm/http-wasm-guest-tinygo v0.1.0 +require github.com/http-wasm/http-wasm-guest-tinygo v0.3.0 diff --git a/middleware/http/wasm/internal/e2e-guests/go.sum b/middleware/http/wasm/internal/e2e-guests/go.sum index 6a27987f7e..cb03367741 100644 --- a/middleware/http/wasm/internal/e2e-guests/go.sum +++ b/middleware/http/wasm/internal/e2e-guests/go.sum @@ -1,2 +1,2 @@ -github.com/http-wasm/http-wasm-guest-tinygo v0.1.0 h1:vcYHJkbfQ2G0bD/zupIzHe/h1LZQJiVGdn5eZZTJM88= -github.com/http-wasm/http-wasm-guest-tinygo v0.1.0/go.mod h1:/3UO8OXP9nxe7d2qJ5ifTVkqM7KjaXxUZLoqBsDXpy0= +github.com/http-wasm/http-wasm-guest-tinygo v0.3.0 h1:J11RX1ajUC6fhVtv3ZU5k66SL4EB4DhThHmz4Ilwevw= +github.com/http-wasm/http-wasm-guest-tinygo v0.3.0/go.mod h1:zcKr7h/t5ha2ZWIMwV4iOqhfC/qno/tNPYgybVkn/MQ= diff --git a/middleware/http/wasm/internal/e2e-guests/output/main.wasm b/middleware/http/wasm/internal/e2e-guests/output/main.wasm index 28b3e3f061..1b2f1e0fe9 100755 Binary files a/middleware/http/wasm/internal/e2e-guests/output/main.wasm and b/middleware/http/wasm/internal/e2e-guests/output/main.wasm differ diff --git a/middleware/http/wasm/internal/e2e-guests/rewrite/main.wasm b/middleware/http/wasm/internal/e2e-guests/rewrite/main.wasm index f3bc7e41bf..382b68f63e 100755 Binary files a/middleware/http/wasm/internal/e2e-guests/rewrite/main.wasm and b/middleware/http/wasm/internal/e2e-guests/rewrite/main.wasm differ diff --git a/pubsub/azure/eventhubs/metadata.yaml b/pubsub/azure/eventhubs/metadata.yaml index 9cbf4edb6f..b9cd436f24 100644 --- a/pubsub/azure/eventhubs/metadata.yaml +++ b/pubsub/azure/eventhubs/metadata.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=../../../../component-metadata-schema.json +# yaml-language-server: $schema=../../../component-metadata-schema.json schemaVersion: v1 type: pubsub name: azure.eventhubs @@ -17,7 +17,8 @@ authenticationProfiles: sensitive: true description: | Connection string for the Event Hub or the Event Hub namespace. - example: '"Endpoint=sb://{EventHubNamespace}.servicebus.windows.net/;SharedAccessKeyName={PolicyName};SharedAccessKey={Key};EntityPath={EventHub}"' + example: | + "Endpoint=sb://{EventHubNamespace}.servicebus.windows.net/;SharedAccessKeyName={PolicyName};SharedAccessKey={Key};EntityPath={EventHub}" builtinAuthenticationProfiles: - name: "azuread" metadata: @@ -31,6 +32,7 @@ builtinAuthenticationProfiles: type: bool required: false default: "false" + example: "false" description: | Allow management of the Event Hub namespace and storage account. @@ -81,6 +83,8 @@ metadata: Connection string for the checkpoint store, alternative to specifying storageAccountKey. Property "storageAccountKey" is ignored when "storageConnectionString" is present + example: | + "BlobEndpoint=https://storagesample.blob.core.windows.net;..." - name: storageAccountName type: string required: true diff --git a/pubsub/azure/servicebus/queues/metadata.yaml b/pubsub/azure/servicebus/queues/metadata.yaml index 613edcf7fd..b931d95a40 100644 --- a/pubsub/azure/servicebus/queues/metadata.yaml +++ b/pubsub/azure/servicebus/queues/metadata.yaml @@ -18,7 +18,8 @@ authenticationProfiles: required: true sensitive: true description: "Shared access policy connection string for the Service Bus." - example: '"Endpoint=sb://{ServiceBusNamespace}.servicebus.windows.net/;SharedAccessKeyName={PolicyName};SharedAccessKey={Key};EntityPath={ServiceBus}"' + example: | + "Endpoint=sb://{ServiceBusNamespace}.servicebus.windows.net/;SharedAccessKeyName={PolicyName};SharedAccessKey={Key};EntityPath={ServiceBus}" # If omitted, uses the same values as ".binding" binding: output: true diff --git a/pubsub/azure/servicebus/topics/metadata.yaml b/pubsub/azure/servicebus/topics/metadata.yaml index daf2d01c24..e54e2b9fe8 100644 --- a/pubsub/azure/servicebus/topics/metadata.yaml +++ b/pubsub/azure/servicebus/topics/metadata.yaml @@ -18,7 +18,8 @@ authenticationProfiles: required: true sensitive: true description: "Shared access policy connection string for the Service Bus." - example: '"Endpoint=sb://{ServiceBusNamespace}.servicebus.windows.net/;SharedAccessKeyName={PolicyName};SharedAccessKey={Key};EntityPath={ServiceBus}"' + example: | + "Endpoint=sb://{ServiceBusNamespace}.servicebus.windows.net/;SharedAccessKeyName={PolicyName};SharedAccessKey={Key};EntityPath={ServiceBus}" # If omitted, uses the same values as ".binding" binding: output: true diff --git a/pubsub/rabbitmq/metadata.go b/pubsub/rabbitmq/metadata.go index f743b64e30..0cbad4584a 100644 --- a/pubsub/rabbitmq/metadata.go +++ b/pubsub/rabbitmq/metadata.go @@ -77,6 +77,7 @@ const ( metadataPublisherConfirmKey = "publisherConfirm" metadataSaslExternal = "saslExternal" metadataMaxPriority = "maxPriority" + metadataQueueNameKey = "queueName" defaultReconnectWaitSeconds = 3 diff --git a/pubsub/rabbitmq/rabbitmq.go b/pubsub/rabbitmq/rabbitmq.go index 4f60f992a2..249d5d8b70 100644 --- a/pubsub/rabbitmq/rabbitmq.go +++ b/pubsub/rabbitmq/rabbitmq.go @@ -308,11 +308,14 @@ func (r *rabbitMQ) Subscribe(ctx context.Context, req pubsub.SubscribeRequest, h return errors.New("component is closed") } - if r.metadata.ConsumerID == "" { - return errors.New("consumerID is required for subscriptions") + queueName := req.Metadata[metadataQueueNameKey] + if queueName == "" { + if r.metadata.ConsumerID == "" { + return errors.New("consumerID is required for subscriptions that don't specify a queue name") + } + queueName = fmt.Sprintf("%s-%s", r.metadata.ConsumerID, req.Topic) } - queueName := fmt.Sprintf("%s-%s", r.metadata.ConsumerID, req.Topic) r.logger.Infof("%s subscribe to topic/queue '%s/%s'", logMessagePrefix, req.Topic, queueName) // Do not set a timeout on the context, as we're just waiting for the first ack; we're using a semaphore instead diff --git a/pubsub/rabbitmq/rabbitmq_test.go b/pubsub/rabbitmq/rabbitmq_test.go index 864b8b5b37..f9248df87c 100644 --- a/pubsub/rabbitmq/rabbitmq_test.go +++ b/pubsub/rabbitmq/rabbitmq_test.go @@ -36,7 +36,7 @@ func newBroker() *rabbitMQInMemoryBroker { } } -func newRabbitMQTest(broker *rabbitMQInMemoryBroker) pubsub.PubSub { +func newRabbitMQTest(broker *rabbitMQInMemoryBroker) *rabbitMQ { return &rabbitMQ{ declaredExchanges: make(map[string]bool), logger: logger.NewLogger("test"), @@ -48,7 +48,7 @@ func newRabbitMQTest(broker *rabbitMQInMemoryBroker) pubsub.PubSub { } } -func TestNoConsumer(t *testing.T) { +func TestNoConsumerOrQueueName(t *testing.T) { broker := newBroker() pubsubRabbitMQ := newRabbitMQTest(broker) metadata := pubsub.Metadata{Base: mdata.Base{ @@ -59,7 +59,7 @@ func TestNoConsumer(t *testing.T) { err := pubsubRabbitMQ.Init(context.Background(), metadata) assert.NoError(t, err) err = pubsubRabbitMQ.Subscribe(context.Background(), pubsub.SubscribeRequest{}, nil) - assert.Contains(t, err.Error(), "consumerID is required for subscriptions") + assert.Contains(t, err.Error(), "consumerID is required for subscriptions that don't specify a queue name") } func TestPublishAndSubscribeWithPriorityQueue(t *testing.T) { @@ -118,7 +118,7 @@ func TestConcurrencyMode(t *testing.T) { }} err := pubsubRabbitMQ.Init(context.Background(), metadata) assert.Nil(t, err) - assert.Equal(t, pubsub.Parallel, pubsubRabbitMQ.(*rabbitMQ).metadata.Concurrency) + assert.Equal(t, pubsub.Parallel, pubsubRabbitMQ.metadata.Concurrency) }) t.Run("single", func(t *testing.T) { @@ -133,7 +133,7 @@ func TestConcurrencyMode(t *testing.T) { }} err := pubsubRabbitMQ.Init(context.Background(), metadata) assert.Nil(t, err) - assert.Equal(t, pubsub.Single, pubsubRabbitMQ.(*rabbitMQ).metadata.Concurrency) + assert.Equal(t, pubsub.Single, pubsubRabbitMQ.metadata.Concurrency) }) t.Run("default", func(t *testing.T) { @@ -147,51 +147,91 @@ func TestConcurrencyMode(t *testing.T) { }} err := pubsubRabbitMQ.Init(context.Background(), metadata) assert.Nil(t, err) - assert.Equal(t, pubsub.Parallel, pubsubRabbitMQ.(*rabbitMQ).metadata.Concurrency) + assert.Equal(t, pubsub.Parallel, pubsubRabbitMQ.metadata.Concurrency) }) } func TestPublishAndSubscribe(t *testing.T) { - broker := newBroker() - pubsubRabbitMQ := newRabbitMQTest(broker) - metadata := pubsub.Metadata{Base: mdata.Base{ - Properties: map[string]string{ - metadataHostnameKey: "anyhost", - metadataConsumerIDKey: "consumer", + tests := []struct { + name string + componentMetadata map[string]string + subscribeMetadata map[string]string + topic string + declaredQueues []string + }{ + { + name: "only consumer id", + componentMetadata: map[string]string{ + metadataHostnameKey: "anyhost", + metadataConsumerIDKey: "consumer", + }, + topic: "mytopic", + declaredQueues: []string{"consumer-mytopic"}, + }, + { + name: "only queue name", + componentMetadata: map[string]string{ + metadataHostnameKey: "anyhost", + }, + subscribeMetadata: map[string]string{ + metadataQueueNameKey: "myqueue", + }, + topic: "mytopic", + declaredQueues: []string{"myqueue"}, + }, + { + name: "queue name takes precedence over consumer id", + componentMetadata: map[string]string{ + metadataHostnameKey: "anyhost", + metadataConsumerIDKey: "consumer", + }, + subscribeMetadata: map[string]string{ + metadataQueueNameKey: "myqueue", + }, + topic: "mytopic", + declaredQueues: []string{"myqueue"}, }, - }} - err := pubsubRabbitMQ.Init(context.Background(), metadata) - assert.Nil(t, err) - assert.Equal(t, int32(1), broker.connectCount.Load()) - assert.Equal(t, int32(0), broker.closeCount.Load()) - - topic := "mytopic" - - messageCount := 0 - lastMessage := "" - processed := make(chan bool) - handler := func(ctx context.Context, msg *pubsub.NewMessage) error { - messageCount++ - lastMessage = string(msg.Data) - processed <- true - - return nil } - - err = pubsubRabbitMQ.Subscribe(context.Background(), pubsub.SubscribeRequest{Topic: topic}, handler) - assert.Nil(t, err) - - err = pubsubRabbitMQ.Publish(context.Background(), &pubsub.PublishRequest{Topic: topic, Data: []byte("hello world")}) - assert.Nil(t, err) - <-processed - assert.Equal(t, 1, messageCount) - assert.Equal(t, "hello world", lastMessage) - - err = pubsubRabbitMQ.Publish(context.Background(), &pubsub.PublishRequest{Topic: topic, Data: []byte("foo bar")}) - assert.Nil(t, err) - <-processed - assert.Equal(t, 2, messageCount) - assert.Equal(t, "foo bar", lastMessage) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + broker := newBroker() + pubsubRabbitMQ := newRabbitMQTest(broker) + metadata := pubsub.Metadata{Base: mdata.Base{ + Properties: test.componentMetadata, + }} + err := pubsubRabbitMQ.Init(context.Background(), metadata) + assert.Nil(t, err) + assert.Equal(t, int32(1), broker.connectCount.Load()) + assert.Equal(t, int32(0), broker.closeCount.Load()) + + messageCount := 0 + lastMessage := "" + processed := make(chan bool) + handler := func(ctx context.Context, msg *pubsub.NewMessage) error { + messageCount++ + lastMessage = string(msg.Data) + processed <- true + return nil + } + + err = pubsubRabbitMQ.Subscribe(context.Background(), pubsub.SubscribeRequest{Topic: test.topic, Metadata: test.subscribeMetadata}, handler) + assert.NoError(t, err) + assert.True(t, pubsubRabbitMQ.declaredExchanges[test.topic]) + assert.ElementsMatch(t, test.declaredQueues, broker.declaredQueues) + + err = pubsubRabbitMQ.Publish(context.Background(), &pubsub.PublishRequest{Topic: test.topic, Data: []byte("hello world")}) + assert.NoError(t, err) + <-processed + assert.Equal(t, 1, messageCount) + assert.Equal(t, "hello world", lastMessage) + + err = pubsubRabbitMQ.Publish(context.Background(), &pubsub.PublishRequest{Topic: test.topic, Data: []byte("foo bar")}) + assert.NoError(t, err) + <-processed + assert.Equal(t, 2, messageCount) + assert.Equal(t, "foo bar", lastMessage) + }) + } } func TestPublishReconnect(t *testing.T) { @@ -385,10 +425,10 @@ func createAMQPMessage(body []byte) amqp.Delivery { } type rabbitMQInMemoryBroker struct { - buffer chan amqp.Delivery - - connectCount atomic.Int32 - closeCount atomic.Int32 + buffer chan amqp.Delivery + declaredQueues []string + connectCount atomic.Int32 + closeCount atomic.Int32 } func (r *rabbitMQInMemoryBroker) Qos(prefetchCount, prefetchSize int, global bool) error { @@ -412,6 +452,7 @@ func (r *rabbitMQInMemoryBroker) PublishWithDeferredConfirmWithContext(ctx conte } func (r *rabbitMQInMemoryBroker) QueueDeclare(name string, durable bool, autoDelete bool, exclusive bool, noWait bool, args amqp.Table) (amqp.Queue, error) { + r.declaredQueues = append(r.declaredQueues, name) return amqp.Queue{Name: name}, nil } diff --git a/secretstores/gcp/secretmanager/metadata.yaml b/secretstores/gcp/secretmanager/metadata.yaml new file mode 100644 index 0000000000..3877d1c27b --- /dev/null +++ b/secretstores/gcp/secretmanager/metadata.yaml @@ -0,0 +1,94 @@ +# yaml-language-server: $schema=../../../component-metadata-schema.json +schemaVersion: v1 +type: secretstores +name: gcp.secretmanager +version: v1 +status: alpha +title: "GCP Secret Manager" +urls: + - title: Reference + url: https://docs.dapr.io/reference/components-reference/supported-secret-stores/gcp-secret-manager/ +authenticationProfiles: + - title: "GCP API call Authentication" + description: | + Authenticate authenticates API calls with the given service account or refresh token JSON credentials. + metadata: + - name: private_key_id + required: true + sensitive: true + description: | + GCP private key id. + example: '"privateKeyId"' + - name: private_key + required: true + sensitive: true + description: | + GCP credentials private key. Replace with x509 cert. + example: '"12345-12345"' +metadata: + - name: type + type: string + required: true + description: | + The GCP credentials type. + example: '"service_account"' + - name: project_id + type: string + required: true + description: | + GCP project id. + example: '"projectId"' + - name: client_email + type: string + required: true + description: | + GCP client email. + example: '"client@email.com"' + - name: client_id + type: string + required: false + description: | + GCP client id. + example: '"11111111"' + - name: auth_uri + type: string + required: false + description: | + Google account OAuth endpoint. + example: '"https://accounts.google.com/o/oauth2/auth"' + - name: token_uri + type: string + required: false + description: | + Google account token uri. + example: '"https://oauth2.googleapis.com/token"' + - name: auth_provider_x509_cert_url + type: string + required: false + description: | + GCP credentials cert url. + example: '"https://www.googleapis.com/oauth2/v1/certs"' + - name: client_x509_cert_url + type: string + required: false + description: | + GCP credentials project x509 cert url. + example: '"https://www.googleapis.com/robot/v1/metadata/x509/.iam.gserviceaccount.com"' + - name: decodeBase64 + type: bool + required: false + default: 'false' + description: | + Configuration to decode base64 file content before saving to bucket storage. + (In case of saving a file with binary content). true is the only allowed positive value. + Other positive variations like "True", "1" are not acceptable. Defaults to false. + example: '"true, false"' + - name: encodeBase64 + type: bool + required: false + default: 'false' + description: | + Configuration to encode base64 file content before return the content. + (In case of opening a file with binary content). true is the only allowed positive value. + Other positive variations like "True", "1" are not acceptable. Defaults to false. + example: '"true, false"' \ No newline at end of file diff --git a/state/azure/blobstorage/metadata.yaml b/state/azure/blobstorage/metadata.yaml index 9e796c1f2b..f90198900d 100644 --- a/state/azure/blobstorage/metadata.yaml +++ b/state/azure/blobstorage/metadata.yaml @@ -14,11 +14,11 @@ capabilities: builtinAuthenticationProfiles: - name: "azuread" metadata: - - name: accountName - required: true - sensitive: false - description: "The storage account name" - example: '"mystorageaccount"' + - name: accountName + required: true + sensitive: false + description: "The storage account name" + example: '"mystorageaccount"' authenticationProfiles: - title: "Connection string" description: "Authenticate using a connection string." diff --git a/state/cassandra/cassandra.go b/state/cassandra/cassandra.go index 0006897d48..9a74a962a0 100644 --- a/state/cassandra/cassandra.go +++ b/state/cassandra/cassandra.go @@ -15,9 +15,11 @@ package cassandra import ( "context" + "errors" "fmt" "reflect" "strconv" + "time" "github.com/gocql/gocql" jsoniter "github.com/json-iterator/go" @@ -240,7 +242,8 @@ func (c *Cassandra) Get(ctx context.Context, req *state.GetRequest) (*state.GetR session = sess } - results, err := session.Query(fmt.Sprintf("SELECT value FROM %s WHERE key = ?", c.table), req.Key).WithContext(ctx).Iter().SliceMap() + const selectQuery = "SELECT value, TTL(value) AS ttl, toTimestamp(now()) AS now FROM %s WHERE key = ?" + results, err := session.Query(fmt.Sprintf(selectQuery, c.table), req.Key).WithContext(ctx).Iter().SliceMap() if err != nil { return nil, err } @@ -249,8 +252,20 @@ func (c *Cassandra) Get(ctx context.Context, req *state.GetRequest) (*state.GetR return &state.GetResponse{}, nil } + var metadata map[string]string + if ttl := results[0]["ttl"].(int); ttl > 0 { + now, ok := results[0]["now"].(time.Time) + if !ok { + return nil, errors.New("failed to parse cassandra timestamp") + } + metadata = map[string]string{ + state.GetRespMetaKeyTTLExpireTime: now.Add(time.Duration(ttl) * time.Second).UTC().Format(time.RFC3339), + } + } + return &state.GetResponse{ - Data: results[0]["value"].([]byte), + Data: results[0]["value"].([]byte), + Metadata: metadata, }, nil } @@ -311,3 +326,14 @@ func (c *Cassandra) GetComponentMetadata() map[string]string { metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) return metadataInfo } + +// Close the connection to Cassandra. +func (c *Cassandra) Close() error { + if c.session == nil { + return nil + } + + c.session.Close() + + return nil +} diff --git a/state/etcd/etcd.go b/state/etcd/etcd.go index 5dc68f60f8..30b5c93df1 100644 --- a/state/etcd/etcd.go +++ b/state/etcd/etcd.go @@ -17,7 +17,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "encoding/json" "errors" "fmt" "reflect" @@ -43,6 +42,7 @@ type Etcd struct { keyPrefixPath string features []state.Feature logger logger.Logger + schema schemaMarshaller } type etcdConfig struct { @@ -55,9 +55,19 @@ type etcdConfig struct { Key string `json:"key"` } -// NewEtcdStateStore returns a new etcd state store. -func NewEtcdStateStore(logger logger.Logger) state.Store { +// NewEtcdStateStoreV1 returns a new etcd state store for schema V1. +func NewEtcdStateStoreV1(logger logger.Logger) state.Store { + return newETCD(logger, schemaV1{}) +} + +// NewEtcdStateStoreV2 returns a new etcd state store for schema V2. +func NewEtcdStateStoreV2(logger logger.Logger) state.Store { + return newETCD(logger, schemaV2{}) +} + +func newETCD(logger logger.Logger, schema schemaMarshaller) state.Store { s := &Etcd{ + schema: schema, logger: logger, features: []state.Feature{state.FeatureETag, state.FeatureTransactional}, } @@ -141,9 +151,15 @@ func (e *Etcd) Get(ctx context.Context, req *state.GetRequest) (*state.GetRespon return &state.GetResponse{}, nil } + data, metadata, err := e.schema.decode(resp.Kvs[0].Value) + if err != nil { + return nil, err + } + return &state.GetResponse{ - Data: resp.Kvs[0].Value, - ETag: ptr.Of(strconv.Itoa(int(resp.Kvs[0].ModRevision))), + Data: data, + ETag: ptr.Of(strconv.Itoa(int(resp.Kvs[0].ModRevision))), + Metadata: metadata, }, nil } @@ -160,20 +176,20 @@ func (e *Etcd) Set(ctx context.Context, req *state.SetRequest) error { return err } - reqVal, err := stateutils.Marshal(req.Value, json.Marshal) - if err != nil { - return err - } - - return e.doSet(ctx, keyWithPath, string(reqVal), req.ETag, ttlInSeconds) + return e.doSet(ctx, keyWithPath, req.Value, req.ETag, ttlInSeconds) } -func (e *Etcd) doSet(ctx context.Context, key, reqVal string, etag *string, ttlInSeconds int64) error { +func (e *Etcd) doSet(ctx context.Context, key string, val any, etag *string, ttlInSeconds *int64) error { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - if ttlInSeconds > 0 { - resp, err := e.client.Grant(ctx, ttlInSeconds) + reqVal, err := e.schema.encode(val, ttlInSeconds) + if err != nil { + return err + } + + if ttlInSeconds != nil { + resp, err := e.client.Grant(ctx, *ttlInSeconds) if err != nil { return fmt.Errorf("couldn't grant lease %s: %w", key, err) } @@ -207,22 +223,18 @@ func (e *Etcd) doSet(ctx context.Context, key, reqVal string, etag *string, ttlI return nil } -func (e *Etcd) doSetValidateParameters(req *state.SetRequest) (int64, error) { +func (e *Etcd) doSetValidateParameters(req *state.SetRequest) (*int64, error) { err := state.CheckRequestOptions(req.Options) if err != nil { - return 0, err + return nil, err } - var ttlVal int64 - ttlInSeconds, err := stateutils.ParseTTL(req.Metadata) + ttlInSeconds, err := stateutils.ParseTTL64(req.Metadata) if err != nil { - return 0, err - } - if ttlInSeconds != nil { - ttlVal = int64(*ttlInSeconds) + return nil, err } - return ttlVal, nil + return ttlInSeconds, nil } // Delete performes a Etcd KV delete operation. @@ -339,7 +351,7 @@ func (e *Etcd) Multi(ctx context.Context, request *state.TransactionalStateReque return err } - reqVal, err := stateutils.Marshal(req.Value, json.Marshal) + reqVal, err := e.schema.encode(req.Value, ttlInSeconds) if err != nil { return err } @@ -348,19 +360,19 @@ func (e *Etcd) Multi(ctx context.Context, request *state.TransactionalStateReque etag, _ := strconv.ParseInt(*req.ETag, 10, 64) cmp = clientv3.Compare(clientv3.ModRevision(keyWithPath), "=", etag) } - if ttlInSeconds > 0 { - resp, err := e.client.Grant(ctx, ttlInSeconds) + if ttlInSeconds != nil { + resp, err := e.client.Grant(ctx, *ttlInSeconds) if err != nil { return fmt.Errorf("couldn't grant lease %s: %w", keyWithPath, err) } - put := clientv3.OpPut(keyWithPath, string(reqVal), clientv3.WithLease(resp.ID)) + put := clientv3.OpPut(keyWithPath, reqVal, clientv3.WithLease(resp.ID)) if req.HasETag() { ops = append(ops, clientv3.OpTxn([]clientv3.Cmp{cmp}, []clientv3.Op{put}, nil)) } else { ops = append(ops, clientv3.OpTxn(nil, []clientv3.Op{put}, nil)) } } else { - put := clientv3.OpPut(keyWithPath, string(reqVal)) + put := clientv3.OpPut(keyWithPath, reqVal) if req.HasETag() { ops = append(ops, clientv3.OpTxn([]clientv3.Cmp{cmp}, []clientv3.Op{put}, nil)) } else { diff --git a/state/etcd/schema.go b/state/etcd/schema.go new file mode 100644 index 0000000000..6587bd6896 --- /dev/null +++ b/state/etcd/schema.go @@ -0,0 +1,91 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "encoding/json" + "time" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + + pbv2 "github.com/dapr/components-contrib/internal/proto/state/etcd/v2" + "github.com/dapr/components-contrib/state" + "github.com/dapr/components-contrib/state/utils" +) + +// schemaMarshaller is an interface for encoding and decoding values which are +// written and read from ETCD. Different storage schema versions store values +// in different formats or envelopes. +type schemaMarshaller interface { + // encode the value in the correct storage schema. + encode(data any, ttlInSeconds *int64) (string, error) + + // decode the value from the correct storage schema, optionally returning + // metadata extracted from the envelope. + decode(data []byte) ([]byte, map[string]string, error) +} + +type schemaV1 struct{} + +func (schemaV1) encode(data any, _ *int64) (string, error) { + reqVal, err := utils.Marshal(data, json.Marshal) + if err != nil { + return "", err + } + return string(reqVal), nil +} + +func (schemaV1) decode(data []byte) ([]byte, map[string]string, error) { + return data, nil, nil +} + +type schemaV2 struct{} + +func (schemaV2) encode(data any, ttlInSeconds *int64) (string, error) { + dataB, err := utils.JSONStringify(data) + if err != nil { + return "", err + } + + var duration durationpb.Duration + if ttlInSeconds != nil { + duration = durationpb.Duration{Seconds: *ttlInSeconds} + } + + value, err := proto.Marshal(&pbv2.Value{ + Data: dataB, + Ts: timestamppb.New(time.Now().UTC()), + Ttl: &duration, + }) + + return string(value), err +} + +func (schemaV2) decode(data []byte) ([]byte, map[string]string, error) { + var value pbv2.Value + if err := proto.Unmarshal(data, &value); err != nil { + return nil, nil, err + } + + var metadata map[string]string + if value.Ttl != nil { + metadata = map[string]string{ + state.GetRespMetaKeyTTLExpireTime: value.Ts.AsTime().Add(value.Ttl.AsDuration()).Format(time.RFC3339), + } + } + + return value.Data, metadata, nil +} diff --git a/state/mongodb/mongodb.go b/state/mongodb/mongodb.go index 5c3fe7b78c..26a8847f6d 100644 --- a/state/mongodb/mongodb.go +++ b/state/mongodb/mongodb.go @@ -681,3 +681,12 @@ func (m *MongoDB) GetComponentMetadata() map[string]string { metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType) return metadataInfo } + +// Close connection to the database. +func (m *MongoDB) Close(ctx context.Context) (err error) { + if m.client == nil { + return nil + } + + return m.client.Disconnect(ctx) +} diff --git a/state/mongodb/mongodb_test.go b/state/mongodb/mongodb_test.go index 1bc32c9375..dec15e52ff 100644 --- a/state/mongodb/mongodb_test.go +++ b/state/mongodb/mongodb_test.go @@ -191,7 +191,7 @@ func TestGetMongoDBMetadata(t *testing.T) { assert.Equal(t, expected, err.Error()) }) - t.Run("Connectionstring ignores all other connection details", func(t *testing.T) { + t.Run("Connection string ignores all other connection details", func(t *testing.T) { properties := map[string]string{ host: "localhost:27017", databaseName: "TestDB", diff --git a/state/postgresql/metadata.yaml b/state/postgresql/metadata.yaml index 6169696cc9..2d88cf6b6b 100644 --- a/state/postgresql/metadata.yaml +++ b/state/postgresql/metadata.yaml @@ -16,14 +16,35 @@ capabilities: - etag - query - ttl +builtinAuthenticationProfiles: + - name: "azuread" + metadata: + - name: useAzureAD + required: true + type: bool + example: '"true"' + description: | + Must be set to `true` to enable the component to retrieve access tokens from Azure AD. + This authentication method only works with Azure Database for PostgreSQL databases. + - name: connectionString + required: true + sensitive: true + description: | + The connection string for the PostgreSQL database + This must contain the user, which corresponds to the name of the user created inside PostgreSQL that maps to the Azure AD identity; this is often the name of the corresponding principal (e.g. the name of the Azure AD application). This connection string should not contain any password. + example: | + "host=mydb.postgres.database.azure.com user=myapplication port=5432 database=dapr_test sslmode=require" + type: string authenticationProfiles: - title: "Connection string" description: "Authenticate using a Connection String." metadata: - name: connectionString required: true + sensitive: true description: The connection string for the PostgreSQL database - example: "host=localhost user=postgres password=example port=5432 connect_timeout=10 database=dapr_test" + example: | + "host=localhost user=postgres password=example port=5432 connect_timeout=10 database=dapr_test" type: string metadata: - name: timeoutInSeconds @@ -56,6 +77,14 @@ metadata: example: "1800" default: "3600" # 1h type: number + - name: maxConns + required: false + description: | + Maximum number of connections pooled by this component. + Set to 0 or lower to use the default value, which is the greater of 4 or the number of CPUs. + example: "4" + default: "0" + type: number - name: connectionMaxIdleTime required: false description: | diff --git a/state/postgresql/postgresql.go b/state/postgresql/postgresql.go index 638999c017..84563242ff 100644 --- a/state/postgresql/postgresql.go +++ b/state/postgresql/postgresql.go @@ -22,8 +22,9 @@ import ( // NewPostgreSQLStateStore creates a new instance of PostgreSQL state store. func NewPostgreSQLStateStore(logger logger.Logger) state.Store { return postgresql.NewPostgreSQLStateStore(logger, postgresql.Options{ - ETagColumn: "xmin", - MigrateFn: performMigration, + ETagColumn: "xmin", + EnableAzureAD: true, + MigrateFn: performMigration, SetQueryFn: func(req *state.SetRequest, opts postgresql.SetQueryOptions) string { // Sprintf is required for table name because the driver does not substitute parameters for table names. if !req.HasETag() { diff --git a/state/sqlserver/metadata.yaml b/state/sqlserver/metadata.yaml index 713631017e..f5129bbb25 100644 --- a/state/sqlserver/metadata.yaml +++ b/state/sqlserver/metadata.yaml @@ -1,10 +1,11 @@ -# yaml-language-server: $schema=../../../component-metadata-schema.json +# yaml-language-server: $schema=../../component-metadata-schema.json schemaVersion: "v1" type: "state" name: "sqlserver" version: "v1" status: "stable" title: "SQL Server" +description: "Microsoft SQL Server and Azure SQL" urls: - title: "Reference" url: "https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-sqlserver/" @@ -17,7 +18,7 @@ capabilities: authenticationProfiles: - title: "Connection string" description: | - Authenticates using a connection string. + Authenticates using a connection string metadata: - name: connectionString required: true @@ -36,6 +37,7 @@ builtinAuthenticationProfiles: description: | Must be set to `true` to enable the component to retrieve access tokens from Azure AD. This authentication method only works with Azure SQL databases. + example: "true" - name: connectionString required: true sensitive: true diff --git a/state/utils/ttl.go b/state/utils/ttl.go index 077be9fc6b..44e43d8b57 100644 --- a/state/utils/ttl.go +++ b/state/utils/ttl.go @@ -24,8 +24,7 @@ const MetadataTTLKey = "ttlInSeconds" // ParseTTL parses the "ttlInSeconds" metadata property. func ParseTTL(requestMetadata map[string]string) (*int, error) { - val, found := requestMetadata[MetadataTTLKey] - if found && val != "" { + if val := requestMetadata[MetadataTTLKey]; val != "" { parsedVal, err := strconv.ParseInt(val, 10, 0) if err != nil { return nil, fmt.Errorf("incorrect value for metadata '%s': %w", MetadataTTLKey, err) @@ -38,3 +37,18 @@ func ParseTTL(requestMetadata map[string]string) (*int, error) { } return nil, nil } + +// ParseTTL64 parses the "ttlInSeconds" metadata property. +func ParseTTL64(requestMetadata map[string]string) (*int64, error) { + if val := requestMetadata[MetadataTTLKey]; val != "" { + parsedVal, err := strconv.ParseInt(val, 10, 0) + if err != nil { + return nil, fmt.Errorf("incorrect value for metadata '%s': %w", MetadataTTLKey, err) + } + if parsedVal < -1 || parsedVal > math.MaxInt32 { + return nil, fmt.Errorf("incorrect value for metadata '%s': must be -1 or greater", MetadataTTLKey) + } + return &parsedVal, nil + } + return nil, nil +} diff --git a/state/utils/utils.go b/state/utils/utils.go index a80ba034da..01c0504bd0 100644 --- a/state/utils/utils.go +++ b/state/utils/utils.go @@ -13,6 +13,13 @@ limitations under the License. package utils +import ( + "bytes" + "encoding/json" + "strconv" + "strings" +) + func Marshal(val interface{}, marshaler func(interface{}) ([]byte, error)) ([]byte, error) { var err error = nil bt, ok := val.([]byte) @@ -22,3 +29,46 @@ func Marshal(val interface{}, marshaler func(interface{}) ([]byte, error)) ([]by return bt, err } + +func JSONStringify(value any) ([]byte, error) { + switch value := value.(type) { + case []byte: + return value, nil + case int: + return []byte(strconv.FormatInt(int64(value), 10)), nil + case int8: + return []byte(strconv.FormatInt(int64(value), 10)), nil + case int16: + return []byte(strconv.FormatInt(int64(value), 10)), nil + case int32: + return []byte(strconv.FormatInt(int64(value), 10)), nil + case int64: + return []byte(strconv.FormatInt(value, 10)), nil + case uint: + return []byte(strconv.FormatUint(uint64(value), 10)), nil + case uint16: + return []byte(strconv.FormatUint(uint64(value), 10)), nil + case uint32: + return []byte(strconv.FormatUint(uint64(value), 10)), nil + case uint64: + return []byte(strconv.FormatUint(value, 10)), nil + case float32: + return []byte(strconv.FormatFloat(float64(value), 'f', -1, 64)), nil + case float64: + return []byte(strconv.FormatFloat(value, 'f', -1, 64)), nil + case bool: + if value { + return []byte("true"), nil + } + return []byte("false"), nil + case string: + return []byte(`"` + strings.ReplaceAll(value, `"`, `\"`) + `"`), nil + default: + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + err := enc.Encode(value) + // Trim newline. + return bytes.TrimSuffix(buf.Bytes(), []byte{0xa}), err + } +} diff --git a/tests/certification/state/cassandra/cassandra_test.go b/tests/certification/state/cassandra/cassandra_test.go index 0111c099bc..f77573e32f 100644 --- a/tests/certification/state/cassandra/cassandra_test.go +++ b/tests/certification/state/cassandra/cassandra_test.go @@ -15,6 +15,13 @@ package cassandra_test import ( "fmt" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/dapr/components-contrib/state" state_cassandra "github.com/dapr/components-contrib/state/cassandra" "github.com/dapr/components-contrib/tests/certification/embedded" @@ -27,10 +34,6 @@ import ( dapr_testing "github.com/dapr/dapr/pkg/testing" goclient "github.com/dapr/go-sdk/client" "github.com/dapr/kit/logger" - "github.com/stretchr/testify/assert" - "strconv" - "testing" - "time" ) const ( @@ -77,12 +80,14 @@ func TestCassandra(t *testing.T) { item, err := client.GetState(ctx, stateStoreName, certificationTestPrefix+"key1", nil) assert.NoError(t, err) assert.Equal(t, "cassandraCert", string(item.Value)) + assert.NotContains(t, item.Metadata, "ttlExpireTime") errUpdate := client.SaveState(ctx, stateStoreName, certificationTestPrefix+"key1", []byte("cassandraCertUpdate"), nil) assert.NoError(t, errUpdate) item, errUpdatedGet := client.GetState(ctx, stateStoreName, certificationTestPrefix+"key1", nil) assert.NoError(t, errUpdatedGet) assert.Equal(t, "cassandraCertUpdate", string(item.Value)) + assert.NotContains(t, item.Metadata, "ttlExpireTime") // delete state err = client.DeleteState(ctx, stateStoreName, certificationTestPrefix+"key1", nil) @@ -128,6 +133,10 @@ func TestCassandra(t *testing.T) { item, err := client.GetState(ctx, stateStoreName, certificationTestPrefix+"ttl3", nil) assert.NoError(t, err) assert.Equal(t, "cassandraCert3", string(item.Value)) + require.Contains(t, item.Metadata, "ttlExpireTime") + expireTime, err := time.Parse(time.RFC3339, item.Metadata["ttlExpireTime"]) + require.NoError(t, err) + assert.InDelta(t, time.Now().Add(time.Second*5).Unix(), expireTime.Unix(), 3) time.Sleep(5 * time.Second) //entry should be expired now itemAgain, errAgain := client.GetState(ctx, stateStoreName, certificationTestPrefix+"ttl3", nil) diff --git a/tests/config/state/etcd/statestore.yaml b/tests/config/state/etcd/v1/statestore.yaml similarity index 92% rename from tests/config/state/etcd/statestore.yaml rename to tests/config/state/etcd/v1/statestore.yaml index ef8fe37da5..40690fe7b8 100644 --- a/tests/config/state/etcd/statestore.yaml +++ b/tests/config/state/etcd/v1/statestore.yaml @@ -11,4 +11,4 @@ spec: - name: keyPrefixPath value: "dapr" - name: tlsEnable - value: "false" \ No newline at end of file + value: "false" diff --git a/tests/config/state/etcd/v2/statestore.yaml b/tests/config/state/etcd/v2/statestore.yaml new file mode 100644 index 0000000000..1893b56cbd --- /dev/null +++ b/tests/config/state/etcd/v2/statestore.yaml @@ -0,0 +1,14 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: statestore +spec: + type: state.etcd + version: v2 + metadata: + - name: endpoints + value: "localhost:12379" + - name: keyPrefixPath + value: "dapr" + - name: tlsEnable + value: "false" diff --git a/tests/config/state/postgresql/azure/statestore.yml b/tests/config/state/postgresql/azure/statestore.yml new file mode 100644 index 0000000000..1788537e8e --- /dev/null +++ b/tests/config/state/postgresql/azure/statestore.yml @@ -0,0 +1,12 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: statestore +spec: + type: state.postgresql + version: v1 + metadata: + - name: connectionString + value: "${{AzureDBPostgresConnectionString}}" + - name: useAzureAD + value: "true" \ No newline at end of file diff --git a/tests/config/state/postgresql/statestore.yml b/tests/config/state/postgresql/docker/statestore.yml similarity index 100% rename from tests/config/state/postgresql/statestore.yml rename to tests/config/state/postgresql/docker/statestore.yml diff --git a/tests/config/state/tests.yml b/tests/config/state/tests.yml index 50b6cae994..199a540c88 100644 --- a/tests/config/state/tests.yml +++ b/tests/config/state/tests.yml @@ -32,7 +32,12 @@ components: config: # This component requires etags to be hex-encoded numbers badEtag: "FFFF" - - component: postgresql + - component: postgresql.docker + operations: [ "transaction", "etag", "first-write", "query", "ttl" ] + config: + # This component requires etags to be numeric + badEtag: "1" + - component: postgresql.azure operations: [ "transaction", "etag", "first-write", "query", "ttl" ] config: # This component requires etags to be numeric @@ -73,9 +78,11 @@ components: operations: [ "transaction", "etag", "first-write" ] - component: aws.dynamodb.terraform operations: [ "transaction", "etag", "first-write" ] - - component: etcd + - component: etcd.v1 + operations: [ "transaction", "etag", "first-write", "ttl" ] + - component: etcd.v2 operations: [ "transaction", "etag", "first-write", "ttl" ] - component: gcp.firestore.docker operations: [] - component: gcp.firestore.cloud - operations: [] \ No newline at end of file + operations: [] diff --git a/tests/conformance/bindings/bindings.go b/tests/conformance/bindings/bindings.go index 9b22b415d9..a587740dbf 100644 --- a/tests/conformance/bindings/bindings.go +++ b/tests/conformance/bindings/bindings.go @@ -126,17 +126,17 @@ func ConformanceTests(t *testing.T, props map[string]string, inputBinding bindin // Check for an output binding specific operation before init if config.HasOperation("operations") { testLogger.Info("Init output binding ...") - err := outputBinding.Init(context.Background(), bindings.Metadata{ - Base: metadata.Base{Properties: props}, - }) + err := outputBinding.Init(context.Background(), bindings.Metadata{Base: metadata.Base{ + Properties: props, + }}) assert.NoError(t, err, "expected no error setting up output binding") } // Check for an input binding specific operation before init if config.HasOperation("read") { testLogger.Info("Init input binding ...") - err := inputBinding.Init(context.Background(), bindings.Metadata{ - Base: metadata.Base{Properties: props}, - }) + err := inputBinding.Init(context.Background(), bindings.Metadata{Base: metadata.Base{ + Properties: props, + }}) assert.NoError(t, err, "expected no error setting up input binding") } testLogger.Info("Init test done.") diff --git a/tests/conformance/common.go b/tests/conformance/common.go index ff0862c9b6..7ba46cd7e5 100644 --- a/tests/conformance/common.go +++ b/tests/conformance/common.go @@ -119,7 +119,6 @@ const ( eventhubs = "azure.eventhubs" redisv6 = "redis.v6" redisv7 = "redis.v7" - postgres = "postgres" kafka = "kafka" generateUUID = "$((uuid))" generateEd25519PrivateKey = "$((ed25519PrivateKey))" @@ -329,7 +328,6 @@ func (tc *TestConfiguration) loadComponentsAndProperties(t *testing.T, filepath require.Equal(t, 1, len(comps)) // We only expect a single component per file c := comps[0] props, err := ConvertMetadataToProperties(c.Spec.Metadata) - return props, err } @@ -439,7 +437,7 @@ func loadConfigurationStore(tc TestComponent) (configuration.Store, configupdate case redisv7: store = c_redis.NewRedisConfigurationStore(testLogger) updater = cu_redis.NewRedisConfigUpdater(testLogger) - case postgres: + case "postgres": store = c_postgres.NewPostgresConfigurationStore(testLogger) updater = cu_postgres.NewPostgresConfigUpdater(testLogger) default: @@ -544,10 +542,12 @@ func loadStateStore(tc TestComponent) state.Store { case "mongodb": store = s_mongodb.NewMongoDB(testLogger) case "azure.sql": - fallthrough + store = s_sqlserver.New(testLogger) case "sqlserver": store = s_sqlserver.New(testLogger) - case "postgresql": + case "postgresql.docker": + store = s_postgresql.NewPostgreSQLStateStore(testLogger) + case "postgresql.azure": store = s_postgresql.NewPostgreSQLStateStore(testLogger) case "sqlite": store = s_sqlite.NewSQLiteStateStore(testLogger) @@ -577,8 +577,10 @@ func loadStateStore(tc TestComponent) state.Store { store = s_awsdynamodb.NewDynamoDBStateStore(testLogger) case "aws.dynamodb.terraform": store = s_awsdynamodb.NewDynamoDBStateStore(testLogger) - case "etcd": - store = s_etcd.NewEtcdStateStore(testLogger) + case "etcd.v1": + store = s_etcd.NewEtcdStateStoreV1(testLogger) + case "etcd.v2": + store = s_etcd.NewEtcdStateStoreV2(testLogger) case "gcp.firestore.docker": store = s_gcpfirestore.NewFirestoreStateStore(testLogger) case "gcp.firestore.cloud": diff --git a/tests/conformance/configuration/configuration.go b/tests/conformance/configuration/configuration.go index bee02c308c..4dd76bac70 100644 --- a/tests/conformance/configuration/configuration.go +++ b/tests/conformance/configuration/configuration.go @@ -159,7 +159,9 @@ func ConformanceTests(t *testing.T, props map[string]string, store configuration // Initializing store err = store.Init(context.Background(), configuration.Metadata{ - Base: metadata.Base{Properties: props}, + Base: metadata.Base{ + Properties: props, + }, }) require.NoError(t, err) }) diff --git a/tests/conformance/crypto/crypto.go b/tests/conformance/crypto/crypto.go index b57d047aeb..743bcbfe56 100644 --- a/tests/conformance/crypto/crypto.go +++ b/tests/conformance/crypto/crypto.go @@ -114,9 +114,9 @@ func ConformanceTests(t *testing.T, props map[string]string, component contribCr // Init t.Run("Init", func(t *testing.T) { - err := component.Init(context.Background(), contribCrypto.Metadata{ - Base: metadata.Base{Properties: props}, - }) + err := component.Init(context.Background(), contribCrypto.Metadata{Base: metadata.Base{ + Properties: props, + }}) require.NoError(t, err, "expected no error on initializing store") }) diff --git a/tests/conformance/pubsub/pubsub.go b/tests/conformance/pubsub/pubsub.go index c5e877e175..b04629e13c 100644 --- a/tests/conformance/pubsub/pubsub.go +++ b/tests/conformance/pubsub/pubsub.go @@ -105,9 +105,9 @@ func ConformanceTests(t *testing.T, props map[string]string, ps pubsub.PubSub, c // Init t.Run("init", func(t *testing.T) { - err := ps.Init(context.Background(), pubsub.Metadata{ - Base: metadata.Base{Properties: props}, - }) + err := ps.Init(context.Background(), pubsub.Metadata{Base: metadata.Base{ + Properties: props, + }}) assert.NoError(t, err, "expected no error on setting up pubsub") }) diff --git a/tests/conformance/secretstores/secretstores.go b/tests/conformance/secretstores/secretstores.go index f06ee89b50..38c585793d 100644 --- a/tests/conformance/secretstores/secretstores.go +++ b/tests/conformance/secretstores/secretstores.go @@ -49,9 +49,9 @@ func ConformanceTests(t *testing.T, props map[string]string, store secretstores. // Init t.Run("init", func(t *testing.T) { - err := store.Init(context.Background(), secretstores.Metadata{ - Base: metadata.Base{Properties: props}, - }) + err := store.Init(context.Background(), secretstores.Metadata{Base: metadata.Base{ + Properties: props, + }}) assert.NoError(t, err, "expected no error on initializing store") }) diff --git a/tests/conformance/state/state.go b/tests/conformance/state/state.go index 9d8bfc9bbe..bbc2f78d0c 100644 --- a/tests/conformance/state/state.go +++ b/tests/conformance/state/state.go @@ -238,9 +238,9 @@ func ConformanceTests(t *testing.T, props map[string]string, statestore state.St } t.Run("init", func(t *testing.T) { - err := statestore.Init(context.Background(), state.Metadata{ - Base: metadata.Base{Properties: props}, - }) + err := statestore.Init(context.Background(), state.Metadata{Base: metadata.Base{ + Properties: props, + }}) assert.NoError(t, err) }) diff --git a/tests/conformance/workflows/workflows.go b/tests/conformance/workflows/workflows.go index be292acbfb..9e96b9ae2f 100644 --- a/tests/conformance/workflows/workflows.go +++ b/tests/conformance/workflows/workflows.go @@ -51,9 +51,9 @@ func NewTestConfig(component string, operations []string, conf map[string]interf func ConformanceTests(t *testing.T, props map[string]string, workflowItem workflows.Workflow, config TestConfig) { // Test vars t.Run("init", func(t *testing.T) { - err := workflowItem.Init(workflows.Metadata{ - Base: metadata.Base{Properties: props}, - }) + err := workflowItem.Init(workflows.Metadata{Base: metadata.Base{ + Properties: props, + }}) assert.NoError(t, err) })