From cb86a7d6b0a4956315f876bac32bc82c2f14ad08 Mon Sep 17 00:00:00 2001 From: Matt Ellis Date: Tue, 12 Dec 2023 11:15:15 -0800 Subject: [PATCH 1/2] aspire: Improve binding expression evaluator (#3095) When we started the Aspire work, we thought we might only support simple binding expressions with a single reference, e.g.: `"{api.bindings.http.url}"`. Over time, this allow grew to allow literals (e.g.: `"true"`). In Preview 2, the manifests now supports multiple replacements in a single expression, to support things like connection strings where you need multiple parts: `"Host={api.bindings.tcp.host},Port={api.bindings.tcp.port}"` To prepare for this, I've split the string evaluation logic from the value resolution logic and then enhanced the string evaluation logic to support more complex strings, while adding some tests to ensure things behave as expected. --- cli/azd/pkg/apphost/eval.go | 57 ++++++++ cli/azd/pkg/apphost/eval_test.go | 60 ++++++++ cli/azd/pkg/apphost/generate.go | 238 +++++++++++++++---------------- 3 files changed, 236 insertions(+), 119 deletions(-) create mode 100644 cli/azd/pkg/apphost/eval.go create mode 100644 cli/azd/pkg/apphost/eval_test.go diff --git a/cli/azd/pkg/apphost/eval.go b/cli/azd/pkg/apphost/eval.go new file mode 100644 index 00000000000..abc9265653b --- /dev/null +++ b/cli/azd/pkg/apphost/eval.go @@ -0,0 +1,57 @@ +package apphost + +import ( + "fmt" + "strings" +) + +// evalString evaluates a given string expression, using the provided evalExpr function to produce values for expressions +// in the string. It supports strings that contain expressions of the form "{expression}" where "expression" is any string +// that does not contain a '}' character. The evalExpr function is called with the expression (without the enclosing '{' +// and '}' characters) and should return the value to be substituted into the string. If the evalExpr function returns +// an error, evalString will return that error. The '{' and '}' characters can be escaped by doubling them, e.g. +// "{{" and "}}". If a string is malformed (e.g. an unmatched '{' or '}' character), evalString will return an error. +func evalString(src string, evalExpr func(string) (string, error)) (string, error) { + var res strings.Builder + + for i := 0; i < len(src); i++ { + switch src[i] { + case '{': + if i+1 < len(src) && src[i+1] == '{' { + res.WriteByte('{') + i++ + continue + } + + closed := false + for j := i + 1; j < len(src); j++ { + if src[j] == '}' { + v, err := evalExpr(src[i+1 : j]) + if err != nil { + return "", err + } + + res.WriteString(v) + i = j + closed = true + break + } + } + + if !closed { + return "", fmt.Errorf("unclosed '{' at position %d", i) + } + case '}': + if i+1 < len(src) && src[i+1] == '}' { + res.WriteByte('}') + i++ + continue + } + return "", fmt.Errorf("unexpected '}' at position %d", i) + default: + res.WriteByte(src[i]) + } + } + + return res.String(), nil +} diff --git a/cli/azd/pkg/apphost/eval_test.go b/cli/azd/pkg/apphost/eval_test.go new file mode 100644 index 00000000000..6a95a9c5e8c --- /dev/null +++ b/cli/azd/pkg/apphost/eval_test.go @@ -0,0 +1,60 @@ +package apphost + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEvalString(t *testing.T) { + cases := []struct { + name string + src string + want string + }{ + {name: "simple", src: "a string with no replacements", want: "a string with no replacements"}, + {name: "replacement", src: "{this.one.has.a.replacement}", want: "this.one.has.a.replacement"}, + {name: "complex", src: "this {one} has {many} replacements", want: "this one has many replacements"}, + {name: "escape", src: "this {{one}} is {{escaped}}", want: "this {one} is {escaped}"}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + res, err := evalString(c.src, func(s string) (string, error) { + return s, nil + }) + + assert.NoError(t, err) + assert.Equal(t, c.want, res) + }) + } + + errorCases := []struct { + name string + src string + }{ + {name: "unclosed open", src: "this { is unclosed"}, + {name: "unmatched close", src: "this } is unmatched"}, + {name: "unmatched escaped close", src: "this {}} is unmatched"}, + {name: "unmatched escaped open", src: "this {{} is unmatched"}, + } + + for _, c := range errorCases { + t.Run(c.name, func(t *testing.T) { + res, err := evalString(c.src, func(s string) (string, error) { + return s, nil + }) + + assert.Error(t, err) + assert.Equal(t, "", res) + }) + } + + res, err := evalString("{this.one.has.a.replacement}", func(s string) (string, error) { + return "", fmt.Errorf("this should cause evalString to fail") + }) + + assert.Error(t, err) + assert.Equal(t, "", res) +} diff --git a/cli/azd/pkg/apphost/generate.go b/cli/azd/pkg/apphost/generate.go index ff421faf4ff..35e35f09db3 100644 --- a/cli/azd/pkg/apphost/generate.go +++ b/cli/azd/pkg/apphost/generate.go @@ -550,143 +550,143 @@ func buildIngress(bindings map[string]*Binding) (*genContainerAppIngress, error) return nil, nil } -// buildEnvBlock creates the environment map in the template context. It does this by copying the values from the given map, -// evaluating any binding expressions that are present. -func (b *infraGenerator) buildEnvBlock(env map[string]string, manifestCtx *genContainerAppManifestTemplateContext) error { - for k, v := range env { - if !strings.HasPrefix(v, "{") || !strings.HasSuffix(v, "}") { - // We want to ensure that we render these values in the YAML as strings. If `v` was the string "true" - // (without the quotes), we would naturally create a value directive in yaml that looks like this: - // - // - name: OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES - // value: true - // - // And YAML rules would treat the above as the value being a boolean instead of a string, which the container - // app service expects. - // - // JSON marshalling the string value will give us something like `"true"` (with the quotes, and any escaping - // that needs to be done), which is what we want here. - jsonStr, err := json.Marshal(v) - if err != nil { - return fmt.Errorf("marshalling env value: %w", err) - } +// evalBindingRef evaluates a binding reference expression based on the state of the manifest loaded into the generator. +func (b infraGenerator) evalBindingRef(v string) (string, error) { + parts := strings.SplitN(v, ".", 2) + if len(parts) != 2 { + return "", fmt.Errorf("malformed binding expression, expected . but was: %s", v) + } - manifestCtx.Env[k] = string(jsonStr) - continue + resource, prop := parts[0], parts[1] + targetType, ok := b.resourceTypes[resource] + if !ok { + return "", fmt.Errorf("unknown resource referenced in binding expression: %s", resource) + } + + switch { + case targetType == "project.v0" || targetType == "container.v0" || targetType == "dockerfile.v0": + if !strings.HasPrefix(prop, "bindings.") { + return "", fmt.Errorf("unsupported property referenced in binding expression: %s for %s", prop, targetType) } - parts := strings.SplitN(v[1:len(v)-1], ".", 2) + parts := strings.Split(prop[len("bindings."):], ".") + if len(parts) != 2 { - return fmt.Errorf("malformed binding expression, expected . but was: %s", v) + return "", fmt.Errorf("malformed binding expression, expected "+ + "bindings.. but was: %s", v) } - resource, prop := parts[0], parts[1] - targetType, ok := b.resourceTypes[resource] - if !ok { - return fmt.Errorf("unknown resource referenced in binding expression: %s", resource) + var binding *Binding + var has bool + + if targetType == "project.v0" { + binding, has = b.projects[resource].Bindings[parts[0]] + } else if targetType == "container.v0" { + binding, has = b.containers[resource].Bindings[parts[0]] + } else if targetType == "dockerfile.v0" { + binding, has = b.dockerfiles[resource].Bindings[parts[0]] } - switch { - case targetType == "project.v0" || targetType == "container.v0" || targetType == "dockerfile.v0": - if !strings.HasPrefix(prop, "bindings.") { - return fmt.Errorf("unsupported property referenced in binding expression: %s for %s", prop, targetType) - } + if !has { + return "", fmt.Errorf("unknown binding referenced in binding expression: %s for resource %s", parts[0], resource) + } - parts := strings.Split(prop[len("bindings."):], ".") + switch parts[1] { + case "port": + return fmt.Sprintf(`%d`, *binding.ContainerPort), nil + case "url": + var urlFormatString string - if len(parts) != 2 { - return fmt.Errorf("malformed binding expression, expected bindings.. but was: %s", v) + if binding.External { + urlFormatString = "%s://%s.{{ .Env.AZURE_CONTAINER_APPS_ENVIRONMENT_DEFAULT_DOMAIN }}" + } else { + urlFormatString = "%s://%s.internal.{{ .Env.AZURE_CONTAINER_APPS_ENVIRONMENT_DEFAULT_DOMAIN }}" } - var binding *Binding - var has bool + return fmt.Sprintf(urlFormatString, binding.Scheme, resource), nil + default: + return "", fmt.Errorf("malformed binding expression, expected bindings..[port|url] but was: %s", v) + } + case targetType == "postgres.database.v0" || targetType == "redis.v0": + switch prop { + case "connectionString": + return fmt.Sprintf(`{{ connectionString "%s" }}`, resource), nil + default: + return "", errUnsupportedProperty(targetType, prop) + } + case targetType == "azure.servicebus.v0": + switch prop { + case "connectionString": + return fmt.Sprintf("{{ urlHost .Env.SERVICE_BINDING_%s_ENDPOINT }}", scaffold.AlphaSnakeUpper(resource)), nil + default: + return "", errUnsupportedProperty("azure.servicebus.v0", prop) + } + case targetType == "azure.appinsights.v0": + switch prop { + case "connectionString": + return fmt.Sprintf("{{ .Env.SERVICE_BINDING_%s_CONNECTION_STRING }}", scaffold.AlphaSnakeUpper(resource)), nil + default: + return "", errUnsupportedProperty("azure.appinsights.v0", prop) + } + case targetType == "azure.cosmosdb.connection.v0" || + targetType == "postgres.connection.v0" || + targetType == "rabbitmq.connection.v0": - if targetType == "project.v0" { - binding, has = b.projects[resource].Bindings[parts[0]] - } else if targetType == "container.v0" { - binding, has = b.containers[resource].Bindings[parts[0]] - } else if targetType == "dockerfile.v0" { - binding, has = b.dockerfiles[resource].Bindings[parts[0]] - } + switch prop { + case "connectionString": + return b.connectionStrings[resource], nil + default: + return "", errUnsupportedProperty(targetType, prop) + } + case targetType == "azure.keyvault.v0" || + targetType == "azure.storage.blob.v0" || + targetType == "azure.storage.queue.v0" || + targetType == "azure.storage.table.v0": + switch prop { + case "connectionString": + return fmt.Sprintf("{{ .Env.SERVICE_BINDING_%s_ENDPOINT }}", scaffold.AlphaSnakeUpper(resource)), nil + default: + return "", errUnsupportedProperty(targetType, prop) + } + default: + ignore, err := strconv.ParseBool(os.Getenv("AZD_DEBUG_DOTNET_APPHOST_IGNORE_UNSUPPORTED_RESOURCES")) + if err == nil && ignore { + log.Printf("ignoring binding reference to resource of type %s since "+ + "AZD_DEBUG_DOTNET_APPHOST_IGNORE_UNSUPPORTED_RESOURCES is set", targetType) - if !has { - return fmt.Errorf( - "unknown binding referenced in binding expression: %s for resource %s", parts[0], resource) - } + return fmt.Sprintf("!!! expression '%s' to type '%s' unsupported by azd !!!", v, targetType), nil + } - switch parts[1] { - case "port": - manifestCtx.Env[k] = fmt.Sprintf(`"%d"`, *binding.ContainerPort) - case "url": - var urlFormatString string - - if binding.External { - urlFormatString = "%s://%s.{{ .Env.AZURE_CONTAINER_APPS_ENVIRONMENT_DEFAULT_DOMAIN }}" - } else { - urlFormatString = "%s://%s.internal.{{ .Env.AZURE_CONTAINER_APPS_ENVIRONMENT_DEFAULT_DOMAIN }}" - } - - manifestCtx.Env[k] = fmt.Sprintf(urlFormatString, binding.Scheme, resource) - default: - return fmt.Errorf("malformed binding expression, expected bindings..[port|url] but was: %s", v) - } - case targetType == "postgres.database.v0" || targetType == "redis.v0": - switch prop { - case "connectionString": - manifestCtx.Env[k] = fmt.Sprintf(`{{ connectionString "%s" }}`, resource) - default: - return errUnsupportedProperty(targetType, prop) - } - case targetType == "azure.servicebus.v0": - switch prop { - case "connectionString": - manifestCtx.Env[k] = fmt.Sprintf( - "{{ urlHost .Env.SERVICE_BINDING_%s_ENDPOINT }}", scaffold.AlphaSnakeUpper(resource)) - default: - return errUnsupportedProperty("azure.servicebus.v0", prop) - } - case targetType == "azure.appinsights.v0": - switch prop { - case "connectionString": - manifestCtx.Env[k] = fmt.Sprintf( - "{{ .Env.SERVICE_BINDING_%s_CONNECTION_STRING }}", scaffold.AlphaSnakeUpper(resource)) - default: - return errUnsupportedProperty("azure.appinsights.v0", prop) - } - case targetType == "azure.cosmosdb.connection.v0" || - targetType == "postgres.connection.v0" || - targetType == "rabbitmq.connection.v0": - - switch prop { - case "connectionString": - manifestCtx.Env[k] = b.connectionStrings[resource] - default: - return errUnsupportedProperty(targetType, prop) - } - case targetType == "azure.keyvault.v0" || - targetType == "azure.storage.blob.v0" || - targetType == "azure.storage.queue.v0" || - targetType == "azure.storage.table.v0": - switch prop { - case "connectionString": - manifestCtx.Env[k] = fmt.Sprintf( - "{{ .Env.SERVICE_BINDING_%s_ENDPOINT }}", scaffold.AlphaSnakeUpper(resource)) - default: - return errUnsupportedProperty(targetType, prop) - } - default: - ignore, err := strconv.ParseBool(os.Getenv("AZD_DEBUG_DOTNET_APPHOST_IGNORE_UNSUPPORTED_RESOURCES")) - if err == nil && ignore { - log.Printf("ignoring binding reference to resource of type %s since "+ - "AZD_DEBUG_DOTNET_APPHOST_IGNORE_UNSUPPORTED_RESOURCES is set", targetType) + return "", fmt.Errorf("unsupported resource type %s referenced in binding expression", targetType) + } +} - manifestCtx.Env[k] = fmt.Sprintf( - "!!! expression '%s' to type '%s' unsupported by azd !!!", v, targetType) - continue - } +// buildEnvBlock creates the environment map in the template context. It does this by copying the values from the given map, +// evaluating any binding expressions that are present. +func (b *infraGenerator) buildEnvBlock(env map[string]string, manifestCtx *genContainerAppManifestTemplateContext) error { + for k, value := range env { + res, err := evalString(value, b.evalBindingRef) + if err != nil { + return fmt.Errorf("evaluating value for %s: %w", k, err) + } - return fmt.Errorf("unsupported resource type %s referenced in binding expression", targetType) + // We want to ensure that we render these values in the YAML as strings. If `res` was the string "true" + // (without the quotes), we would naturally create a value directive in yaml that looks like this: + // + // - name: OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES + // value: true + // + // And YAML rules would treat the above as the value being a boolean instead of a string, which the container + // app service expects. + // + // JSON marshalling the string value will give us something like `"true"` (with the quotes, and any escaping + // that needs to be done), which is what we want here. + jsonStr, err := json.Marshal(res) + if err != nil { + return fmt.Errorf("marshalling env value: %w", err) } + + manifestCtx.Env[k] = string(jsonStr) } return nil From f385e0bc935d146d10d5bdf1bfcc6f4a409fcd82 Mon Sep 17 00:00:00 2001 From: Wei Lim Date: Tue, 12 Dec 2023 16:20:25 -0800 Subject: [PATCH 2/2] buildpacks: Update dotnet base image to bullseye (#3112) Upgrade dotnet base images to be based on bullseye. Also, bump to 20131107.2 which is the current stable version and supports .NET 8. Fixes https://github.com/Azure/azure-dev/issues/3108 --- cli/azd/pkg/project/framework_service_docker.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cli/azd/pkg/project/framework_service_docker.go b/cli/azd/pkg/project/framework_service_docker.go index d67d8663ed7..33edeeb1560 100644 --- a/cli/azd/pkg/project/framework_service_docker.go +++ b/cli/azd/pkg/project/framework_service_docker.go @@ -289,8 +289,7 @@ func (p *dockerProject) Package( } // Default builder image to produce container images from source -const DefaultBuilderImage = "mcr.microsoft.com/oryx/builder:debian-bullseye-20231004.1" -const DefaultDotNetBuilderImage = "mcr.microsoft.com/oryx/builder:debian-buster-20231004.1" +const DefaultBuilderImage = "mcr.microsoft.com/oryx/builder:debian-bullseye-20231107.2" func (p *dockerProject) packBuild( ctx context.Context, @@ -302,9 +301,6 @@ func (p *dockerProject) packBuild( return nil, err } builder := DefaultBuilderImage - if svc.Language == ServiceLanguageDotNet { - builder = DefaultDotNetBuilderImage - } environ := []string{} userDefinedImage := false