diff --git a/api/v1beta1/azuremanagedcontrolplane_types.go b/api/v1beta1/azuremanagedcontrolplane_types.go index a144886c2af..5da267e0b61 100644 --- a/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/api/v1beta1/azuremanagedcontrolplane_types.go @@ -48,6 +48,20 @@ const ( ManagedControlPlaneOutboundTypeUserDefinedRouting ManagedControlPlaneOutboundType = "userDefinedRouting" ) +// ManagedControlPlaneIdentityType enumerates the values for managed control plane identity type. +type ManagedControlPlaneIdentityType string + +const ( + // ManagedControlPlaneIdentityTypeSystemAssigned Use an implicitly created system-assigned managed identity to manage + // cluster resources. Components in the control plane such as kube-controller-manager will use the + // system-assigned managed identity to manipulate Azure resources. + ManagedControlPlaneIdentityTypeSystemAssigned ManagedControlPlaneIdentityType = ManagedControlPlaneIdentityType(VMIdentitySystemAssigned) + // ManagedControlPlaneIdentityTypeUserAssigned Use a user-assigned identity to manage cluster resources. + // Components in the control plane such as kube-controller-manager will use the specified user-assigned + // managed identity to manipulate Azure resources. + ManagedControlPlaneIdentityTypeUserAssigned ManagedControlPlaneIdentityType = ManagedControlPlaneIdentityType(VMIdentityUserAssigned) +) + // AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane. type AzureManagedControlPlaneSpec struct { // Version defines the desired Kubernetes version. @@ -161,6 +175,15 @@ type AzureManagedControlPlaneSpec struct { // - USGovernmentCloud: "AzureUSGovernmentCloud" // +optional AzureEnvironment string `json:"azureEnvironment,omitempty"` + + // Identity configuration used by the AKS control plane. + // +optional + Identity *Identity `json:"identity,omitempty"` + + // KubeletUserAssignedIdentity is the user-assigned identity for kubelet. + // For authentication with Azure Container Registry. + // +optional + KubeletUserAssignedIdentity string `json:"kubeletUserAssignedIdentity,omitempty"` } // AADProfile - AAD integration managed by AKS. @@ -421,6 +444,21 @@ const ( ExpanderRandom Expander = "random" ) +// Identity represents the Identity configuration for an AKS control plane. +// See also [AKS doc]. +// +// [AKS doc]: https://learn.microsoft.com/en-us/azure/aks/use-managed-identity +type Identity struct { + // Type - The Identity type to use. + // +kubebuilder:validation:Enum=SystemAssigned;UserAssigned + // +optional + Type ManagedControlPlaneIdentityType `json:"type,omitempty"` + + // UserAssignedIdentityResourceID - Identity ARM resource ID when using user-assigned identity. + // +optional + UserAssignedIdentityResourceID string `json:"userAssignedIdentityResourceID,omitempty"` +} + // +kubebuilder:object:root=true // +kubebuilder:resource:path=azuremanagedcontrolplanes,scope=Namespaced,categories=cluster-api,shortName=amcp // +kubebuilder:storageversion diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook.go b/api/v1beta1/azuremanagedcontrolplane_webhook.go index 710b332e668..1f5df9f8833 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -85,6 +85,12 @@ func (mw *azureManagedControlPlaneWebhook) Default(ctx context.Context, obj runt m.Spec.Version = normalizedVersion } + if m.Spec.Identity == nil { + m.Spec.Identity = &Identity{ + Type: ManagedControlPlaneIdentityTypeSystemAssigned, + } + } + if err := m.setDefaultSSHPublicKey(); err != nil { ctrl.Log.WithName("AzureManagedControlPlaneWebHookLogger").Error(err, "setDefaultSSHPublicKey failed") } @@ -266,6 +272,7 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { m.validateAPIServerAccessProfile, m.validateManagedClusterNetwork, m.validateAutoScalerProfile, + m.validateIdentity, } var errs []error @@ -676,3 +683,26 @@ func (m *AzureManagedControlPlane) validateIntegerStringGreaterThanZero(input *s return allErrs } + +// validateIdentity validates an Identity. +func (m *AzureManagedControlPlane) validateIdentity(_ client.Client) error { + var allErrs field.ErrorList + + if m.Spec.Identity != nil { + if m.Spec.Identity.Type == ManagedControlPlaneIdentityTypeUserAssigned { + if m.Spec.Identity.UserAssignedIdentityResourceID == "" { + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "Identity", "UserAssignedIdentityResourceID"), m.Spec.Identity.UserAssignedIdentityResourceID, "cannot be empty if Identity.Type is UserAssigned")) + } + } else { + if m.Spec.Identity.UserAssignedIdentityResourceID != "" { + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "Identity", "UserAssignedIdentityResourceID"), m.Spec.Identity.UserAssignedIdentityResourceID, "should be empty if Identity.Type is SystemAssigned")) + } + } + } + + if len(allErrs) > 0 { + return kerrors.NewAggregate(allErrs.ToAggregate().Errors()) + } + + return nil +} diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 4e46d28e377..3137c0b9c15 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -54,6 +54,7 @@ func TestDefaultingWebhook(t *testing.T) { g.Expect(amcp.Spec.VirtualNetwork.Name).To(Equal("fooName")) g.Expect(amcp.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooName")) g.Expect(amcp.Spec.SKU.Tier).To(Equal(FreeManagedControlPlaneTier)) + g.Expect(amcp.Spec.Identity.Type).To(Equal(ManagedControlPlaneIdentityTypeSystemAssigned)) t.Logf("Testing amcp defaulting webhook with baseline") netPlug := "kubenet" @@ -548,6 +549,56 @@ func TestValidatingWebhook(t *testing.T) { }, expectErr: false, }, + { + name: "Testing valid Identity: SystemAssigned", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.24.1", + Identity: &Identity{ + Type: ManagedControlPlaneIdentityTypeSystemAssigned, + }, + }, + }, + expectErr: false, + }, + { + name: "Testing valid Identity: UserAssigned", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.24.1", + Identity: &Identity{ + Type: ManagedControlPlaneIdentityTypeUserAssigned, + UserAssignedIdentityResourceID: "/resource/id", + }, + }, + }, + expectErr: false, + }, + { + name: "Testing invalid Identity: SystemAssigned with UserAssigned values", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.24.1", + Identity: &Identity{ + Type: ManagedControlPlaneIdentityTypeSystemAssigned, + UserAssignedIdentityResourceID: "/resource/id", + }, + }, + }, + expectErr: true, + }, + { + name: "Testing invalid Identity: UserAssigned with missing properties", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.24.1", + Identity: &Identity{ + Type: ManagedControlPlaneIdentityTypeUserAssigned, + }, + }, + }, + expectErr: true, + }, } for _, tt := range tests { diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index b7437e6aefb..4ed6bbeefaf 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1222,6 +1222,11 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(AutoScalerProfile) (*in).DeepCopyInto(*out) } + if in.Identity != nil { + in, out := &in.Identity, &out.Identity + *out = new(Identity) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneSpec. @@ -1847,6 +1852,21 @@ func (in *IPTag) DeepCopy() *IPTag { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Identity) DeepCopyInto(out *Identity) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Identity. +func (in *Identity) DeepCopy() *Identity { + if in == nil { + return nil + } + out := new(Identity) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Image) DeepCopyInto(out *Image) { *out = *in diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 66678df642c..5a4ad4eb956 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -473,8 +473,10 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() azure.ResourceSpecGetter s.ControlPlane.Spec.VirtualNetwork.Name, s.ControlPlane.Spec.VirtualNetwork.Subnet.Name, ), - GetAllAgentPools: s.GetAllAgentPoolSpecs, - OutboundType: s.ControlPlane.Spec.OutboundType, + GetAllAgentPools: s.GetAllAgentPoolSpecs, + OutboundType: s.ControlPlane.Spec.OutboundType, + Identity: s.ControlPlane.Spec.Identity, + KubeletUserAssignedIdentity: s.ControlPlane.Spec.KubeletUserAssignedIdentity, } if s.ControlPlane.Spec.NetworkPlugin != nil { diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index 40fc101eea7..b263c960a09 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -106,6 +106,12 @@ type ManagedClusterSpec struct { // AutoScalerProfile is the parameters to be applied to the cluster-autoscaler when enabled. AutoScalerProfile *AutoScalerProfile + + // Identity is the AKS control plane Identity configuration + Identity *infrav1.Identity + + // KubeletUserAssignedIdentity is the user-assigned identity for kubelet to authenticate to ACR. + KubeletUserAssignedIdentity string } // AADProfile is Azure Active Directory configuration to integrate with AKS, for aad authentication. @@ -361,23 +367,7 @@ func (s *ManagedClusterSpec) Parameters(ctx context.Context, existing interface{ } if s.LoadBalancerProfile != nil { - managedCluster.NetworkProfile.LoadBalancerProfile = &containerservice.ManagedClusterLoadBalancerProfile{ - AllocatedOutboundPorts: s.LoadBalancerProfile.AllocatedOutboundPorts, - IdleTimeoutInMinutes: s.LoadBalancerProfile.IdleTimeoutInMinutes, - } - if s.LoadBalancerProfile.ManagedOutboundIPs != nil { - managedCluster.NetworkProfile.LoadBalancerProfile.ManagedOutboundIPs = &containerservice.ManagedClusterLoadBalancerProfileManagedOutboundIPs{Count: s.LoadBalancerProfile.ManagedOutboundIPs} - } - if len(s.LoadBalancerProfile.OutboundIPPrefixes) > 0 { - managedCluster.NetworkProfile.LoadBalancerProfile.OutboundIPPrefixes = &containerservice.ManagedClusterLoadBalancerProfileOutboundIPPrefixes{ - PublicIPPrefixes: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPPrefixes), - } - } - if len(s.LoadBalancerProfile.OutboundIPs) > 0 { - managedCluster.NetworkProfile.LoadBalancerProfile.OutboundIPs = &containerservice.ManagedClusterLoadBalancerProfileOutboundIPs{ - PublicIPs: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPs), - } - } + managedCluster.NetworkProfile.LoadBalancerProfile = s.GetLoadBalancerProfile() } if s.APIServerAccessProfile != nil { @@ -398,6 +388,21 @@ func (s *ManagedClusterSpec) Parameters(ctx context.Context, existing interface{ managedCluster.AutoScalerProfile = buildAutoScalerProfile(s.AutoScalerProfile) + if s.Identity != nil { + managedCluster.Identity, err = getIdentity(s.Identity) + if err != nil { + return nil, errors.Wrapf(err, "Identity is not valid: %s", err) + } + } + + if s.KubeletUserAssignedIdentity != "" { + managedCluster.ManagedClusterProperties.IdentityProfile = map[string]*containerservice.UserAssignedIdentity{ + "kubeletidentity": { + ResourceID: pointer.String(s.KubeletUserAssignedIdentity), + }, + } + } + if existing != nil { existingMC, ok := existing.(containerservice.ManagedCluster) if !ok { @@ -453,6 +458,29 @@ func (s *ManagedClusterSpec) Parameters(ctx context.Context, existing interface{ return managedCluster, nil } +// GetLoadBalancerProfile returns a containerservice.ManagedClusterLoadBalancerProfile from the +// information present in ManagedClusterSpec.LoadBalancerProfile. +func (s *ManagedClusterSpec) GetLoadBalancerProfile() (loadBalancerProfile *containerservice.ManagedClusterLoadBalancerProfile) { + loadBalancerProfile = &containerservice.ManagedClusterLoadBalancerProfile{ + AllocatedOutboundPorts: s.LoadBalancerProfile.AllocatedOutboundPorts, + IdleTimeoutInMinutes: s.LoadBalancerProfile.IdleTimeoutInMinutes, + } + if s.LoadBalancerProfile.ManagedOutboundIPs != nil { + loadBalancerProfile.ManagedOutboundIPs = &containerservice.ManagedClusterLoadBalancerProfileManagedOutboundIPs{Count: s.LoadBalancerProfile.ManagedOutboundIPs} + } + if len(s.LoadBalancerProfile.OutboundIPPrefixes) > 0 { + loadBalancerProfile.OutboundIPPrefixes = &containerservice.ManagedClusterLoadBalancerProfileOutboundIPPrefixes{ + PublicIPPrefixes: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPPrefixes), + } + } + if len(s.LoadBalancerProfile.OutboundIPs) > 0 { + loadBalancerProfile.OutboundIPs = &containerservice.ManagedClusterLoadBalancerProfileOutboundIPs{ + PublicIPs: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPs), + } + } + return +} + func convertToResourceReferences(resources []string) *[]containerservice.ResourceReference { resourceReferences := make([]containerservice.ResourceReference, len(resources)) for i := range resources { @@ -560,6 +588,22 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus } } + if managedCluster.IdentityProfile != nil { + propertiesNormalized.IdentityProfile = map[string]*containerservice.UserAssignedIdentity{ + "kubeletidentity": { + ResourceID: managedCluster.IdentityProfile["kubeletidentity"].ResourceID, + }, + } + } + + if existingMC.IdentityProfile != nil { + existingMCPropertiesNormalized.IdentityProfile = map[string]*containerservice.UserAssignedIdentity{ + "kubeletidentity": { + ResourceID: existingMC.IdentityProfile["kubeletidentity"].ResourceID, + }, + } + } + // Once the AKS autoscaler has been updated it will always return values so we need to // respect those values even though the settings are now not being explicitly set by CAPZ. if existingMC.AutoScalerProfile != nil && managedCluster.AutoScalerProfile == nil { @@ -574,6 +618,20 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus ManagedClusterProperties: existingMCPropertiesNormalized, } + if managedCluster.Identity != nil { + clusterNormalized.Identity = &containerservice.ManagedClusterIdentity{ + Type: managedCluster.Identity.Type, + UserAssignedIdentities: managedCluster.Identity.UserAssignedIdentities, + } + } + + if existingMC.Identity != nil { + existingMCClusterNormalized.Identity = &containerservice.ManagedClusterIdentity{ + Type: existingMC.Identity.Type, + UserAssignedIdentities: existingMC.Identity.UserAssignedIdentities, + } + } + if managedCluster.Sku != nil { clusterNormalized.Sku = managedCluster.Sku } @@ -584,3 +642,23 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus diff := cmp.Diff(clusterNormalized, existingMCClusterNormalized) return diff } + +func getIdentity(identity *infrav1.Identity) (managedClusterIdentity *containerservice.ManagedClusterIdentity, err error) { + if identity.Type == "" { + return + } + + managedClusterIdentity = &containerservice.ManagedClusterIdentity{ + Type: containerservice.ResourceIdentityType(identity.Type), + } + if managedClusterIdentity.Type == containerservice.ResourceIdentityTypeUserAssigned { + if identity.UserAssignedIdentityResourceID == "" { + err = errors.Errorf("Identity is set to \"UserAssigned\" but no UserAssignedIdentityResourceID is present") + return + } + managedClusterIdentity.UserAssignedIdentities = map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{ + identity.UserAssignedIdentityResourceID: {}, + } + } + return +} diff --git a/azure/services/managedclusters/spec_test.go b/azure/services/managedclusters/spec_test.go index c0307ff79ab..86aef8a0a18 100644 --- a/azure/services/managedclusters/spec_test.go +++ b/azure/services/managedclusters/spec_test.go @@ -23,6 +23,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2022-03-01/containerservice" "github.com/google/go-cmp/cmp" . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" "k8s.io/utils/pointer" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" @@ -131,10 +132,18 @@ func TestParameters(t *testing.T) { }, Version: "v1.22.99", LoadBalancerSKU: "Standard", + Identity: &infrav1.Identity{ + Type: infrav1.ManagedControlPlaneIdentityTypeUserAssigned, + UserAssignedIdentityResourceID: "/resource/ID", + }, + KubeletUserAssignedIdentity: "/resource/ID", }, expect: func(g *WithT, result interface{}) { g.Expect(result).To(BeAssignableToTypeOf(containerservice.ManagedCluster{})) g.Expect(result.(containerservice.ManagedCluster).KubernetesVersion).To(Equal(pointer.String("v1.22.99"))) + g.Expect(result.(containerservice.ManagedCluster).Identity.Type).To(Equal(containerservice.ResourceIdentityType("UserAssigned"))) + g.Expect(result.(containerservice.ManagedCluster).Identity.UserAssignedIdentities).To(Equal(map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{"/resource/ID": {}})) + g.Expect(result.(containerservice.ManagedCluster).IdentityProfile).To(Equal(map[string]*containerservice.UserAssignedIdentity{"kubeletidentity": {ResourceID: pointer.String("/resource/ID")}})) }, }, { @@ -181,6 +190,7 @@ func TestParameters(t *testing.T) { for _, tc := range testcases { tc := tc t.Run(tc.name, func(t *testing.T) { + format.MaxLength = 10000 g := NewWithT(t) t.Parallel() @@ -196,6 +206,55 @@ func TestParameters(t *testing.T) { } } +func TestGetIdentity(t *testing.T) { + testcases := []struct { + name string + identity *infrav1.Identity + expectedType containerservice.ResourceIdentityType + }{ + { + name: "default", + identity: &infrav1.Identity{}, + }, + { + name: "user-assigned identity", + identity: &infrav1.Identity{ + Type: infrav1.ManagedControlPlaneIdentityTypeUserAssigned, + UserAssignedIdentityResourceID: "/subscriptions/fae7cc14-bfba-4471-9435-f945b42a16dd/resourcegroups/my-identities/providers/Microsoft.ManagedIdentity/userAssignedIdentities/my-cluster-user-identity", + }, + expectedType: containerservice.ResourceIdentityTypeUserAssigned, + }, + { + name: "system-assigned identity", + identity: &infrav1.Identity{ + Type: infrav1.ManagedControlPlaneIdentityTypeSystemAssigned, + }, + expectedType: containerservice.ResourceIdentityTypeSystemAssigned, + }, + } + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + result, err := getIdentity(tc.identity) + g.Expect(err).To(BeNil()) + if tc.identity.Type != "" { + g.Expect(result.Type).To(Equal(tc.expectedType)) + if tc.identity.Type == infrav1.ManagedControlPlaneIdentityTypeUserAssigned { + g.Expect(result.UserAssignedIdentities).To(Not(BeEmpty())) + g.Expect(*result.UserAssignedIdentities[tc.identity.UserAssignedIdentityResourceID]).To(Equal(containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{})) + } else { + g.Expect(result.UserAssignedIdentities).To(BeEmpty()) + } + } else { + g.Expect(result).To(BeNil()) + } + }) + } +} + func getExistingClusterWithAPIServerAccessProfile() containerservice.ManagedCluster { mc := getExistingCluster() mc.APIServerAccessProfile = &containerservice.ManagedClusterAPIServerAccessProfile{ diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index c26d17abbf9..2e6b1085cf9 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -244,6 +244,20 @@ spec: DNS service. It must be within the Kubernetes service address range specified in serviceCidr. Immutable. type: string + identity: + description: Identity configuration used by the AKS control plane. + properties: + type: + description: Type - The Identity type to use. + enum: + - SystemAssigned + - UserAssigned + type: string + userAssignedIdentityResourceID: + description: UserAssignedIdentityResourceID - Identity ARM resource + ID when using user-assigned identity. + type: string + type: object identityRef: description: IdentityRef is a reference to a AzureClusterIdentity to be used when reconciling this cluster @@ -282,6 +296,10 @@ spec: type: string type: object x-kubernetes-map-type: atomic + kubeletUserAssignedIdentity: + description: KubeletUserAssignedIdentity is the user-assigned identity + for kubelet. For authentication with Azure Container Registry. + type: string loadBalancerProfile: description: LoadBalancerProfile is the profile of the cluster load balancer.