Merge pull request #3425 from jackfrancis/aks-user-assigned-identity

feat: AKS user-assigned identity for control plane and kubelet

api/v1beta1/azuremanagedcontrolplane_types.go

@@ -48,6 +48,20 @@ const (
 	ManagedControlPlaneOutboundTypeUserDefinedRouting ManagedControlPlaneOutboundType = "userDefinedRouting"
 )
 
+// ManagedControlPlaneIdentityType enumerates the values for managed control plane identity type.
+type ManagedControlPlaneIdentityType string
+
+const (
+	// ManagedControlPlaneIdentityTypeSystemAssigned Use an implicitly created system-assigned managed identity to manage
+	// cluster resources. Components in the control plane such as kube-controller-manager will use the
+	// system-assigned managed identity to manipulate Azure resources.
+	ManagedControlPlaneIdentityTypeSystemAssigned ManagedControlPlaneIdentityType = ManagedControlPlaneIdentityType(VMIdentitySystemAssigned)
+	// ManagedControlPlaneIdentityTypeUserAssigned Use a user-assigned identity to manage cluster resources.
+	// Components in the control plane such as kube-controller-manager will use the specified user-assigned
+	// managed identity to manipulate Azure resources.
+	ManagedControlPlaneIdentityTypeUserAssigned ManagedControlPlaneIdentityType = ManagedControlPlaneIdentityType(VMIdentityUserAssigned)
+)
+
 // AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane.
 type AzureManagedControlPlaneSpec struct {
 	// Version defines the desired Kubernetes version.
@@ -161,6 +175,15 @@ type AzureManagedControlPlaneSpec struct {
 	// - USGovernmentCloud: "AzureUSGovernmentCloud"
 	// +optional
 	AzureEnvironment string `json:"azureEnvironment,omitempty"`
+
+	// Identity configuration used by the AKS control plane.
+	// +optional
+	Identity *Identity `json:"identity,omitempty"`
+
+	// KubeletUserAssignedIdentity is the user-assigned identity for kubelet.
+	// For authentication with Azure Container Registry.
+	// +optional
+	KubeletUserAssignedIdentity string `json:"kubeletUserAssignedIdentity,omitempty"`
 }
 
 // AADProfile - AAD integration managed by AKS.
@@ -421,6 +444,21 @@ const (
 	ExpanderRandom Expander = "random"
 )
 
+// Identity represents the Identity configuration for an AKS control plane.
+// See also [AKS doc].
+//
+// [AKS doc]: https://learn.microsoft.com/en-us/azure/aks/use-managed-identity
+type Identity struct {
+	// Type - The Identity type to use.
+	// +kubebuilder:validation:Enum=SystemAssigned;UserAssigned
+	// +optional
+	Type ManagedControlPlaneIdentityType `json:"type,omitempty"`
+
+	// UserAssignedIdentityResourceID - Identity ARM resource ID when using user-assigned identity.
+	// +optional
+	UserAssignedIdentityResourceID string `json:"userAssignedIdentityResourceID,omitempty"`
+}
+
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:path=azuremanagedcontrolplanes,scope=Namespaced,categories=cluster-api,shortName=amcp
 // +kubebuilder:storageversion

api/v1beta1/azuremanagedcontrolplane_webhook.go

@@ -85,6 +85,12 @@ func (mw *azureManagedControlPlaneWebhook) Default(ctx context.Context, obj runt
 		m.Spec.Version = normalizedVersion
 	}
 
+	if m.Spec.Identity == nil {
+		m.Spec.Identity = &Identity{
+			Type: ManagedControlPlaneIdentityTypeSystemAssigned,
+		}
+	}
+
 	if err := m.setDefaultSSHPublicKey(); err != nil {
 		ctrl.Log.WithName("AzureManagedControlPlaneWebHookLogger").Error(err, "setDefaultSSHPublicKey failed")
 	}
@@ -266,6 +272,7 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error {
 		m.validateAPIServerAccessProfile,
 		m.validateManagedClusterNetwork,
 		m.validateAutoScalerProfile,
+		m.validateIdentity,
 	}
 
 	var errs []error
@@ -676,3 +683,26 @@ func (m *AzureManagedControlPlane) validateIntegerStringGreaterThanZero(input *s
 
 	return allErrs
 }
+
+// validateIdentity validates an Identity.
+func (m *AzureManagedControlPlane) validateIdentity(_ client.Client) error {
+	var allErrs field.ErrorList
+
+	if m.Spec.Identity != nil {
+		if m.Spec.Identity.Type == ManagedControlPlaneIdentityTypeUserAssigned {
+			if m.Spec.Identity.UserAssignedIdentityResourceID == "" {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "Identity", "UserAssignedIdentityResourceID"), m.Spec.Identity.UserAssignedIdentityResourceID, "cannot be empty if Identity.Type is UserAssigned"))
+			}
+		} else {
+			if m.Spec.Identity.UserAssignedIdentityResourceID != "" {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "Identity", "UserAssignedIdentityResourceID"), m.Spec.Identity.UserAssignedIdentityResourceID, "should be empty if Identity.Type is SystemAssigned"))
+			}
+		}
+	}
+
+	if len(allErrs) > 0 {
+		return kerrors.NewAggregate(allErrs.ToAggregate().Errors())
+	}
+
+	return nil
+}

api/v1beta1/azuremanagedcontrolplane_webhook_test.go

@@ -54,6 +54,7 @@ func TestDefaultingWebhook(t *testing.T) {
 	g.Expect(amcp.Spec.VirtualNetwork.Name).To(Equal("fooName"))
 	g.Expect(amcp.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooName"))
 	g.Expect(amcp.Spec.SKU.Tier).To(Equal(FreeManagedControlPlaneTier))
+	g.Expect(amcp.Spec.Identity.Type).To(Equal(ManagedControlPlaneIdentityTypeSystemAssigned))
 
 	t.Logf("Testing amcp defaulting webhook with baseline")
 	netPlug := "kubenet"
@@ -548,6 +549,56 @@ func TestValidatingWebhook(t *testing.T) {
 			},
 			expectErr: false,
 		},
+		{
+			name: "Testing valid Identity: SystemAssigned",
+			amcp: AzureManagedControlPlane{
+				Spec: AzureManagedControlPlaneSpec{
+					Version: "v1.24.1",
+					Identity: &Identity{
+						Type: ManagedControlPlaneIdentityTypeSystemAssigned,
+					},
+				},
+			},
+			expectErr: false,
+		},
+		{
+			name: "Testing valid Identity: UserAssigned",
+			amcp: AzureManagedControlPlane{
+				Spec: AzureManagedControlPlaneSpec{
+					Version: "v1.24.1",
+					Identity: &Identity{
+						Type:                           ManagedControlPlaneIdentityTypeUserAssigned,
+						UserAssignedIdentityResourceID: "/resource/id",
+					},
+				},
+			},
+			expectErr: false,
+		},
+		{
+			name: "Testing invalid Identity: SystemAssigned with UserAssigned values",
+			amcp: AzureManagedControlPlane{
+				Spec: AzureManagedControlPlaneSpec{
+					Version: "v1.24.1",
+					Identity: &Identity{
+						Type:                           ManagedControlPlaneIdentityTypeSystemAssigned,
+						UserAssignedIdentityResourceID: "/resource/id",
+					},
+				},
+			},
+			expectErr: true,
+		},
+		{
+			name: "Testing invalid Identity: UserAssigned with missing properties",
+			amcp: AzureManagedControlPlane{
+				Spec: AzureManagedControlPlaneSpec{
+					Version: "v1.24.1",
+					Identity: &Identity{
+						Type: ManagedControlPlaneIdentityTypeUserAssigned,
+					},
+				},
+			},
+			expectErr: true,
+		},
 	}
 
 	for _, tt := range tests {

azure/scope/managedcontrolplane.go

@@ -473,8 +473,10 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() azure.ResourceSpecGetter
 			s.ControlPlane.Spec.VirtualNetwork.Name,
 			s.ControlPlane.Spec.VirtualNetwork.Subnet.Name,
 		),
-		GetAllAgentPools: s.GetAllAgentPoolSpecs,
-		OutboundType:     s.ControlPlane.Spec.OutboundType,
+		GetAllAgentPools:            s.GetAllAgentPoolSpecs,
+		OutboundType:                s.ControlPlane.Spec.OutboundType,
+		Identity:                    s.ControlPlane.Spec.Identity,
+		KubeletUserAssignedIdentity: s.ControlPlane.Spec.KubeletUserAssignedIdentity,
 	}
 
 	if s.ControlPlane.Spec.NetworkPlugin != nil {

azure/services/managedclusters/spec.go

@@ -106,6 +106,12 @@ type ManagedClusterSpec struct {
 
 	// AutoScalerProfile is the parameters to be applied to the cluster-autoscaler when enabled.
 	AutoScalerProfile *AutoScalerProfile
+
+	// Identity is the AKS control plane Identity configuration
+	Identity *infrav1.Identity
+
+	// KubeletUserAssignedIdentity is the user-assigned identity for kubelet to authenticate to ACR.
+	KubeletUserAssignedIdentity string
 }
 
 // AADProfile is Azure Active Directory configuration to integrate with AKS, for aad authentication.
@@ -361,23 +367,7 @@ func (s *ManagedClusterSpec) Parameters(ctx context.Context, existing interface{
 	}
 
 	if s.LoadBalancerProfile != nil {
-		managedCluster.NetworkProfile.LoadBalancerProfile = &containerservice.ManagedClusterLoadBalancerProfile{
-			AllocatedOutboundPorts: s.LoadBalancerProfile.AllocatedOutboundPorts,
-			IdleTimeoutInMinutes:   s.LoadBalancerProfile.IdleTimeoutInMinutes,
-		}
-		if s.LoadBalancerProfile.ManagedOutboundIPs != nil {
-			managedCluster.NetworkProfile.LoadBalancerProfile.ManagedOutboundIPs = &containerservice.ManagedClusterLoadBalancerProfileManagedOutboundIPs{Count: s.LoadBalancerProfile.ManagedOutboundIPs}
-		}
-		if len(s.LoadBalancerProfile.OutboundIPPrefixes) > 0 {
-			managedCluster.NetworkProfile.LoadBalancerProfile.OutboundIPPrefixes = &containerservice.ManagedClusterLoadBalancerProfileOutboundIPPrefixes{
-				PublicIPPrefixes: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPPrefixes),
-			}
-		}
-		if len(s.LoadBalancerProfile.OutboundIPs) > 0 {
-			managedCluster.NetworkProfile.LoadBalancerProfile.OutboundIPs = &containerservice.ManagedClusterLoadBalancerProfileOutboundIPs{
-				PublicIPs: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPs),
-			}
-		}
+		managedCluster.NetworkProfile.LoadBalancerProfile = s.GetLoadBalancerProfile()
 	}
 
 	if s.APIServerAccessProfile != nil {
@@ -398,6 +388,21 @@ func (s *ManagedClusterSpec) Parameters(ctx context.Context, existing interface{
 
 	managedCluster.AutoScalerProfile = buildAutoScalerProfile(s.AutoScalerProfile)
 
+	if s.Identity != nil {
+		managedCluster.Identity, err = getIdentity(s.Identity)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Identity is not valid: %s", err)
+		}
+	}
+
+	if s.KubeletUserAssignedIdentity != "" {
+		managedCluster.ManagedClusterProperties.IdentityProfile = map[string]*containerservice.UserAssignedIdentity{
+			"kubeletidentity": {
+				ResourceID: pointer.String(s.KubeletUserAssignedIdentity),
+			},
+		}
+	}
+
 	if existing != nil {
 		existingMC, ok := existing.(containerservice.ManagedCluster)
 		if !ok {
@@ -453,6 +458,29 @@ func (s *ManagedClusterSpec) Parameters(ctx context.Context, existing interface{
 	return managedCluster, nil
 }
 
+// GetLoadBalancerProfile returns a containerservice.ManagedClusterLoadBalancerProfile from the
+// information present in ManagedClusterSpec.LoadBalancerProfile.
+func (s *ManagedClusterSpec) GetLoadBalancerProfile() (loadBalancerProfile *containerservice.ManagedClusterLoadBalancerProfile) {
+	loadBalancerProfile = &containerservice.ManagedClusterLoadBalancerProfile{
+		AllocatedOutboundPorts: s.LoadBalancerProfile.AllocatedOutboundPorts,
+		IdleTimeoutInMinutes:   s.LoadBalancerProfile.IdleTimeoutInMinutes,
+	}
+	if s.LoadBalancerProfile.ManagedOutboundIPs != nil {
+		loadBalancerProfile.ManagedOutboundIPs = &containerservice.ManagedClusterLoadBalancerProfileManagedOutboundIPs{Count: s.LoadBalancerProfile.ManagedOutboundIPs}
+	}
+	if len(s.LoadBalancerProfile.OutboundIPPrefixes) > 0 {
+		loadBalancerProfile.OutboundIPPrefixes = &containerservice.ManagedClusterLoadBalancerProfileOutboundIPPrefixes{
+			PublicIPPrefixes: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPPrefixes),
+		}
+	}
+	if len(s.LoadBalancerProfile.OutboundIPs) > 0 {
+		loadBalancerProfile.OutboundIPs = &containerservice.ManagedClusterLoadBalancerProfileOutboundIPs{
+			PublicIPs: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPs),
+		}
+	}
+	return
+}
+
 func convertToResourceReferences(resources []string) *[]containerservice.ResourceReference {
 	resourceReferences := make([]containerservice.ResourceReference, len(resources))
 	for i := range resources {
@@ -560,6 +588,22 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus
 		}
 	}
 
+	if managedCluster.IdentityProfile != nil {
+		propertiesNormalized.IdentityProfile = map[string]*containerservice.UserAssignedIdentity{
+			"kubeletidentity": {
+				ResourceID: managedCluster.IdentityProfile["kubeletidentity"].ResourceID,
+			},
+		}
+	}
+
+	if existingMC.IdentityProfile != nil {
+		existingMCPropertiesNormalized.IdentityProfile = map[string]*containerservice.UserAssignedIdentity{
+			"kubeletidentity": {
+				ResourceID: existingMC.IdentityProfile["kubeletidentity"].ResourceID,
+			},
+		}
+	}
+
 	// Once the AKS autoscaler has been updated it will always return values so we need to
 	// respect those values even though the settings are now not being explicitly set by CAPZ.
 	if existingMC.AutoScalerProfile != nil && managedCluster.AutoScalerProfile == nil {
@@ -574,6 +618,20 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus
 		ManagedClusterProperties: existingMCPropertiesNormalized,
 	}
 
+	if managedCluster.Identity != nil {
+		clusterNormalized.Identity = &containerservice.ManagedClusterIdentity{
+			Type:                   managedCluster.Identity.Type,
+			UserAssignedIdentities: managedCluster.Identity.UserAssignedIdentities,
+		}
+	}
+
+	if existingMC.Identity != nil {
+		existingMCClusterNormalized.Identity = &containerservice.ManagedClusterIdentity{
+			Type:                   existingMC.Identity.Type,
+			UserAssignedIdentities: existingMC.Identity.UserAssignedIdentities,
+		}
+	}
+
 	if managedCluster.Sku != nil {
 		clusterNormalized.Sku = managedCluster.Sku
 	}
@@ -584,3 +642,23 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus
 	diff := cmp.Diff(clusterNormalized, existingMCClusterNormalized)
 	return diff
 }
+
+func getIdentity(identity *infrav1.Identity) (managedClusterIdentity *containerservice.ManagedClusterIdentity, err error) {
+	if identity.Type == "" {
+		return
+	}
+
+	managedClusterIdentity = &containerservice.ManagedClusterIdentity{
+		Type: containerservice.ResourceIdentityType(identity.Type),
+	}
+	if managedClusterIdentity.Type == containerservice.ResourceIdentityTypeUserAssigned {
+		if identity.UserAssignedIdentityResourceID == "" {
+			err = errors.Errorf("Identity is set to \"UserAssigned\" but no UserAssignedIdentityResourceID is present")
+			return
+		}
+		managedClusterIdentity.UserAssignedIdentities = map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{
+			identity.UserAssignedIdentityResourceID: {},
+		}
+	}
+	return
+}