Add control plane failure domains feature for Nutanix provider (#8192)

* Add failure domains support for Nutanix Provider

 - change Nutanix Datacenter CRD
 - change templates
 - generate manifests
 - add unittest

* Fix lint error

* Regenerate deepcopy files

* Fix PR comments

 - add validation
 - fix template
 - add unittest for validation

* Add validations and unit tests

* Allow updating failure domains for existing clusters

config/crd/bases/anywhere.eks.amazonaws.com_nutanixdatacenterconfigs.yaml

@@ -56,6 +56,70 @@ spec:
               endpoint:
                 description: Endpoint is the Endpoint of Nutanix Prism Central
                 type: string
+              failureDomains:
+                description: FailureDomains is the optional list of failure domains
+                  for the Nutanix Datacenter.
+                items:
+                  description: NutanixDatacenterFailureDomain defines the failure
+                    domain for the Nutanix Datacenter.
+                  properties:
+                    cluster:
+                      description: Cluster is the Prism Element cluster name or uuid
+                        that is connected to the Prism Central.
+                      properties:
+                        name:
+                          description: name is the resource name in the PC
+                          type: string
+                        type:
+                          description: Type is the identifier type to use for this
+                            resource.
+                          enum:
+                          - uuid
+                          - name
+                          type: string
+                        uuid:
+                          description: uuid is the UUID of the resource in the PC.
+                          type: string
+                      required:
+                      - type
+                      type: object
+                    name:
+                      description: Name is the unique name of the failure domain.
+                        Name must be between 1 and 64 characters long. It must consist
+                        of only lower case alphanumeric characters and hyphens (-).
+                        It must start and end with an alphanumeric character.
+                      maxLength: 64
+                      minLength: 1
+                      pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                      type: string
+                    subnets:
+                      description: Subnets holds the list of subnets identifiers cluster's
+                        network subnets.
+                      items:
+                        description: NutanixResourceIdentifier holds the identity
+                          of a Nutanix Prism resource (cluster, image, subnet, etc.)
+                        properties:
+                          name:
+                            description: name is the resource name in the PC
+                            type: string
+                          type:
+                            description: Type is the identifier type to use for this
+                              resource.
+                            enum:
+                            - uuid
+                            - name
+                            type: string
+                          uuid:
+                            description: uuid is the UUID of the resource in the PC.
+                            type: string
+                        required:
+                        - type
+                        type: object
+                      type: array
+                  required:
+                  - name
+                  type: object
+                type: array
               insecure:
                 description: Insecure is the optional flag to skip TLS verification.
                   Nutanix Prism Central installation by default ships with a self-signed

config/manifest/eksa-components.yaml

@@ -5507,6 +5507,70 @@ spec:
               endpoint:
                 description: Endpoint is the Endpoint of Nutanix Prism Central
                 type: string
+              failureDomains:
+                description: FailureDomains is the optional list of failure domains
+                  for the Nutanix Datacenter.
+                items:
+                  description: NutanixDatacenterFailureDomain defines the failure
+                    domain for the Nutanix Datacenter.
+                  properties:
+                    cluster:
+                      description: Cluster is the Prism Element cluster name or uuid
+                        that is connected to the Prism Central.
+                      properties:
+                        name:
+                          description: name is the resource name in the PC
+                          type: string
+                        type:
+                          description: Type is the identifier type to use for this
+                            resource.
+                          enum:
+                          - uuid
+                          - name
+                          type: string
+                        uuid:
+                          description: uuid is the UUID of the resource in the PC.
+                          type: string
+                      required:
+                      - type
+                      type: object
+                    name:
+                      description: Name is the unique name of the failure domain.
+                        Name must be between 1 and 64 characters long. It must consist
+                        of only lower case alphanumeric characters and hyphens (-).
+                        It must start and end with an alphanumeric character.
+                      maxLength: 64
+                      minLength: 1
+                      pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                      type: string
+                    subnets:
+                      description: Subnets holds the list of subnets identifiers cluster's
+                        network subnets.
+                      items:
+                        description: NutanixResourceIdentifier holds the identity
+                          of a Nutanix Prism resource (cluster, image, subnet, etc.)
+                        properties:
+                          name:
+                            description: name is the resource name in the PC
+                            type: string
+                          type:
+                            description: Type is the identifier type to use for this
+                              resource.
+                            enum:
+                            - uuid
+                            - name
+                            type: string
+                          uuid:
+                            description: uuid is the UUID of the resource in the PC.
+                            type: string
+                        required:
+                        - type
+                        type: object
+                      type: array
+                  required:
+                  - name
+                  type: object
+                type: array
               insecure:
                 description: Insecure is the optional flag to skip TLS verification.
                   Nutanix Prism Central installation by default ships with a self-signed

pkg/api/v1alpha1/nutanixdatacenterconfig_test.go

@@ -175,6 +175,22 @@ func TestGetNutanixDatacenterConfigValidConfig(t *testing.T) {
 				assert.Contains(t, err.Error(), "NutanixDatacenterConfig credentialRef name is not set or is empty")
 			},
 		},
+		{
+			name:     "datacenterconfig-valid-failure-domains",
+			fileName: "testdata/nutanix/datacenterconfig-valid-failuredomains.yaml",
+			assertions: func(t *testing.T, dcConf *v1alpha1.NutanixDatacenterConfig) {
+				assert.NoError(t, dcConf.Validate())
+			},
+		},
+		{
+			name:     "datecenterconfig-invalid-failure-domains",
+			fileName: "testdata/nutanix/datacenterconfig-invalid-failuredomains.yaml",
+			assertions: func(t *testing.T, dcConf *v1alpha1.NutanixDatacenterConfig) {
+				err := dcConf.Validate()
+				assert.Error(t, err)
+				assert.Contains(t, err.Error(), "NutanixDatacenterConfig.Spec.FailureDomains.Subnets: missing subnet UUID: default/eksa-unit-test")
+			},
+		},
 	}
 
 	for _, test := range tests {

pkg/api/v1alpha1/nutanixdatacenterconfig_types.go

@@ -43,6 +43,31 @@ type NutanixDatacenterConfigSpec struct {
 	// for the Nutanix Prism Central. The namespace for the secret is assumed to be a constant i.e. eksa-system.
 	// +optional
 	CredentialRef *Ref `json:"credentialRef,omitempty"`
+
+	// FailureDomains is the optional list of failure domains for the Nutanix Datacenter.
+	// +optional
+	FailureDomains []NutanixDatacenterFailureDomain `json:"failureDomains,omitempty"`
+}
+
+// NutanixDatacenterFailureDomain defines the failure domain for the Nutanix Datacenter.
+type NutanixDatacenterFailureDomain struct {
+	// Name is the unique name of the failure domain.
+	// Name must be between 1 and 64 characters long.
+	// It must consist of only lower case alphanumeric characters and hyphens (-).
+	// It must start and end with an alphanumeric character.
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=64
+	// +kubebuilder:validation:Pattern=^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+	Name string `json:"name"`
+
+	// Cluster is the Prism Element cluster name or uuid that is connected to the Prism Central.
+	// +kubebuilder:validation:Required
+	Cluster NutanixResourceIdentifier `json:"cluster,omitempty"`
+
+	// Subnets holds the list of subnets identifiers cluster's network subnets.
+	// +kubebuilder:validation:Required
+	Subnets []NutanixResourceIdentifier `json:"subnets,omitempty"`
 }
 
 // NutanixDatacenterConfigStatus defines the observed state of NutanixDatacenterConfig.
@@ -140,9 +165,42 @@ func (in *NutanixDatacenterConfig) Validate() error {
 		}
 	}
 
+	if in.Spec.FailureDomains != nil && len(in.Spec.FailureDomains) != 0 {
+		dccName := in.Namespace + "/" + in.Name
+		validateClusterResourceIdentifier := createValidateNutanixResourceFunc("NutanixDatacenterConfig.Spec.FailureDomains.Cluster", "cluster", dccName)
+		validateSubnetResourceIdentifier := createValidateNutanixResourceFunc("NutanixDatacenterConfig.Spec.FailureDomains.Subnets", "subnet", dccName)
+		for _, fd := range in.Spec.FailureDomains {
+			if err := validateClusterResourceIdentifier(&fd.Cluster); err != nil {
+				return err
+			}
+
+			for _, subnet := range fd.Subnets {
+				if err := validateSubnetResourceIdentifier(&subnet); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
 	return nil
 }
 
+func createValidateNutanixResourceFunc(msgPrefix, entityName, mfstName string) func(*NutanixResourceIdentifier) error {
+	return func(ntnxRId *NutanixResourceIdentifier) error {
+		if ntnxRId.Type != NutanixIdentifierName && ntnxRId.Type != NutanixIdentifierUUID {
+			return fmt.Errorf("%s: invalid identifier type for %s: %s", msgPrefix, entityName, ntnxRId.Type)
+		}
+
+		if ntnxRId.Type == NutanixIdentifierName && (ntnxRId.Name == nil || *ntnxRId.Name == "") {
+			return fmt.Errorf("%s: missing %s name: %s", msgPrefix, entityName, mfstName)
+		} else if ntnxRId.Type == NutanixIdentifierUUID && (ntnxRId.UUID == nil || *ntnxRId.UUID == "") {
+			return fmt.Errorf("%s: missing %s UUID: %s", msgPrefix, entityName, mfstName)
+		}
+
+		return nil
+	}
+}
+
 // SetDefaults sets default values for the NutanixDatacenterConfig object.
 func (in *NutanixDatacenterConfig) SetDefaults() {
 	if in.Spec.CredentialRef == nil {

pkg/api/v1alpha1/testdata/nutanix/datacenterconfig-invalid-failuredomains.yaml

@@ -0,0 +1,30 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    name: eksa-unit-test
+    kind: Secret
+  failureDomains:
+  - name: "pe1"
+    cluster:
+      type: name
+      name: "prism-cluster-1"
+    subnets:
+    - name: "prism-subnet-1"
+      type: "name"
+    - uuid: ""
+      type: "uuid"
+  - name: "pe2"
+    cluster:
+      type: "uuid"
+      uuid: "468b7b36-d15b-406a-90f7-46d1560c4f4e"
+    subnets:
+    - name: "prism-subnet-1"
+      type: "name"
+    - uuid: "3e716c09-0613-46f3-b46a-beb89aa02295"
+      type: "uuid"

pkg/api/v1alpha1/testdata/nutanix/datacenterconfig-valid-failuredomains.yaml

@@ -0,0 +1,30 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    name: eksa-unit-test
+    kind: Secret
+  failureDomains:
+  - name: "pe1"
+    cluster:
+      type: name
+      name: "prism-cluster-1"
+    subnets:
+    - name: "prism-subnet-1"
+      type: "name"
+    - uuid: "3e716c09-0613-46f3-b46a-beb89aa02295"
+      type: "uuid"
+  - name: "pe2"
+    cluster:
+      type: "uuid"
+      uuid: "468b7b36-d15b-406a-90f7-46d1560c4f4e"
+    subnets:
+    - name: "prism-subnet-1"
+      type: "name"
+    - uuid: "3e716c09-0613-46f3-b46a-beb89aa02295"
+      type: "uuid"

pkg/providers/nutanix/config/cp-template.yaml

@@ -5,7 +5,33 @@ metadata:
   name: "{{.clusterName}}"
   namespace: "{{.eksaSystemNamespace}}"
 spec:
+{{- if .failureDomains }}
+  failureDomains:
+  {{- range $index, $value := .failureDomains}}
+  - name: "{{ $value.Name }}"
+    cluster:
+    {{- if (eq $value.Cluster.Type "uuid") }}
+      type: "uuid"
+      uuid: "{{ $value.Cluster.UUID }}"
+    {{- else if (eq $value.Cluster.Type "name") }}
+      type: "name"
+      name: "{{ $value.Cluster.Name }}"
+    {{- end}}
+    subnets:
+    {{- range $value.Subnets}}
+      {{- if (eq .Type "uuid") }}
+    - type: "uuid"
+      uuid: "{{ .UUID }}"
+      {{- else if (eq .Type "name") }}
+    - type: "name"
+      name: "{{ .Name }}"
+      {{- end}}
+    {{- end}}
+    controlPlane: true
+  {{- end }}
+{{- else }}
   failureDomains: []
+{{- end}}
   prismCentral:
 {{- if .nutanixAdditionalTrustBundle }}
     additionalTrustBundle:

pkg/providers/nutanix/provider.go

@@ -419,7 +419,6 @@ func needsNewEtcdTemplate(oldSpec, newSpec *cluster.Spec, oldNmc, newNmc *v1alph
 	if oldSpec.Bundles.Spec.Number != newSpec.Bundles.Spec.Number {
 		return true
 	}
-
 	return AnyImmutableFieldChanged(oldNmc, newNmc)
 }