diff --git a/apis/infrastructure/v1alpha1/zz_generated.deepcopy.go b/apis/infrastructure/v1alpha1/zz_generated.deepcopy.go index d1c90df..1b3293f 100644 --- a/apis/infrastructure/v1alpha1/zz_generated.deepcopy.go +++ b/apis/infrastructure/v1alpha1/zz_generated.deepcopy.go @@ -22,8 +22,9 @@ package v1alpha1 import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/apis/v1beta1" runtime "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/cluster-api/api/v1beta1" + apiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -100,6 +101,13 @@ func (in *KopsMachinePoolSpec) DeepCopyInto(out *KopsMachinePoolSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.KarpenterNodePools != nil { + in, out := &in.KarpenterNodePools, &out.KarpenterNodePools + *out = make([]v1beta1.NodePool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.KopsInstanceGroupSpec.DeepCopyInto(&out.KopsInstanceGroupSpec) if in.SpotInstOptions != nil { in, out := &in.SpotInstOptions, &out.SpotInstOptions @@ -130,7 +138,7 @@ func (in *KopsMachinePoolStatus) DeepCopyInto(out *KopsMachinePoolStatus) { } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make(v1beta1.Conditions, len(*in)) + *out = make(apiv1beta1.Conditions, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/controllers/controlplane/kopscontrolplane_controller.go b/controllers/controlplane/kopscontrolplane_controller.go index ad30b71..54796c5 100644 --- a/controllers/controlplane/kopscontrolplane_controller.go +++ b/controllers/controlplane/kopscontrolplane_controller.go @@ -59,6 +59,7 @@ import ( "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/pkg/validation" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/controllers/external" @@ -170,11 +171,11 @@ func (r *KopsControlPlaneReconciler) PrepareCustomCloudResources(ctx context.Con } if shouldEnableKarpenter { - karpenterProvisionersContent, err := os.Create(terraformOutputDir + "/data/aws_s3_object_karpenter_provisioners_content") + karpenterResourcesContent, err := os.Create(terraformOutputDir + "/data/aws_s3_object_karpenter_resources_content") if err != nil { return err } - defer karpenterProvisionersContent.Close() + defer karpenterResourcesContent.Close() // This is needed because the apply will fail if the file is empty placeholder := corev1.ConfigMap{ @@ -183,21 +184,21 @@ func (r *KopsControlPlaneReconciler) PrepareCustomCloudResources(ctx context.Con APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "placeholder-karpenter-provisioners", + Name: "placeholder-karpenter-resources", Namespace: "kube-system", }, } - output, err := yaml.Marshal(placeholder) + placeholderBytes, err := yaml.Marshal(placeholder) if err != nil { return err } - if _, err := karpenterProvisionersContent.Write([]byte("---\n")); err != nil { + if _, err := karpenterResourcesContent.Write([]byte("---\n")); err != nil { return err } - if _, err := karpenterProvisionersContent.Write(output); err != nil { + if _, err := karpenterResourcesContent.Write(placeholderBytes); err != nil { return err } @@ -206,19 +207,48 @@ func (r *KopsControlPlaneReconciler) PrepareCustomCloudResources(ctx context.Con provisioner.SetLabels(map[string]string{ "kops.k8s.io/managed-by": "kops-controller", }) - if _, err := karpenterProvisionersContent.Write([]byte("---\n")); err != nil { + if _, err := karpenterResourcesContent.Write([]byte("---\n")); err != nil { + return err + } + provisionerBytes, err := yaml.Marshal(provisioner) + if err != nil { + return err + } + if _, err := karpenterResourcesContent.Write(provisionerBytes); err != nil { + return err + } + } + + for _, nodePool := range kmp.Spec.KarpenterNodePools { + nodePool.SetLabels(map[string]string{ + "kops.k8s.io/managed-by": "kops-controller", + }) + // Create NodePool + if _, err := karpenterResourcesContent.Write([]byte("---\n")); err != nil { + return err + } + nodePoolBytes, err := yaml.Marshal(nodePool) + if err != nil { + return err + } + if _, err := karpenterResourcesContent.Write(nodePoolBytes); err != nil { return err } - output, err := yaml.Marshal(provisioner) + // Create EC2NodeClass + if _, err := karpenterResourcesContent.Write([]byte("---\n")); err != nil { + return err + } + ec2NodeClassString, err := utils.CreateEC2NodeClassFromKops(kopsCluster, kmp.DeepCopy(), terraformOutputDir) if err != nil { return err } - if _, err := karpenterProvisionersContent.Write(output); err != nil { + + if _, err := karpenterResourcesContent.Write([]byte(ec2NodeClassString)); err != nil { return err } } } - fileData, err := os.ReadFile(karpenterProvisionersContent.Name()) + fileData, err := os.ReadFile(karpenterResourcesContent.Name()) if err != nil { return err } @@ -584,7 +614,6 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req kopsControlPlaneHelper := kopsControlPlane.DeepCopy() if err := reconciler.Update(ctx, kopsControlPlane); err != nil { - reconciler.log.Info(fmt.Sprintf("%+v", kopsControlPlane)) r.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, controlplanev1alpha1.FailedToUpdateKopsControlPlane, "failed to update kopsControlPlane: %s", err) } @@ -766,6 +795,9 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req if len(kopsMachinePool.Spec.KarpenterProvisioners) > 0 { shouldEnableKarpenter = true } + if len(kopsMachinePool.Spec.KarpenterNodePools) > 0 { + shouldEnableKarpenter = true + } } if shouldEnableKarpenter { @@ -827,12 +859,6 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req reconciler.log.Info(fmt.Sprintf("generating Terraform files for %s", kopsControlPlane.ObjectMeta.GetName())) - // Prepare custom cloud resources - err = reconciler.PrepareCustomCloudResources(ctx, kopsCluster, kopsControlPlane, existingKopsMachinePool, shouldEnableKarpenter, fullCluster.Spec.ConfigStore.Base, terraformOutputDir, shouldIgnoreSG) - if err != nil { - return resultError, err - } - err = reconciler.PrepareKopsCloudResourcesFactory(ctx, kopsClientset, kopsCluster, terraformOutputDir, cloud) if err != nil { conditions.MarkFalse(kopsControlPlane, controlplanev1alpha1.KopsTerraformGenerationReadyCondition, controlplanev1alpha1.KopsTerraformGenerationReconciliationFailedReason, clusterv1.ConditionSeverityError, "failed to prepare cloud resources: %s", err.Error()) @@ -841,6 +867,12 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req } conditions.MarkTrue(kopsControlPlane, controlplanev1alpha1.KopsTerraformGenerationReadyCondition) + // Prepare custom cloud resources + err = reconciler.PrepareCustomCloudResources(ctx, kopsCluster, kopsControlPlane, existingKopsMachinePool, shouldEnableKarpenter, fullCluster.Spec.ConfigStore.Base, terraformOutputDir, shouldIgnoreSG) + if err != nil { + return resultError, err + } + // TODO: This is needed because we are using a method from kops lib // we should check alternatives kubeConfig, err := utils.GetKubeconfigFromKopsState(ctx, kopsCluster, kopsClientset) @@ -870,7 +902,7 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req reconciler.log.Info(fmt.Sprintf("Terraform applied for %s", kopsControlPlane.ObjectMeta.GetName())) reconciler.Recorder.Event(kopsControlPlane, corev1.EventTypeNormal, "TerraformApplied", "Terraform applied") - err = reconciler.updateKopsMachinePoolWithProviderIDList(ctx, kopsControlPlane, kmps, &reconciler.awsCredentials) + err = reconciler.updateKopsMachinePoolWithProviderIDList(kopsControlPlane, kmps, &reconciler.awsCredentials) if err != nil { if apierrors.IsNotFound(err) { return requeue1min, nil @@ -895,7 +927,7 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req return resultDefault, nil } -func (r *KopsControlPlaneReconciliation) updateKopsMachinePoolWithProviderIDList(ctx context.Context, kopsControlPlane *controlplanev1alpha1.KopsControlPlane, kmps []infrastructurev1alpha1.KopsMachinePool, credentials *aws.Credentials) error { +func (r *KopsControlPlaneReconciliation) updateKopsMachinePoolWithProviderIDList(kopsControlPlane *controlplanev1alpha1.KopsControlPlane, kmps []infrastructurev1alpha1.KopsMachinePool, credentials *aws.Credentials) error { for i, kopsMachinePool := range kmps { // TODO: retrieve karpenter providerIDList if len(kopsMachinePool.Spec.SpotInstOptions) == 0 && kopsMachinePool.Spec.KopsInstanceGroupSpec.Manager != "Karpenter" { diff --git a/controllers/controlplane/kopscontrolplane_controller_test.go b/controllers/controlplane/kopscontrolplane_controller_test.go index 2a848b7..0f840ed 100644 --- a/controllers/controlplane/kopscontrolplane_controller_test.go +++ b/controllers/controlplane/kopscontrolplane_controller_test.go @@ -1,11 +1,13 @@ package controlplane import ( + "bytes" "context" "errors" "fmt" "os" "strings" + "text/template" dto "github.com/prometheus/client_model/go" @@ -17,6 +19,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/topfreegames/kubernetes-kops-operator/pkg/helpers" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -1888,19 +1891,378 @@ func TestGetRegionBySubnet(t *testing.T) { func TestPrepareCustomCloudResources(t *testing.T) { var testCases = []struct { - description string - spotInstEnabled bool - expectedError bool + description string + kopsMachinePoolFunction func(*infrastructurev1alpha1.KopsMachinePool) *infrastructurev1alpha1.KopsMachinePool + karpenterResourcesOutput string + manifestHash string + spotInstEnabled bool }{ { - description: "Should generate files based on template", - spotInstEnabled: false, - expectedError: false, + description: "Should generate files based on template with one Provisioner", + kopsMachinePoolFunction: func(kmp *infrastructurev1alpha1.KopsMachinePool) *infrastructurev1alpha1.KopsMachinePool { + kmp.Spec.KopsInstanceGroupSpec.NodeLabels = map[string]string{ + "kops.k8s.io/instance-group-role": "Node", + } + kmp.Spec.KarpenterProvisioners = []v1alpha5.Provisioner{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Provisioner", + APIVersion: "karpenter.sh/v1alpha5", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-provisioner", + }, + Spec: v1alpha5.ProvisionerSpec{ + Consolidation: &v1alpha5.Consolidation{ + Enabled: aws.Bool(true), + }, + KubeletConfiguration: &v1alpha5.KubeletConfiguration{ + KubeReserved: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("150m"), + corev1.ResourceMemory: resource.MustParse("150Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + SystemReserved: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("150m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + }, + Labels: map[string]string{ + "kops.k8s.io/cluster": helpers.GetFQDN("test-cluster"), + "kops.k8s.io/cluster-name": helpers.GetFQDN("test-cluster"), + "kops.k8s.io/instance-group-name": kmp.Name, + "kops.k8s.io/instance-group-role": "Node", + "kops.k8s.io/instancegroup": kmp.Name, + "kops.k8s.io/managed-by": "kops-controller", + }, + Provider: &v1alpha5.Provider{ + Raw: []byte("{\"launchTemplate\":\"" + kmp.Name + "." + helpers.GetFQDN("test-cluster") + "\",\"subnetSelector\":{\"kops.k8s.io/instance-group/" + kmp.Name + "\":\"*\",\"kubernetes.io/cluster/" + helpers.GetFQDN("test-cluster") + "\":\"*\"}}"), + }, + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/arch", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"amd64"}, + }, + { + Key: "kubernetes.io/os", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"linux"}, + }, + { + Key: "node.kubernetes.io/instance-type", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"m5.large"}, + }, + }, + StartupTaints: []corev1.Taint{ + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Effect: corev1.TaintEffect(corev1.TaintEffectNoSchedule), + }, + }, + }, + }, + } + return kmp + }, + karpenterResourcesOutput: "karpenter_resource_output_provisioner.yaml", + manifestHash: "d67c9504589dd859e46f1913780fb69bafb8df5328d90e6675affc79d3573f78", + }, + { + description: "Should generate files based on template with one NodePool", + kopsMachinePoolFunction: func(kmp *infrastructurev1alpha1.KopsMachinePool) *infrastructurev1alpha1.KopsMachinePool { + kmp.Spec.KopsInstanceGroupSpec.NodeLabels = map[string]string{ + "kops.k8s.io/instance-group-role": "Node", + } + kmp.Spec.KarpenterNodePools = []v1beta1.NodePool{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "NodePool", + APIVersion: "karpenter.sh/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node-pool", + }, + Spec: v1beta1.NodePoolSpec{ + Disruption: v1beta1.Disruption{ + ConsolidationPolicy: v1beta1.ConsolidationPolicyWhenUnderutilized, + }, + Template: v1beta1.NodeClaimTemplate{ + ObjectMeta: v1beta1.ObjectMeta{ + Labels: map[string]string{ + "kops.k8s.io/cluster": helpers.GetFQDN("test-cluster"), + "kops.k8s.io/cluster-name": helpers.GetFQDN("test-cluster"), + "kops.k8s.io/instance-group-name": kmp.Name, + "kops.k8s.io/instance-group-role": "Node", + "kops.k8s.io/instancegroup": kmp.Name, + "kops.k8s.io/managed-by": "kops-controller", + }, + }, + Spec: v1beta1.NodeClaimSpec{ + Kubelet: &v1beta1.KubeletConfiguration{ + KubeReserved: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("150m"), + corev1.ResourceMemory: resource.MustParse("150Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + SystemReserved: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("150m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + }, + NodeClassRef: &v1beta1.NodeClassReference{ + Name: "test-ig", + }, + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/arch", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"amd64"}, + }, + { + Key: "kubernetes.io/os", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"linux"}, + }, + { + Key: "node.kubernetes.io/instance-type", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"m5.large"}, + }, + }, + StartupTaints: []corev1.Taint{ + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Effect: corev1.TaintEffect(corev1.TaintEffectNoSchedule), + }, + }, + }, + }, + }, + }, + } + return kmp + }, + karpenterResourcesOutput: "karpenter_resource_output_node_pool.yaml", + manifestHash: "3c4ab5f41dc8a8148fd93d7d5b3e0fd0c2127b7f7eed5c547444d1817cec275d", }, { - description: "Should generate files based on with spotinst enabled", - spotInstEnabled: true, - expectedError: false, + description: "Should generate files based on template with one NodePool and one Provisioner", + kopsMachinePoolFunction: func(kmp *infrastructurev1alpha1.KopsMachinePool) *infrastructurev1alpha1.KopsMachinePool { + kmp.Spec.KopsInstanceGroupSpec.NodeLabels = map[string]string{ + "kops.k8s.io/instance-group-role": "Node", + } + kmp.Spec.KarpenterNodePools = []v1beta1.NodePool{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "NodePool", + APIVersion: "karpenter.sh/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node-pool", + }, + Spec: v1beta1.NodePoolSpec{ + Disruption: v1beta1.Disruption{ + ConsolidationPolicy: v1beta1.ConsolidationPolicyWhenUnderutilized, + }, + Template: v1beta1.NodeClaimTemplate{ + ObjectMeta: v1beta1.ObjectMeta{ + Labels: map[string]string{ + "kops.k8s.io/cluster": helpers.GetFQDN("test-cluster"), + "kops.k8s.io/cluster-name": helpers.GetFQDN("test-cluster"), + "kops.k8s.io/instance-group-name": kmp.Name, + "kops.k8s.io/instance-group-role": "Node", + "kops.k8s.io/instancegroup": kmp.Name, + "kops.k8s.io/managed-by": "kops-controller", + }, + }, + Spec: v1beta1.NodeClaimSpec{ + Kubelet: &v1beta1.KubeletConfiguration{ + KubeReserved: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("150m"), + corev1.ResourceMemory: resource.MustParse("150Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + SystemReserved: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("150m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + }, + NodeClassRef: &v1beta1.NodeClassReference{ + Name: "test-ig", + }, + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/arch", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"amd64"}, + }, + { + Key: "kubernetes.io/os", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"linux"}, + }, + { + Key: "node.kubernetes.io/instance-type", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"m5.large"}, + }, + }, + StartupTaints: []corev1.Taint{ + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Effect: corev1.TaintEffect(corev1.TaintEffectNoSchedule), + }, + }, + }, + }, + }, + }, + } + kmp.Spec.KarpenterProvisioners = []v1alpha5.Provisioner{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Provisioner", + APIVersion: "karpenter.sh/v1alpha5", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-provisioner", + }, + Spec: v1alpha5.ProvisionerSpec{ + Consolidation: &v1alpha5.Consolidation{ + Enabled: aws.Bool(true), + }, + KubeletConfiguration: &v1alpha5.KubeletConfiguration{ + KubeReserved: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("150m"), + corev1.ResourceMemory: resource.MustParse("150Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + SystemReserved: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("150m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + }, + Labels: map[string]string{ + "kops.k8s.io/cluster": helpers.GetFQDN("test-cluster"), + "kops.k8s.io/cluster-name": helpers.GetFQDN("test-cluster"), + "kops.k8s.io/instance-group-name": kmp.Name, + "kops.k8s.io/instance-group-role": "Node", + "kops.k8s.io/instancegroup": kmp.Name, + "kops.k8s.io/managed-by": "kops-controller", + }, + Provider: &v1alpha5.Provider{ + Raw: []byte("{\"launchTemplate\":\"" + kmp.Name + "." + helpers.GetFQDN("test-cluster") + "\",\"subnetSelector\":{\"kops.k8s.io/instance-group/" + kmp.Name + "\":\"*\",\"kubernetes.io/cluster/" + helpers.GetFQDN("test-cluster") + "\":\"*\"}}"), + }, + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/arch", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"amd64"}, + }, + { + Key: "kubernetes.io/os", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"linux"}, + }, + { + Key: "node.kubernetes.io/instance-type", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"m5.large"}, + }, + }, + StartupTaints: []corev1.Taint{ + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Effect: corev1.TaintEffect(corev1.TaintEffectNoSchedule), + }, + }, + }, + }, + } + return kmp + }, + karpenterResourcesOutput: "karpenter_resource_output_node_pool_and_provisioner.yaml", + manifestHash: "0ac4ecf6655af3b0e163f109a98d88e5b500a045cbe52e47aa0052d977289bd5", + }, + { + description: "Should generate files based on with spotinst enabled", + kopsMachinePoolFunction: func(kmp *infrastructurev1alpha1.KopsMachinePool) *infrastructurev1alpha1.KopsMachinePool { + kmp.Spec.KopsInstanceGroupSpec.NodeLabels = map[string]string{ + "kops.k8s.io/instance-group-role": "Node", + } + kmp.Spec.KarpenterProvisioners = []v1alpha5.Provisioner{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Provisioner", + APIVersion: "karpenter.sh/v1alpha5", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-provisioner", + }, + Spec: v1alpha5.ProvisionerSpec{ + Consolidation: &v1alpha5.Consolidation{ + Enabled: aws.Bool(true), + }, + KubeletConfiguration: &v1alpha5.KubeletConfiguration{ + KubeReserved: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("150m"), + corev1.ResourceMemory: resource.MustParse("150Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + SystemReserved: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("150m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + }, + Labels: map[string]string{ + "kops.k8s.io/cluster": helpers.GetFQDN("test-cluster"), + "kops.k8s.io/cluster-name": helpers.GetFQDN("test-cluster"), + "kops.k8s.io/instance-group-name": kmp.Name, + "kops.k8s.io/instance-group-role": "Node", + "kops.k8s.io/instancegroup": kmp.Name, + "kops.k8s.io/managed-by": "kops-controller", + }, + Provider: &v1alpha5.Provider{ + Raw: []byte("{\"launchTemplate\":\"" + kmp.Name + "." + helpers.GetFQDN("test-cluster") + "\",\"subnetSelector\":{\"kops.k8s.io/instance-group/" + kmp.Name + "\":\"*\",\"kubernetes.io/cluster/" + helpers.GetFQDN("test-cluster") + "\":\"*\"}}"), + }, + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/arch", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"amd64"}, + }, + { + Key: "kubernetes.io/os", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"linux"}, + }, + { + Key: "node.kubernetes.io/instance-type", + Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), + Values: []string{"m5.large"}, + }, + }, + StartupTaints: []corev1.Taint{ + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Effect: corev1.TaintEffect(corev1.TaintEffectNoSchedule), + }, + }, + }, + }, + } + return kmp + }, + karpenterResourcesOutput: "karpenter_resource_output_provisioner.yaml", + manifestHash: "d67c9504589dd859e46f1913780fb69bafb8df5328d90e6675affc79d3573f78", + spotInstEnabled: true, }, } @@ -1914,76 +2276,14 @@ func TestPrepareCustomCloudResources(t *testing.T) { vfs.Context.ResetMemfsContext(true) bareKopsCluster := helpers.NewKopsCluster("test-cluster") kopsCluster, err := fakeKopsClientset.CreateCluster(ctx, bareKopsCluster) + kmp := helpers.NewKopsMachinePool("test-ig", metav1.NamespaceDefault, "test-cluster") g.Expect(err).NotTo(HaveOccurred()) g.Expect(kopsCluster).NotTo(BeNil()) kcp := &controlplanev1alpha1.KopsControlPlane{} kcp.Spec.SpotInst.Enabled = tc.spotInstEnabled - kmp := helpers.NewKopsMachinePool("test-ig", metav1.NamespaceDefault, "test-cluster") - kmp.Spec.KopsInstanceGroupSpec.NodeLabels = map[string]string{ - "kops.k8s.io/instance-group-role": "Node", - } - kmp.Spec.KarpenterProvisioners = []v1alpha5.Provisioner{ - { - TypeMeta: metav1.TypeMeta{ - Kind: "Provisioner", - APIVersion: "karpenter.sh/v1alpha5", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-provisioner", - }, - Spec: v1alpha5.ProvisionerSpec{ - Consolidation: &v1alpha5.Consolidation{ - Enabled: aws.Bool(true), - }, - KubeletConfiguration: &v1alpha5.KubeletConfiguration{ - KubeReserved: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("150m"), - corev1.ResourceMemory: resource.MustParse("150Mi"), - corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), - }, - SystemReserved: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("150m"), - corev1.ResourceMemory: resource.MustParse("200Mi"), - corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), - }, - }, - Labels: map[string]string{ - "kops.k8s.io/cluster": kopsCluster.Name, - "kops.k8s.io/cluster-name": kopsCluster.Name, - "kops.k8s.io/instance-group-name": kmp.Name, - "kops.k8s.io/instance-group-role": "Node", - "kops.k8s.io/instancegroup": kmp.Name, - "kops.k8s.io/managed-by": "kops-controller", - }, - Provider: &v1alpha5.Provider{ - Raw: []byte("{\"launchTemplate\":\"" + kmp.Name + "." + kopsCluster.Name + "\",\"subnetSelector\":{\"kops.k8s.io/instance-group/" + kmp.Name + "\":\"*\",\"kubernetes.io/cluster/" + kopsCluster.Name + "\":\"*\"}}"), - }, - Requirements: []corev1.NodeSelectorRequirement{ - { - Key: "kubernetes.io/arch", - Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), - Values: []string{"amd64"}, - }, - { - Key: "kubernetes.io/os", - Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), - Values: []string{"linux"}, - }, - { - Key: "node.kubernetes.io/instance-type", - Operator: corev1.NodeSelectorOperator(corev1.NodeSelectorOpIn), - Values: []string{"m5.large"}, - }, - }, - StartupTaints: []corev1.Taint{ - { - Key: "node.cloudprovider.kubernetes.io/uninitialized", - Effect: corev1.TaintEffect(corev1.TaintEffectNoSchedule), - }, - }, - }, - }, + if tc.kopsMachinePoolFunction != nil { + kmp = tc.kopsMachinePoolFunction(kmp) } if tc.spotInstEnabled { @@ -1993,12 +2293,16 @@ func TestPrepareCustomCloudResources(t *testing.T) { } } - reconciler := &KopsControlPlaneReconciler{} terraformOutputDir := fmt.Sprintf("/tmp/%s", kopsCluster.Name) + templateTestDir := "../../utils/templates/tests" + + err = os.WriteFile(terraformOutputDir+"/data/aws_launch_template_"+kmp.Name+"."+kopsCluster.Name+"_user_data", []byte("dummy content"), 0644) + g.Expect(err).NotTo(HaveOccurred()) + + reconciler := &KopsControlPlaneReconciler{} err = reconciler.PrepareCustomCloudResources(ctx, kopsCluster, kcp, []infrastructurev1alpha1.KopsMachinePool{*kmp}, true, kopsCluster.Spec.ConfigStore.Base, terraformOutputDir, true) g.Expect(err).NotTo(HaveOccurred()) - templateTestDir := "../../utils/templates/tests" generatedBackendTF, err := os.ReadFile(terraformOutputDir + "/backend.tf") g.Expect(err).NotTo(HaveOccurred()) templatedBackendTF, err := os.ReadFile(templateTestDir + "/backend.tf") @@ -2007,9 +2311,24 @@ func TestPrepareCustomCloudResources(t *testing.T) { generatedKarpenterBoostrapTF, err := os.ReadFile(terraformOutputDir + "/karpenter_custom_addon_boostrap.tf") g.Expect(err).NotTo(HaveOccurred()) - templatedKarpenterBoostrapTF, err := os.ReadFile(templateTestDir + "/karpenter_custom_addon_boostrap.tf") + + content, err := os.ReadFile(templateTestDir + "/karpenter_custom_addon_boostrap.tf") g.Expect(err).NotTo(HaveOccurred()) - g.Expect(string(generatedKarpenterBoostrapTF)).To(BeEquivalentTo(string(templatedKarpenterBoostrapTF))) + + templ, err := template.New(templateTestDir + "/karpenter_custom_addon_boostrap.tf").Parse(string(content)) + g.Expect(err).NotTo(HaveOccurred()) + + var templatedKarpenterBoostrapTF bytes.Buffer + data := struct { + ManifestHash string + }{ + ManifestHash: tc.manifestHash, + } + + err = templ.Execute(&templatedKarpenterBoostrapTF, data) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(string(generatedKarpenterBoostrapTF)).To(BeEquivalentTo(templatedKarpenterBoostrapTF.String())) generatedLaunchTemplateTF, err := os.ReadFile(terraformOutputDir + "/launch_template_override.tf") g.Expect(err).NotTo(HaveOccurred()) @@ -2017,11 +2336,11 @@ func TestPrepareCustomCloudResources(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(string(generatedLaunchTemplateTF)).To(BeEquivalentTo(string(templatedLaunchTemplateTF))) - generatedProvisionerContentTF, err := os.ReadFile(terraformOutputDir + "/data/aws_s3_object_karpenter_provisioners_content") + generatedKarpenterResources, err := os.ReadFile(terraformOutputDir + "/data/aws_s3_object_karpenter_resources_content") g.Expect(err).NotTo(HaveOccurred()) - templatedProvisionerContentTF, err := os.ReadFile(templateTestDir + "/data/aws_s3_object_karpenter_provisioners_content") + templatedKarpenterResources, err := os.ReadFile(templateTestDir + "/data/" + tc.karpenterResourcesOutput) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(string(generatedProvisionerContentTF)).To(BeEquivalentTo(string(templatedProvisionerContentTF))) + g.Expect(string(generatedKarpenterResources)).To(BeEquivalentTo(string(templatedKarpenterResources))) if tc.spotInstEnabled { generatedSpotinstLaunchSpecTF, err := os.ReadFile(terraformOutputDir + "/spotinst_launch_spec_override.tf") diff --git a/go.mod b/go.mod index d03abcf..6f59490 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.11.1 github.com/aws/aws-sdk-go-v2/credentials v1.6.5 github.com/aws/aws-sdk-go-v2/service/autoscaling v1.28.3 - github.com/aws/karpenter v0.32.4 github.com/aws/karpenter-core v0.32.4 github.com/crossplane-contrib/provider-aws v0.30.1 github.com/go-logr/logr v1.4.2 @@ -119,7 +118,7 @@ require ( github.com/GoogleCloudPlatform/k8s-cloud-provider v1.20.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect - github.com/Masterminds/sprig/v3 v3.2.3 // indirect + github.com/Masterminds/sprig/v3 v3.2.3 github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/service/ec2 v1.26.0 // indirect github.com/aws/smithy-go v1.13.5 // indirect diff --git a/go.sum b/go.sum index b118067..97b54fe 100644 --- a/go.sum +++ b/go.sum @@ -161,8 +161,6 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.12.8/go.mod h1:GNIveDnP+aE3jujyUSH5a github.com/aws/aws-sdk-go-v2/service/sts v1.12.0/go.mod h1:UV2N5HaPfdbDpkgkz4sRzWCvQswZjdO1FfqCWl0t7RA= github.com/aws/aws-sdk-go-v2/service/sts v1.18.9 h1:Qf1aWwnsNkyAoqDqmdM3nHwN78XQjec27LjM6b9vyfI= github.com/aws/aws-sdk-go-v2/service/sts v1.18.9/go.mod h1:yyW88BEPXA2fGFyI2KCcZC3dNpiT0CZAHaF+i656/tQ= -github.com/aws/karpenter v0.32.4 h1:IKCupG5sbgb8QfBSsjWbm317t+uyTHbsOlCwzBHaK0I= -github.com/aws/karpenter v0.32.4/go.mod h1:M5Xhy7P/OM+IU2DvoZJkciOhC3M78pURIxaVq7sBLIE= github.com/aws/karpenter-core v0.32.4 h1:rK9QLoWPZSxMXSreV+4aZ6Tt91Zh7Z/p3imM3Oi7eTQ= github.com/aws/karpenter-core v0.32.4/go.mod h1:RNih2g6qCiah8rFaZ7HkmClIK66Hjj38z3DbWnWGM2w= github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= @@ -286,6 +284,8 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= diff --git a/pkg/helpers/helpers.go b/pkg/helpers/helpers.go index 7f34dfb..557f293 100644 --- a/pkg/helpers/helpers.go +++ b/pkg/helpers/helpers.go @@ -239,7 +239,8 @@ func NewKopsMachinePool(name, namespace, clusterName string) *infrastructurev1al Spec: infrastructurev1alpha1.KopsMachinePoolSpec{ ClusterName: clusterName, KopsInstanceGroupSpec: kopsapi.InstanceGroupSpec{ - Role: "ControlPlane", + Image: "xxxx/ubuntu-v1", + Role: "ControlPlane", Subnets: []string{ "dummy-subnet", }, diff --git a/utils/fixtures/karpenter/test_successful_ec2_node_class.tpl b/utils/fixtures/karpenter/test_successful_ec2_node_class.tpl new file mode 100644 index 0000000..c78808b --- /dev/null +++ b/utils/fixtures/karpenter/test_successful_ec2_node_class.tpl @@ -0,0 +1,28 @@ +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: test-machine-pool + labels: + kops.k8s.io/managed-by: kops-controller +spec: + amiFamily: Custom + amiSelectorTerms: + - name: ubuntu-v1 + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: required + role: nodes.test-cluster.test.k8s.cluster + securityGroupSelectorTerms: + - name: nodes.test-cluster.test.k8s.cluster + - tags: + karpenter/owner: test-cluster.test.k8s.cluster/test-machine-pool + subnetSelectorTerms: + - tags: + kops.k8s.io/instance-group/test-machine-pool: '*' + kubernetes.io/cluster/test-cluster.test.k8s.cluster: '*' + tags: + k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node: '' + userData: | + dummy content \ No newline at end of file diff --git a/utils/karpenter_utils.go b/utils/karpenter_utils.go new file mode 100644 index 0000000..c33cb35 --- /dev/null +++ b/utils/karpenter_utils.go @@ -0,0 +1,54 @@ +package utils + +import ( + "bytes" + "text/template" + + "github.com/Masterminds/sprig/v3" + infrastructurev1alpha1 "github.com/topfreegames/kubernetes-kops-operator/apis/infrastructure/v1alpha1" + kopsapi "k8s.io/kops/pkg/apis/kops" +) + +func CreateEC2NodeClassFromKops(kopsCluster *kopsapi.Cluster, kmp *infrastructurev1alpha1.KopsMachinePool, terraformOutputDir string) (string, error) { + amiName, err := GetAmiNameFromImageSource(kmp.Spec.KopsInstanceGroupSpec.Image) + if err != nil { + return "", err + } + + userData, err := GetUserDataFromTerraformFile(kopsCluster.Name, kmp.Name, terraformOutputDir) + if err != nil { + return "", err + } + + data := struct { + Name string + AmiName string + ClusterName string + Tags map[string]string + UserData string + }{ + Name: kmp.Name, + AmiName: amiName, + ClusterName: kopsCluster.Name, + Tags: kopsCluster.Spec.CloudLabels, + UserData: userData, + } + + content, err := templates.ReadFile("templates/ec2nodeclass.yaml.tpl") + if err != nil { + return "", err + } + + t, err := template.New("template").Funcs(sprig.TxtFuncMap()).Parse(string(content)) + if err != nil { + return "", err + } + + var buf bytes.Buffer + err = t.Execute(&buf, data) + if err != nil { + return "", err + } + + return buf.String(), nil +} diff --git a/utils/karpenter_utils_test.go b/utils/karpenter_utils_test.go new file mode 100644 index 0000000..3b46636 --- /dev/null +++ b/utils/karpenter_utils_test.go @@ -0,0 +1,76 @@ +package utils + +import ( + "fmt" + "os" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + infrastructurev1alpha1 "github.com/topfreegames/kubernetes-kops-operator/apis/infrastructure/v1alpha1" + "github.com/topfreegames/kubernetes-kops-operator/pkg/helpers" +) + +func TestCreateEC2NodeClassFromKops(t *testing.T) { + testCases := []struct { + description string + kopsMachinePoolFunction func(kopsMachinePool *infrastructurev1alpha1.KopsMachinePool) *infrastructurev1alpha1.KopsMachinePool + userData string + expectedError error + expectedOutputFile string + }{ + { + description: "should return the populated ec2 node class", + expectedOutputFile: "fixtures/karpenter/test_successful_ec2_node_class.tpl", + userData: "dummy content", + }, + { + description: "should fail with a invalid image", + kopsMachinePoolFunction: func(kopsMachinePool *infrastructurev1alpha1.KopsMachinePool) *infrastructurev1alpha1.KopsMachinePool { + kopsMachinePool.Spec.KopsInstanceGroupSpec.Image = "invalid-image" + return kopsMachinePool + }, + expectedError: fmt.Errorf("invalid image format, should receive image source"), + userData: "dummy content", + }, + { + description: "should fail with empty user data", + expectedError: fmt.Errorf("user data file is empty"), + }, + } + RegisterFailHandler(Fail) + g := NewWithT(t) + + for _, tc := range testCases { + + t.Run(tc.description, func(t *testing.T) { + + kopsCluster := helpers.NewKopsCluster("test-cluster") + + kmp := helpers.NewKopsMachinePool("test-machine-pool", "default", "test-cluster") + + terraformOutputDir := fmt.Sprintf("/tmp/%s", kopsCluster.Name) + + err := os.WriteFile(terraformOutputDir+"/data/aws_launch_template_"+kmp.Name+"."+kopsCluster.Name+"_user_data", []byte(tc.userData), 0644) + g.Expect(err).NotTo(HaveOccurred()) + + if tc.kopsMachinePoolFunction != nil { + kmp = tc.kopsMachinePoolFunction(kmp) + } + + ec2NodeClassString, err := CreateEC2NodeClassFromKops(kopsCluster, kmp, terraformOutputDir) + if tc.expectedError != nil { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(Equal(tc.expectedError)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + expectedOutput, err := os.ReadFile(tc.expectedOutputFile) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ec2NodeClassString).To(Equal(string(expectedOutput))) + + } + }) + } + +} diff --git a/utils/kops_utils.go b/utils/kops_utils.go index 8d0ca4f..97426d5 100644 --- a/utils/kops_utils.go +++ b/utils/kops_utils.go @@ -6,6 +6,7 @@ import ( "crypto/x509/pkix" "errors" "fmt" + "io" "os" "strings" @@ -101,7 +102,7 @@ func ParseSpotinstFeatureflags(kopsControlPlane *controlplanev1alpha1.KopsContro func BuildCloud(kopscluster *kopsapi.Cluster) (_ fi.Cloud, rerr error) { defer func() { if r := recover(); r != nil { - rerr = fmt.Errorf("failed to instantiate cloud for %s", kopscluster.ObjectMeta.GetName()) + rerr = fmt.Errorf("failed to instantiate cloud for %s", kopscluster.ObjectMeta.GetName()) } }() awsup.ResetAWSCloudInstances() @@ -335,3 +336,28 @@ func KopsDeleteResources(ctx context.Context, cloud fi.Cloud, kopsClientset simp return nil } + +func GetAmiNameFromImageSource(image string) (string, error) { + parts := strings.SplitN(image, "/", 2) + if len(parts) > 1 { + return parts[1], nil + } else { + return "", errors.New("invalid image format, should receive image source") + } +} + +func GetUserDataFromTerraformFile(clusterName, igName, terraformOutputDir string) (string, error) { + userDataFile, err := os.Open(fmt.Sprintf(terraformOutputDir+"/data/aws_launch_template_%s.%s_user_data", igName, clusterName)) + if err != nil { + return "", err + } + defer userDataFile.Close() + userData, err := io.ReadAll(userDataFile) + if err != nil { + return "", err + } + if len(userData) == 0 { + return "", errors.New("user data file is empty") + } + return string(userData), nil +} diff --git a/utils/kops_utils_test.go b/utils/kops_utils_test.go index 867e832..b246fee 100644 --- a/utils/kops_utils_test.go +++ b/utils/kops_utils_test.go @@ -3,8 +3,13 @@ package utils import ( "bytes" "context" + "errors" + "fmt" + "io/fs" "os" + "path/filepath" "strings" + "syscall" "testing" "github.com/topfreegames/kubernetes-kops-operator/pkg/helpers" @@ -443,3 +448,108 @@ func TestReconcileKopsSecrets(t *testing.T) { }) } } + +func TestGetAmiNameFromImageSource(t *testing.T) { + testCases := []struct { + description string + input string + output string + expectedError error + }{ + { + description: "should return the ami name from the image source", + input: "000000000000/ubuntu-v1.0.0", + output: "ubuntu-v1.0.0", + }, + { + description: "should fail when receiving ami id", + input: "ami-000000000000", + expectedError: errors.New("invalid image format, should receive image source"), + }, + { + description: "should fail when receiving ami name", + input: "ubuntu-v1.0.0", + expectedError: errors.New("invalid image format, should receive image source"), + }, + } + RegisterFailHandler(Fail) + g := NewWithT(t) + + for _, tc := range testCases { + + t.Run(tc.description, func(t *testing.T) { + + amiName, err := GetAmiNameFromImageSource(tc.input) + if tc.expectedError != nil { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(Equal(tc.expectedError)) + } else { + g.Expect(err).To(BeNil()) + g.Expect(amiName).To(Equal(tc.output)) + } + + }) + } + +} + +func TestGetUserDataFromTerraformFile(t *testing.T) { + testCases := []struct { + description string + userData string + terraformFile string + output string + expectedError error + writeTerraformFile bool + }{ + { + description: "should return the user data from the terraform file", + userData: "dummy content", + writeTerraformFile: true, + }, + { + description: "should fail if the user data is not found", + expectedError: &fs.PathError{Op: "open", Path: "/tmp/test-cluster.test.k8s.cluster/data/aws_launch_template_test-machine-pool.test-cluster.test.k8s.cluster_user_data", Err: syscall.Errno(0x2)}, + }, + { + description: "should fail if the user data is empty", + writeTerraformFile: true, + expectedError: errors.New("user data file is empty"), + }, + } + RegisterFailHandler(Fail) + g := NewWithT(t) + + for _, tc := range testCases { + + t.Run(tc.description, func(t *testing.T) { + + kopsCluster := helpers.NewKopsCluster("test-cluster") + + terraformOutputDir := fmt.Sprintf("/tmp/%s", kopsCluster.Name) + + err := os.RemoveAll(terraformOutputDir) + g.Expect(err).NotTo(HaveOccurred()) + + err = os.MkdirAll(filepath.Join(terraformOutputDir, "data"), os.ModePerm) + g.Expect(err).NotTo(HaveOccurred()) + + kmp := helpers.NewKopsMachinePool("test-machine-pool", "default", "test-cluster") + + if tc.writeTerraformFile { + err := os.WriteFile(terraformOutputDir+"/data/aws_launch_template_"+kmp.Name+"."+kopsCluster.Name+"_user_data", []byte(tc.userData), 0644) + g.Expect(err).NotTo(HaveOccurred()) + } + userDataString, err := GetUserDataFromTerraformFile(kopsCluster.Name, kmp.Name, terraformOutputDir) + if tc.expectedError != nil { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(Equal(tc.expectedError)) + } else { + g.Expect(err).To(BeNil()) + g.Expect(userDataString).To(Equal(tc.userData)) + } + + }) + } + +} diff --git a/utils/templates/ec2nodeclass.yaml.tpl b/utils/templates/ec2nodeclass.yaml.tpl new file mode 100644 index 0000000..c65b871 --- /dev/null +++ b/utils/templates/ec2nodeclass.yaml.tpl @@ -0,0 +1,31 @@ +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: {{ .Name }} + labels: + kops.k8s.io/managed-by: kops-controller +spec: + amiFamily: Custom + amiSelectorTerms: + - name: {{ .AmiName }} + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: required + role: nodes.{{ .ClusterName }} + securityGroupSelectorTerms: + - name: nodes.{{ .ClusterName }} + - tags: + karpenter/owner: {{ .ClusterName }}/{{ .Name }} + subnetSelectorTerms: + - tags: + kops.k8s.io/instance-group/{{ .Name }}: '*' + kubernetes.io/cluster/{{ .ClusterName }}: '*' + tags: + k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node: '' + {{- range $key, $value := .Tags }} + {{ $key }}: {{ $value | quote }} + {{- end }} + userData: | +{{ .UserData | indent 4 }} \ No newline at end of file diff --git a/utils/templates/karpenter_custom_addon_boostrap.tf.tpl b/utils/templates/karpenter_custom_addon_boostrap.tf.tpl index 23d6178..4f2776e 100644 --- a/utils/templates/karpenter_custom_addon_boostrap.tf.tpl +++ b/utils/templates/karpenter_custom_addon_boostrap.tf.tpl @@ -6,26 +6,32 @@ metadata: name: addons spec: addons: - - name: karpenter-provisioners.wildlife.io + - name: karpenter-resources.wildlife.io version: 0.0.1 selector: - k8s-addon: karpenter-provisioners.wildlife.io - manifest: karpenter-provisioners.wildlife.io/provisioners.yaml + k8s-addon: karpenter-resources.wildlife.io + manifest: karpenter-resources.wildlife.io/resources.yaml manifestHash: "{{ .ManifestHash }}" prune: kinds: - group: karpenter.sh kind: Provisioner labelSelector: "kops.k8s.io/managed-by=kops-controller" + - group: karpenter.sh + kind: NodePool + labelSelector: "kops.k8s.io/managed-by=kops-controller" + - group: karpenter.k8s.aws + kind: EC2NodeClass + labelSelector: "kops.k8s.io/managed-by=kops-controller" EOF key = "{{ .ClusterName }}/custom-addons/addon.yaml" provider = aws.files } -resource "aws_s3_object" "custom-addon-karpenter-provisioners" { +resource "aws_s3_object" "custom-addon-karpenter-resources" { bucket = "{{ .Bucket }}" - content = file("${path.module}/data/aws_s3_object_karpenter_provisioners_content") - key = "{{ .ClusterName }}/custom-addons/karpenter-provisioners.wildlife.io/provisioners.yaml" + content = file("${path.module}/data/aws_s3_object_karpenter_resources_content") + key = "{{ .ClusterName }}/custom-addons/karpenter-resources.wildlife.io/resources.yaml" provider = aws.files } diff --git a/utils/templates/tests/data/karpenter_resource_output_node_pool.yaml b/utils/templates/tests/data/karpenter_resource_output_node_pool.yaml new file mode 100644 index 0000000..d115e8d --- /dev/null +++ b/utils/templates/tests/data/karpenter_resource_output_node_pool.yaml @@ -0,0 +1,87 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + creationTimestamp: null + name: placeholder-karpenter-resources + namespace: kube-system +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + creationTimestamp: null + labels: + kops.k8s.io/managed-by: kops-controller + name: test-node-pool +spec: + disruption: + consolidationPolicy: WhenUnderutilized + expireAfter: Never + template: + metadata: + labels: + kops.k8s.io/cluster: test-cluster.test.k8s.cluster + kops.k8s.io/cluster-name: test-cluster.test.k8s.cluster + kops.k8s.io/instance-group-name: test-ig + kops.k8s.io/instance-group-role: Node + kops.k8s.io/instancegroup: test-ig + kops.k8s.io/managed-by: kops-controller + spec: + kubelet: + kubeReserved: + cpu: 150m + ephemeral-storage: 1Gi + memory: 150Mi + systemReserved: + cpu: 150m + ephemeral-storage: 1Gi + memory: 200Mi + nodeClassRef: + name: test-ig + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: kubernetes.io/os + operator: In + values: + - linux + - key: node.kubernetes.io/instance-type + operator: In + values: + - m5.large + resources: {} + startupTaints: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized +status: {} +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: test-ig + labels: + kops.k8s.io/managed-by: kops-controller +spec: + amiFamily: Custom + amiSelectorTerms: + - name: ubuntu-v1 + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: required + role: nodes.test-cluster.test.k8s.cluster + securityGroupSelectorTerms: + - name: nodes.test-cluster.test.k8s.cluster + - tags: + karpenter/owner: test-cluster.test.k8s.cluster/test-ig + subnetSelectorTerms: + - tags: + kops.k8s.io/instance-group/test-ig: '*' + kubernetes.io/cluster/test-cluster.test.k8s.cluster: '*' + tags: + k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node: '' + userData: | + dummy content \ No newline at end of file diff --git a/utils/templates/tests/data/karpenter_resource_output_node_pool_and_provisioner.yaml b/utils/templates/tests/data/karpenter_resource_output_node_pool_and_provisioner.yaml new file mode 100644 index 0000000..ce6a44e --- /dev/null +++ b/utils/templates/tests/data/karpenter_resource_output_node_pool_and_provisioner.yaml @@ -0,0 +1,136 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + creationTimestamp: null + name: placeholder-karpenter-resources + namespace: kube-system +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + creationTimestamp: null + labels: + kops.k8s.io/managed-by: kops-controller + name: test-provisioner +spec: + consolidation: + enabled: true + kubeletConfiguration: + kubeReserved: + cpu: 150m + ephemeral-storage: 1Gi + memory: 150Mi + systemReserved: + cpu: 150m + ephemeral-storage: 1Gi + memory: 200Mi + labels: + kops.k8s.io/cluster: test-cluster.test.k8s.cluster + kops.k8s.io/cluster-name: test-cluster.test.k8s.cluster + kops.k8s.io/instance-group-name: test-ig + kops.k8s.io/instance-group-role: Node + kops.k8s.io/instancegroup: test-ig + kops.k8s.io/managed-by: kops-controller + provider: + launchTemplate: test-ig.test-cluster.test.k8s.cluster + subnetSelector: + kops.k8s.io/instance-group/test-ig: '*' + kubernetes.io/cluster/test-cluster.test.k8s.cluster: '*' + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: kubernetes.io/os + operator: In + values: + - linux + - key: node.kubernetes.io/instance-type + operator: In + values: + - m5.large + startupTaints: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized +status: {} +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + creationTimestamp: null + labels: + kops.k8s.io/managed-by: kops-controller + name: test-node-pool +spec: + disruption: + consolidationPolicy: WhenUnderutilized + expireAfter: Never + template: + metadata: + labels: + kops.k8s.io/cluster: test-cluster.test.k8s.cluster + kops.k8s.io/cluster-name: test-cluster.test.k8s.cluster + kops.k8s.io/instance-group-name: test-ig + kops.k8s.io/instance-group-role: Node + kops.k8s.io/instancegroup: test-ig + kops.k8s.io/managed-by: kops-controller + spec: + kubelet: + kubeReserved: + cpu: 150m + ephemeral-storage: 1Gi + memory: 150Mi + systemReserved: + cpu: 150m + ephemeral-storage: 1Gi + memory: 200Mi + nodeClassRef: + name: test-ig + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: kubernetes.io/os + operator: In + values: + - linux + - key: node.kubernetes.io/instance-type + operator: In + values: + - m5.large + resources: {} + startupTaints: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized +status: {} +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: test-ig + labels: + kops.k8s.io/managed-by: kops-controller +spec: + amiFamily: Custom + amiSelectorTerms: + - name: ubuntu-v1 + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: required + role: nodes.test-cluster.test.k8s.cluster + securityGroupSelectorTerms: + - name: nodes.test-cluster.test.k8s.cluster + - tags: + karpenter/owner: test-cluster.test.k8s.cluster/test-ig + subnetSelectorTerms: + - tags: + kops.k8s.io/instance-group/test-ig: '*' + kubernetes.io/cluster/test-cluster.test.k8s.cluster: '*' + tags: + k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node: '' + userData: | + dummy content \ No newline at end of file diff --git a/utils/templates/tests/data/aws_s3_object_karpenter_provisioners_content b/utils/templates/tests/data/karpenter_resource_output_provisioner.yaml similarity index 96% rename from utils/templates/tests/data/aws_s3_object_karpenter_provisioners_content rename to utils/templates/tests/data/karpenter_resource_output_provisioner.yaml index da78b32..b365e55 100644 --- a/utils/templates/tests/data/aws_s3_object_karpenter_provisioners_content +++ b/utils/templates/tests/data/karpenter_resource_output_provisioner.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: creationTimestamp: null - name: placeholder-karpenter-provisioners + name: placeholder-karpenter-resources namespace: kube-system --- apiVersion: karpenter.sh/v1alpha5 diff --git a/utils/templates/tests/karpenter_custom_addon_boostrap.tf b/utils/templates/tests/karpenter_custom_addon_boostrap.tf index 04e69c7..af8266f 100644 --- a/utils/templates/tests/karpenter_custom_addon_boostrap.tf +++ b/utils/templates/tests/karpenter_custom_addon_boostrap.tf @@ -6,26 +6,32 @@ metadata: name: addons spec: addons: - - name: karpenter-provisioners.wildlife.io + - name: karpenter-resources.wildlife.io version: 0.0.1 selector: - k8s-addon: karpenter-provisioners.wildlife.io - manifest: karpenter-provisioners.wildlife.io/provisioners.yaml - manifestHash: "b3d32a0baf397c2f500b8936bad9e0581c6b234b7549d5aa624bc383722daaae" + k8s-addon: karpenter-resources.wildlife.io + manifest: karpenter-resources.wildlife.io/resources.yaml + manifestHash: "{{ .ManifestHash }}" prune: kinds: - group: karpenter.sh kind: Provisioner labelSelector: "kops.k8s.io/managed-by=kops-controller" + - group: karpenter.sh + kind: NodePool + labelSelector: "kops.k8s.io/managed-by=kops-controller" + - group: karpenter.k8s.aws + kind: EC2NodeClass + labelSelector: "kops.k8s.io/managed-by=kops-controller" EOF key = "test-cluster.test.k8s.cluster/custom-addons/addon.yaml" provider = aws.files } -resource "aws_s3_object" "custom-addon-karpenter-provisioners" { +resource "aws_s3_object" "custom-addon-karpenter-resources" { bucket = "tests" - content = file("${path.module}/data/aws_s3_object_karpenter_provisioners_content") - key = "test-cluster.test.k8s.cluster/custom-addons/karpenter-provisioners.wildlife.io/provisioners.yaml" + content = file("${path.module}/data/aws_s3_object_karpenter_resources_content") + key = "test-cluster.test.k8s.cluster/custom-addons/karpenter-resources.wildlife.io/resources.yaml" provider = aws.files } diff --git a/utils/terraform_utils.go b/utils/terraform_utils.go index 4aacc98..f7a20cb 100644 --- a/utils/terraform_utils.go +++ b/utils/terraform_utils.go @@ -21,13 +21,13 @@ type Template struct { } //go:embed templates/*.tpl -var terraformTemplates embed.FS +var templates embed.FS // CreateTerraformFileFromTemplate populates a Terraform template and create files in the state func CreateTerraformFilesFromTemplate(terraformTemplateFilePath string, TerraformOutputFileName string, terraformOutputDir string, templateData any) error { template := Template{ TemplateFilename: terraformTemplateFilePath, - EmbeddedFiles: terraformTemplates, + EmbeddedFiles: templates, OutputFilename: fmt.Sprintf("%s/%s", terraformOutputDir, TerraformOutputFileName), Data: templateData, } diff --git a/utils/terraform_utils_test.go b/utils/terraform_utils_test.go index c878b5e..979ab41 100644 --- a/utils/terraform_utils_test.go +++ b/utils/terraform_utils_test.go @@ -13,7 +13,7 @@ import ( var ( //go:embed fixtures/*.tpl - templates embed.FS + testTemplates embed.FS ) func TestCreateAdditionalTerraformFiles(t *testing.T) { @@ -38,7 +38,7 @@ func TestCreateAdditionalTerraformFiles(t *testing.T) { { TemplateFilename: "fixtures/test_template.tpl", OutputFilename: fmt.Sprintf("%s/test_output", tmpDir), - EmbeddedFiles: templates, + EmbeddedFiles: testTemplates, Data: "test.test.us-east-1.k8s.tfgco.com", }, }, @@ -59,13 +59,13 @@ func TestCreateAdditionalTerraformFiles(t *testing.T) { { TemplateFilename: "fixtures/test_multiple_template_a.tpl", OutputFilename: fmt.Sprintf("%s/test_output_A", tmpDir), - EmbeddedFiles: templates, + EmbeddedFiles: testTemplates, Data: "test.test.us-east-1.k8s.tfgco.com", }, { TemplateFilename: "fixtures/test_multiple_template_b.tpl", OutputFilename: fmt.Sprintf("%s/test_output_B", tmpDir), - EmbeddedFiles: templates, + EmbeddedFiles: testTemplates, Data: "test.test.us-east-1.k8s.tfgco.com", }, }, @@ -90,7 +90,7 @@ func TestCreateAdditionalTerraformFiles(t *testing.T) { { TemplateFilename: "fixtures/test_template.tpl", OutputFilename: fmt.Sprintf("%s/invalid-directory/test_output", tmpDir), - EmbeddedFiles: templates, + EmbeddedFiles: testTemplates, Data: "test.test.us-east-1.k8s.tfgco.com", }, }, @@ -102,7 +102,7 @@ func TestCreateAdditionalTerraformFiles(t *testing.T) { { TemplateFilename: "fixtures/inexisting_template.tpl", OutputFilename: fmt.Sprintf("%s/test_output", tmpDir), - EmbeddedFiles: templates, + EmbeddedFiles: testTemplates, Data: "test.test.us-east-1.k8s.tfgco.com", }, }, @@ -114,7 +114,7 @@ func TestCreateAdditionalTerraformFiles(t *testing.T) { { TemplateFilename: "fixtures/test_invalid_template.tpl", OutputFilename: fmt.Sprintf("%s/test_output", tmpDir), - EmbeddedFiles: templates, + EmbeddedFiles: testTemplates, Data: "test.test.us-east-1.k8s.tfgco.com", }, },