feat: add tags to the securityGroups with the infraRef information

topfreegames · Aug 28, 2024 · 669fa4b · 669fa4b
1 parent 51ec818
commit 669fa4b
Show file tree

Hide file tree

Showing 6 changed files with 243 additions and 26 deletions.
diff --git a/go.mod b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.11.1
 	github.com/aws/aws-sdk-go-v2/service/autoscaling v1.28.3
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.26.0
+	github.com/aws/karpenter-core v0.29.0
 	github.com/crossplane-contrib/provider-aws v0.35.0-rc.0.0.20221220103156-b59bf4223ff4
 	github.com/crossplane/crossplane-runtime v0.19.0-rc.0.0.20220930073209-84e629b95898
 	github.com/go-logr/logr v1.2.4
@@ -30,7 +31,6 @@ require (
 )
 
 require (
-	github.com/aws/karpenter-core v0.29.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect

diff --git a/internal/controller/ec2.aws/securitygroup_controller.go b/internal/controller/ec2.aws/securitygroup_controller.go
@@ -31,6 +31,7 @@ import (
 	"github.com/topfreegames/kubernetes-crossplane-infrastructure-operator/pkg/aws/ec2"
 	"github.com/topfreegames/kubernetes-crossplane-infrastructure-operator/pkg/crossplane"
 	kopsutils "github.com/topfreegames/kubernetes-crossplane-infrastructure-operator/pkg/kops"
+
 	kcontrolplanev1alpha1 "github.com/topfreegames/kubernetes-kops-operator/apis/controlplane/v1alpha1"
 	kinfrastructurev1alpha1 "github.com/topfreegames/kubernetes-kops-operator/apis/infrastructure/v1alpha1"
 	"github.com/topfreegames/kubernetes-kops-operator/pkg/kops"
@@ -95,6 +96,7 @@ type SecurityGroupReconciliation struct {
 	region             *string
 	vpcId              *string
 	providerConfigName string
+	clusterName        string
 }
 
 func DefaultReconciler(mgr manager.Manager) *SecurityGroupReconciler {
@@ -241,6 +243,7 @@ func (r *SecurityGroupReconciliation) retrieveInfraRefInfo(ctx context.Context)
 			r.providerConfigName = *providerConfigName
 			r.region = region
 			r.vpcId = vpcId
+			r.clusterName = kmp.Spec.ClusterName
 
 			return nil
 		case "KopsControlPlane":
@@ -253,6 +256,7 @@ func (r *SecurityGroupReconciliation) retrieveInfraRefInfo(ctx context.Context)
 			r.providerConfigName = *providerConfigName
 			r.region = region
 			r.vpcId = vpcId
+			r.clusterName = infrastructureRef.Name
 
 			return nil
 		default:
@@ -270,7 +274,7 @@ func (r *SecurityGroupReconciliation) reconcileNormal(ctx context.Context) (ctrl
 		controllerutil.AddFinalizer(r.sg, securityGroupFinalizer)
 	}
 
-	csg, err := crossplane.CreateOrUpdateCrossplaneSecurityGroup(ctx, r.Client, r.vpcId, r.region, r.providerConfigName, r.sg)
+	csg, err := crossplane.CreateOrUpdateCrossplaneSecurityGroup(ctx, r.Client, r.vpcId, r.region, r.providerConfigName, r.clusterName, r.sg)
 	if err != nil {
 		conditions.MarkFalse(r.sg,
 			securitygroupv1alpha2.CrossplaneResourceReadyCondition,
@@ -341,18 +345,45 @@ func (r *SecurityGroupReconciliation) attachKopsMachinePool(ctx context.Context,
 				continue
 			}
 		} else if kmp.Spec.KopsInstanceGroupSpec.Manager == "Karpenter" {
-			launchTemplateName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp)
-			if err != nil {
-				attachErr = multierror.Append(attachErr, err)
-				continue
-			}
+			if len(kmp.Spec.KarpenterProvisioners) > 0 {
+				launchTemplateName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp)
+				if err != nil {
+					attachErr = multierror.Append(attachErr, err)
+					continue
+				}
 
-			err = r.attachSGToLaunchTemplate(ctx, kcp.Name, launchTemplateName, csg.Status.AtProvider.SecurityGroupID)
-			if err != nil {
-				r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error())
-				attachErr = multierror.Append(attachErr, err)
-				continue
+				err = r.attachSGToLaunchTemplate(ctx, kcp.Name, launchTemplateName, csg.Status.AtProvider.SecurityGroupID)
+				if err != nil {
+					r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error())
+					attachErr = multierror.Append(attachErr, err)
+					continue
+				}
+			} else {
+				// We are assuming that this is a NodePool for now
+				filters := []ec2types.Filter{
+					{
+						Name:   aws.String("tag:karpenter.sh/managed-by"),
+						Values: []string{r.clusterName},
+					},
+					{
+						Name:   aws.String("tag:karpenter.sh/nodepool"),
+						Values: []string{kmp.Name},
+					},
+				}
+
+				instanceIDs, err := ec2.GetInstancesWithFilter(ctx, r.ec2Client, filters)
+				if err != nil {
+					r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error())
+					attachErr = multierror.Append(attachErr, err)
+					continue
+				}
+
+				err = ec2.AttachSecurityGroupToInstances(ctx, r.ec2Client, instanceIDs, csg.Status.AtProvider.SecurityGroupID)
+				if err != nil {
+					r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, "SecurityGroupInstancesAttachmentFailed", err.Error())
+				}
 			}
+
 		} else {
 			asgName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp)
 			if err != nil {
@@ -643,17 +674,43 @@ func (r *SecurityGroupReconciliation) detachSGFromKopsMachinePool(ctx context.Co
 				continue
 			}
 		} else if kmp.Spec.KopsInstanceGroupSpec.Manager == "Karpenter" {
-			launchTemplateName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp)
-			if err != nil {
-				detachErr = multierror.Append(detachErr, err)
-				continue
-			}
+			if len(kmp.Spec.KarpenterProvisioners) > 0 {
 
-			err = r.detachSGFromLaunchTemplate(ctx, kmp.Spec.ClusterName, launchTemplateName, csg.Status.AtProvider.SecurityGroupID)
-			if err != nil {
-				r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error())
-				detachErr = multierror.Append(detachErr, err)
-				continue
+				launchTemplateName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp)
+				if err != nil {
+					detachErr = multierror.Append(detachErr, err)
+					continue
+				}
+
+				err = r.detachSGFromLaunchTemplate(ctx, kmp.Spec.ClusterName, launchTemplateName, csg.Status.AtProvider.SecurityGroupID)
+				if err != nil {
+					r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error())
+					detachErr = multierror.Append(detachErr, err)
+					continue
+				}
+			} else {
+				// We are assuming that this is a NodePool for now
+				filters := []ec2types.Filter{
+					{
+						Name:   aws.String("tag:karpenter.sh/managed-by"),
+						Values: []string{r.clusterName},
+					},
+					{
+						Name:   aws.String("tag:karpenter.sh/nodepool"),
+						Values: []string{kmp.Name},
+					},
+				}
+
+				instanceIDs, err := ec2.GetInstancesWithFilter(ctx, r.ec2Client, filters)
+				if err != nil {
+					r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error())
+					detachErr = multierror.Append(detachErr, err)
+					continue
+				}
+				err = ec2.DetachSecurityGroupFromInstances(ctx, r.ec2Client, instanceIDs, csg.Status.AtProvider.SecurityGroupID)
+				if err != nil {
+					r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, "SecurityGroupInstancesDetachmentFailed", err.Error())
+				}
 			}
 		} else {
 			asgName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp)

diff --git a/internal/controller/ec2.aws/securitygroup_controller_test.go b/internal/controller/ec2.aws/securitygroup_controller_test.go
@@ -23,6 +23,8 @@ import (
 	"testing"
 	"time"
 
+	karpenter "github.com/aws/karpenter-core/pkg/apis/v1alpha5"
+
 	"github.com/aws/aws-sdk-go-v2/aws"
 	awsautoscaling "github.com/aws/aws-sdk-go-v2/service/autoscaling"
 	autoscalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
@@ -293,6 +295,13 @@ var (
 		},
 		Spec: kinfrastructurev1alpha1.KopsMachinePoolSpec{
 			ClusterName: "test-cluster",
+			KarpenterProvisioners: []karpenter.Provisioner{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "test-provisioner",
+					},
+				},
+			},
 			KopsInstanceGroupSpec: kopsapi.InstanceGroupSpec{
 				Manager: "Karpenter",
 				NodeLabels: map[string]string{
@@ -1853,6 +1862,13 @@ func TestAttachKopsMachinePool(t *testing.T) {
 				},
 				Spec: kinfrastructurev1alpha1.KopsMachinePoolSpec{
 					ClusterName: "test-cluster",
+					KarpenterProvisioners: []karpenter.Provisioner{
+						{
+							ObjectMeta: metav1.ObjectMeta{
+								Name: "test-provisioner",
+							},
+						},
+					},
 					KopsInstanceGroupSpec: kopsapi.InstanceGroupSpec{
 						Manager: "Karpenter",
 						NodeLabels: map[string]string{

diff --git a/pkg/aws/ec2/vpc.go b/pkg/aws/ec2/vpc.go
@@ -368,3 +368,27 @@ func GetLaunchTemplateFromInstanceGroup(ctx context.Context, ec2Client EC2Client
 
 	return &launchTemplates[0], nil
 }
+func GetInstancesWithFilter(ctx context.Context, ec2Client EC2Client, filters []ec2types.Filter) ([]string, error) {
+
+	input := &ec2.DescribeInstancesInput{
+		Filters: filters,
+	}
+
+	output, err := ec2Client.DescribeInstances(ctx, input)
+	if err != nil {
+		return nil, fmt.Errorf("failed to describe instances: %w", err)
+	}
+
+	var instanceIDs []string
+	for _, reservation := range output.Reservations {
+		for _, instance := range reservation.Instances {
+			instanceIDs = append(instanceIDs, *instance.InstanceId)
+		}
+	}
+
+	if len(instanceIDs) == 0 {
+		return nil, fmt.Errorf("no instances found with filter")
+	}
+
+	return instanceIDs, nil
+}
diff --git a/pkg/crossplane/crossplane.go b/pkg/crossplane/crossplane.go
@@ -7,6 +7,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	clustermeshv1alpha1 "github.com/topfreegames/kubernetes-crossplane-infrastructure-operator/api/clustermesh.infrastructure/v1alpha1"
+	"github.com/topfreegames/kubernetes-kops-operator/pkg/kops"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	crossec2v1alphav1 "github.com/crossplane-contrib/provider-aws/apis/ec2/v1alpha1"
@@ -76,6 +77,12 @@ func NewCrossplaneSecurityGroup(sg *securitygroupv1alpha2.SecurityGroup, vpcId,
 				Ingress:     []crossec2v1beta1.IPPermission{},
 				VPCID:       vpcId,
 				Region:      region,
+				Tags: []crossec2v1beta1.Tag{
+					{
+						Key:   "ManagedBy",
+						Value: "kubernetes-crossplane-infrastructure-operator",
+					},
+				},
 			},
 			ResourceSpec: crossplanev1.ResourceSpec{
 				ProviderConfigReference: &crossplanev1.Reference{
@@ -125,7 +132,7 @@ func NewCrossplaneRoute(region, destinationCIDRBlock, routeTable string, provide
 	return croute
 }
 
-func CreateOrUpdateCrossplaneSecurityGroup(ctx context.Context, kubeClient client.Client, vpcId, region *string, providerConfigName string, sg *securitygroupv1alpha2.SecurityGroup) (*crossec2v1beta1.SecurityGroup, error) {
+func CreateOrUpdateCrossplaneSecurityGroup(ctx context.Context, kubeClient client.Client, vpcId, region *string, providerConfigName, clusterName string, sg *securitygroupv1alpha2.SecurityGroup) (*crossec2v1beta1.SecurityGroup, error) {
 	csg := NewCrossplaneSecurityGroup(sg, vpcId, region, providerConfigName)
 	_, err := controllerutil.CreateOrUpdate(ctx, kubeClient, csg, func() error {
 		var ingressRules []crossec2v1beta1.IPPermission
@@ -154,6 +161,38 @@ func CreateOrUpdateCrossplaneSecurityGroup(ctx context.Context, kubeClient clien
 			ipPermission.IPRanges = allowedCIDRBlocks
 			ingressRules = append(ingressRules, ipPermission)
 		}
+
+		for _, infraRef := range sg.Spec.InfrastructureRef {
+			switch infraRef.Kind {
+			case "KopsMachinePool":
+				if checkTagAlreadyExists(csg.Spec.ForProvider.Tags, fmt.Sprintf("kops.k8s.io/instance-group/%s", infraRef.Name)) {
+					continue
+				}
+				csg.Spec.ForProvider.Tags = append(csg.Spec.ForProvider.Tags, crossec2v1beta1.Tag{
+					Key:   fmt.Sprintf("kops.k8s.io/instance-group/%s", infraRef.Name),
+					Value: "owned",
+				})
+			case "KopsControlPlane":
+				kmps, err := kops.GetKopsMachinePoolsWithLabel(ctx, kubeClient, "cluster.x-k8s.io/cluster-name", clusterName)
+				if err != nil {
+					return err
+				}
+
+				for _, kmp := range kmps {
+					if checkTagAlreadyExists(csg.Spec.ForProvider.Tags, fmt.Sprintf("kops.k8s.io/instance-group/%s", kmp.Name)) {
+						continue
+					}
+					csg.Spec.ForProvider.Tags = append(csg.Spec.ForProvider.Tags, crossec2v1beta1.Tag{
+						Key:   fmt.Sprintf("kops.k8s.io/instance-group/%s", kmp.Name),
+						Value: "owned",
+					})
+				}
+			default:
+				continue
+			}
+
+		}
+
 		csg.Spec.ForProvider.Ingress = ingressRules
 		csg.Annotations = sg.Annotations
 		csg.Spec.ResourceSpec.ProviderConfigReference = &crossplanev1.Reference{Name: providerConfigName}
@@ -374,3 +413,12 @@ func GetOwnedRoutesRef(ctx context.Context, owner client.Object, kubeclient clie
 
 	return ss, nil
 }
+
+func checkTagAlreadyExists(tags []crossec2v1beta1.Tag, key string) bool {
+	for _, tag := range tags {
+		if tag.Key == key {
+			return true
+		}
+	}
+	return false
+}