diff --git a/go.mod b/go.mod index 20802c8..b949ffd 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.11.1 github.com/aws/aws-sdk-go-v2/service/autoscaling v1.28.3 github.com/aws/aws-sdk-go-v2/service/ec2 v1.26.0 + github.com/aws/karpenter-core v0.29.0 github.com/crossplane-contrib/provider-aws v0.35.0-rc.0.0.20221220103156-b59bf4223ff4 github.com/crossplane/crossplane-runtime v0.19.0-rc.0.0.20220930073209-84e629b95898 github.com/go-logr/logr v1.2.4 @@ -30,7 +31,6 @@ require ( ) require ( - github.com/aws/karpenter-core v0.29.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect diff --git a/internal/controller/ec2.aws/securitygroup_controller.go b/internal/controller/ec2.aws/securitygroup_controller.go index 6ff6c3d..06311ee 100644 --- a/internal/controller/ec2.aws/securitygroup_controller.go +++ b/internal/controller/ec2.aws/securitygroup_controller.go @@ -31,6 +31,7 @@ import ( "github.com/topfreegames/kubernetes-crossplane-infrastructure-operator/pkg/aws/ec2" "github.com/topfreegames/kubernetes-crossplane-infrastructure-operator/pkg/crossplane" kopsutils "github.com/topfreegames/kubernetes-crossplane-infrastructure-operator/pkg/kops" + kcontrolplanev1alpha1 "github.com/topfreegames/kubernetes-kops-operator/apis/controlplane/v1alpha1" kinfrastructurev1alpha1 "github.com/topfreegames/kubernetes-kops-operator/apis/infrastructure/v1alpha1" "github.com/topfreegames/kubernetes-kops-operator/pkg/kops" @@ -95,6 +96,7 @@ type SecurityGroupReconciliation struct { region *string vpcId *string providerConfigName string + clusterName string } func DefaultReconciler(mgr manager.Manager) *SecurityGroupReconciler { @@ -241,6 +243,7 @@ func (r *SecurityGroupReconciliation) retrieveInfraRefInfo(ctx context.Context) r.providerConfigName = *providerConfigName r.region = region r.vpcId = vpcId + r.clusterName = kmp.Spec.ClusterName return nil case "KopsControlPlane": @@ -253,6 +256,7 @@ func (r *SecurityGroupReconciliation) retrieveInfraRefInfo(ctx context.Context) r.providerConfigName = *providerConfigName r.region = region r.vpcId = vpcId + r.clusterName = infrastructureRef.Name return nil default: @@ -270,7 +274,7 @@ func (r *SecurityGroupReconciliation) reconcileNormal(ctx context.Context) (ctrl controllerutil.AddFinalizer(r.sg, securityGroupFinalizer) } - csg, err := crossplane.CreateOrUpdateCrossplaneSecurityGroup(ctx, r.Client, r.vpcId, r.region, r.providerConfigName, r.sg) + csg, err := crossplane.CreateOrUpdateCrossplaneSecurityGroup(ctx, r.Client, r.vpcId, r.region, r.providerConfigName, r.clusterName, r.sg) if err != nil { conditions.MarkFalse(r.sg, securitygroupv1alpha2.CrossplaneResourceReadyCondition, @@ -341,18 +345,45 @@ func (r *SecurityGroupReconciliation) attachKopsMachinePool(ctx context.Context, continue } } else if kmp.Spec.KopsInstanceGroupSpec.Manager == "Karpenter" { - launchTemplateName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp) - if err != nil { - attachErr = multierror.Append(attachErr, err) - continue - } + if len(kmp.Spec.KarpenterProvisioners) > 0 { + launchTemplateName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp) + if err != nil { + attachErr = multierror.Append(attachErr, err) + continue + } - err = r.attachSGToLaunchTemplate(ctx, kcp.Name, launchTemplateName, csg.Status.AtProvider.SecurityGroupID) - if err != nil { - r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error()) - attachErr = multierror.Append(attachErr, err) - continue + err = r.attachSGToLaunchTemplate(ctx, kcp.Name, launchTemplateName, csg.Status.AtProvider.SecurityGroupID) + if err != nil { + r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error()) + attachErr = multierror.Append(attachErr, err) + continue + } + } else { + // We are assuming that this is a NodePool for now + filters := []ec2types.Filter{ + { + Name: aws.String("tag:karpenter.sh/managed-by"), + Values: []string{r.clusterName}, + }, + { + Name: aws.String("tag:karpenter.sh/nodepool"), + Values: []string{kmp.Name}, + }, + } + + instanceIDs, err := ec2.GetInstancesWithFilter(ctx, r.ec2Client, filters) + if err != nil { + r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error()) + attachErr = multierror.Append(attachErr, err) + continue + } + + err = ec2.AttachSecurityGroupToInstances(ctx, r.ec2Client, instanceIDs, csg.Status.AtProvider.SecurityGroupID) + if err != nil { + r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, "SecurityGroupInstancesAttachmentFailed", err.Error()) + } } + } else { asgName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp) if err != nil { @@ -643,17 +674,43 @@ func (r *SecurityGroupReconciliation) detachSGFromKopsMachinePool(ctx context.Co continue } } else if kmp.Spec.KopsInstanceGroupSpec.Manager == "Karpenter" { - launchTemplateName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp) - if err != nil { - detachErr = multierror.Append(detachErr, err) - continue - } + if len(kmp.Spec.KarpenterProvisioners) > 0 { - err = r.detachSGFromLaunchTemplate(ctx, kmp.Spec.ClusterName, launchTemplateName, csg.Status.AtProvider.SecurityGroupID) - if err != nil { - r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error()) - detachErr = multierror.Append(detachErr, err) - continue + launchTemplateName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp) + if err != nil { + detachErr = multierror.Append(detachErr, err) + continue + } + + err = r.detachSGFromLaunchTemplate(ctx, kmp.Spec.ClusterName, launchTemplateName, csg.Status.AtProvider.SecurityGroupID) + if err != nil { + r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error()) + detachErr = multierror.Append(detachErr, err) + continue + } + } else { + // We are assuming that this is a NodePool for now + filters := []ec2types.Filter{ + { + Name: aws.String("tag:karpenter.sh/managed-by"), + Values: []string{r.clusterName}, + }, + { + Name: aws.String("tag:karpenter.sh/nodepool"), + Values: []string{kmp.Name}, + }, + } + + instanceIDs, err := ec2.GetInstancesWithFilter(ctx, r.ec2Client, filters) + if err != nil { + r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, securitygroupv1alpha2.SecurityGroupAttachmentFailedReason, err.Error()) + detachErr = multierror.Append(detachErr, err) + continue + } + err = ec2.DetachSecurityGroupFromInstances(ctx, r.ec2Client, instanceIDs, csg.Status.AtProvider.SecurityGroupID) + if err != nil { + r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, "SecurityGroupInstancesDetachmentFailed", err.Error()) + } } } else { asgName, err := kops.GetCloudResourceNameFromKopsMachinePool(kmp) diff --git a/internal/controller/ec2.aws/securitygroup_controller_test.go b/internal/controller/ec2.aws/securitygroup_controller_test.go index 7f0e717..f505f09 100644 --- a/internal/controller/ec2.aws/securitygroup_controller_test.go +++ b/internal/controller/ec2.aws/securitygroup_controller_test.go @@ -23,6 +23,8 @@ import ( "testing" "time" + karpenter "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/aws-sdk-go-v2/aws" awsautoscaling "github.com/aws/aws-sdk-go-v2/service/autoscaling" autoscalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" @@ -293,6 +295,13 @@ var ( }, Spec: kinfrastructurev1alpha1.KopsMachinePoolSpec{ ClusterName: "test-cluster", + KarpenterProvisioners: []karpenter.Provisioner{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-provisioner", + }, + }, + }, KopsInstanceGroupSpec: kopsapi.InstanceGroupSpec{ Manager: "Karpenter", NodeLabels: map[string]string{ @@ -1853,6 +1862,13 @@ func TestAttachKopsMachinePool(t *testing.T) { }, Spec: kinfrastructurev1alpha1.KopsMachinePoolSpec{ ClusterName: "test-cluster", + KarpenterProvisioners: []karpenter.Provisioner{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-provisioner", + }, + }, + }, KopsInstanceGroupSpec: kopsapi.InstanceGroupSpec{ Manager: "Karpenter", NodeLabels: map[string]string{ diff --git a/pkg/aws/ec2/vpc.go b/pkg/aws/ec2/vpc.go index d05dcdf..ea749dc 100644 --- a/pkg/aws/ec2/vpc.go +++ b/pkg/aws/ec2/vpc.go @@ -368,3 +368,27 @@ func GetLaunchTemplateFromInstanceGroup(ctx context.Context, ec2Client EC2Client return &launchTemplates[0], nil } +func GetInstancesWithFilter(ctx context.Context, ec2Client EC2Client, filters []ec2types.Filter) ([]string, error) { + + input := &ec2.DescribeInstancesInput{ + Filters: filters, + } + + output, err := ec2Client.DescribeInstances(ctx, input) + if err != nil { + return nil, fmt.Errorf("failed to describe instances: %w", err) + } + + var instanceIDs []string + for _, reservation := range output.Reservations { + for _, instance := range reservation.Instances { + instanceIDs = append(instanceIDs, *instance.InstanceId) + } + } + + if len(instanceIDs) == 0 { + return nil, fmt.Errorf("no instances found with filter") + } + + return instanceIDs, nil +} diff --git a/pkg/crossplane/crossplane.go b/pkg/crossplane/crossplane.go index 4ed4429..e11e2ea 100644 --- a/pkg/crossplane/crossplane.go +++ b/pkg/crossplane/crossplane.go @@ -7,6 +7,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" clustermeshv1alpha1 "github.com/topfreegames/kubernetes-crossplane-infrastructure-operator/api/clustermesh.infrastructure/v1alpha1" + "github.com/topfreegames/kubernetes-kops-operator/pkg/kops" "github.com/aws/aws-sdk-go-v2/aws" crossec2v1alphav1 "github.com/crossplane-contrib/provider-aws/apis/ec2/v1alpha1" @@ -76,6 +77,12 @@ func NewCrossplaneSecurityGroup(sg *securitygroupv1alpha2.SecurityGroup, vpcId, Ingress: []crossec2v1beta1.IPPermission{}, VPCID: vpcId, Region: region, + Tags: []crossec2v1beta1.Tag{ + { + Key: "ManagedBy", + Value: "kubernetes-crossplane-infrastructure-operator", + }, + }, }, ResourceSpec: crossplanev1.ResourceSpec{ ProviderConfigReference: &crossplanev1.Reference{ @@ -125,7 +132,7 @@ func NewCrossplaneRoute(region, destinationCIDRBlock, routeTable string, provide return croute } -func CreateOrUpdateCrossplaneSecurityGroup(ctx context.Context, kubeClient client.Client, vpcId, region *string, providerConfigName string, sg *securitygroupv1alpha2.SecurityGroup) (*crossec2v1beta1.SecurityGroup, error) { +func CreateOrUpdateCrossplaneSecurityGroup(ctx context.Context, kubeClient client.Client, vpcId, region *string, providerConfigName, clusterName string, sg *securitygroupv1alpha2.SecurityGroup) (*crossec2v1beta1.SecurityGroup, error) { csg := NewCrossplaneSecurityGroup(sg, vpcId, region, providerConfigName) _, err := controllerutil.CreateOrUpdate(ctx, kubeClient, csg, func() error { var ingressRules []crossec2v1beta1.IPPermission @@ -154,6 +161,38 @@ func CreateOrUpdateCrossplaneSecurityGroup(ctx context.Context, kubeClient clien ipPermission.IPRanges = allowedCIDRBlocks ingressRules = append(ingressRules, ipPermission) } + + for _, infraRef := range sg.Spec.InfrastructureRef { + switch infraRef.Kind { + case "KopsMachinePool": + if checkTagAlreadyExists(csg.Spec.ForProvider.Tags, fmt.Sprintf("kops.k8s.io/instance-group/%s", infraRef.Name)) { + continue + } + csg.Spec.ForProvider.Tags = append(csg.Spec.ForProvider.Tags, crossec2v1beta1.Tag{ + Key: fmt.Sprintf("kops.k8s.io/instance-group/%s", infraRef.Name), + Value: "owned", + }) + case "KopsControlPlane": + kmps, err := kops.GetKopsMachinePoolsWithLabel(ctx, kubeClient, "cluster.x-k8s.io/cluster-name", clusterName) + if err != nil { + return err + } + + for _, kmp := range kmps { + if checkTagAlreadyExists(csg.Spec.ForProvider.Tags, fmt.Sprintf("kops.k8s.io/instance-group/%s", kmp.Name)) { + continue + } + csg.Spec.ForProvider.Tags = append(csg.Spec.ForProvider.Tags, crossec2v1beta1.Tag{ + Key: fmt.Sprintf("kops.k8s.io/instance-group/%s", kmp.Name), + Value: "owned", + }) + } + default: + continue + } + + } + csg.Spec.ForProvider.Ingress = ingressRules csg.Annotations = sg.Annotations csg.Spec.ResourceSpec.ProviderConfigReference = &crossplanev1.Reference{Name: providerConfigName} @@ -374,3 +413,12 @@ func GetOwnedRoutesRef(ctx context.Context, owner client.Object, kubeclient clie return ss, nil } + +func checkTagAlreadyExists(tags []crossec2v1beta1.Tag, key string) bool { + for _, tag := range tags { + if tag.Key == key { + return true + } + } + return false +} diff --git a/pkg/crossplane/crossplane_test.go b/pkg/crossplane/crossplane_test.go index 2566b57..97a0b54 100644 --- a/pkg/crossplane/crossplane_test.go +++ b/pkg/crossplane/crossplane_test.go @@ -6,6 +6,8 @@ import ( "reflect" "testing" + kinfrastructurev1alpha1 "github.com/topfreegames/kubernetes-kops-operator/apis/infrastructure/v1alpha1" + "github.com/aws/aws-sdk-go-v2/aws" crossec2v1alphav1 "github.com/crossplane-contrib/provider-aws/apis/ec2/v1alpha1" "github.com/google/go-cmp/cmp" @@ -1469,6 +1471,13 @@ func TestCreateOrUpdateCrossplaneSecurityGroup(t *testing.T) { }, }, }, + InfrastructureRef: []*corev1.ObjectReference{ + { + Kind: "KopsMachinePool", + Name: "testIG", + Namespace: "testNamespace", + }, + }, }, }, validateOutput: func(_ *securitygroupv1alpha2.SecurityGroup, csg *crossec2v1beta1.SecurityGroup) bool { @@ -1478,7 +1487,19 @@ func TestCreateOrUpdateCrossplaneSecurityGroup(t *testing.T) { if csg.Name != "test-sg" { return false } - return true + + expectedTags := []crossec2v1beta1.Tag{ + { + Key: "ManagedBy", + Value: "kubernetes-crossplane-infrastructure-operator", + }, + { + Key: "kops.k8s.io/instance-group/testIG", + Value: "owned", + }, + } + + return reflect.DeepEqual(csg.Spec.ForProvider.Tags, expectedTags) }, }, { @@ -1494,6 +1515,30 @@ func TestCreateOrUpdateCrossplaneSecurityGroup(t *testing.T) { Description: "test-sg", GroupName: "test-sg", VPCID: &testVPCId, + Tags: []crossec2v1beta1.Tag{ + { + Key: "ManagedBy", + Value: "kubernetes-crossplane-infrastructure-operator", + }, + }, + }, + }, + }, + &kinfrastructurev1alpha1.KopsMachinePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testIGA", + Namespace: "testNamespace", + Labels: map[string]string{ + "cluster.x-k8s.io/cluster-name": "testCluster", + }, + }, + }, + &kinfrastructurev1alpha1.KopsMachinePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testIGB", + Namespace: "testNamespace", + Labels: map[string]string{ + "cluster.x-k8s.io/cluster-name": "testCluster", }, }, }, @@ -1524,6 +1569,13 @@ func TestCreateOrUpdateCrossplaneSecurityGroup(t *testing.T) { }, }, }, + InfrastructureRef: []*corev1.ObjectReference{ + { + Kind: "KopsControlPlane", + Name: "testCluster", + Namespace: "testNamespace", + }, + }, }, }, validateOutput: func(_ *securitygroupv1alpha2.SecurityGroup, csg *crossec2v1beta1.SecurityGroup) bool { @@ -1538,7 +1590,24 @@ func TestCreateOrUpdateCrossplaneSecurityGroup(t *testing.T) { }) { return false } - return true + + expectedTags := []crossec2v1beta1.Tag{ + { + Key: "ManagedBy", + Value: "kubernetes-crossplane-infrastructure-operator", + }, + { + Key: "kops.k8s.io/instance-group/testIGA", + Value: "owned", + }, + { + Key: "kops.k8s.io/instance-group/testIGB", + Value: "owned", + }, + } + + return reflect.DeepEqual(csg.Spec.ForProvider.Tags, expectedTags) + }, }, { @@ -1599,13 +1668,16 @@ func TestCreateOrUpdateCrossplaneSecurityGroup(t *testing.T) { err := crossec2v1beta1.SchemeBuilder.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = kinfrastructurev1alpha1.SchemeBuilder.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { ctx := context.TODO() fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(tc.k8sObjects...).Build() - _, err := CreateOrUpdateCrossplaneSecurityGroup(ctx, fakeClient, aws.String("vpc-xxx"), aws.String("us-east-1"), defaultProviderConfigName, tc.wildlifeSecurityGroup) + _, err := CreateOrUpdateCrossplaneSecurityGroup(ctx, fakeClient, aws.String("vpc-xxx"), aws.String("us-east-1"), defaultProviderConfigName, "testCluster", tc.wildlifeSecurityGroup) g.Expect(err).To(BeNil()) csg := &crossec2v1beta1.SecurityGroup{} key := client.ObjectKey{