Skip to content

Commit

Permalink
Add feature to allow dry-run for change plans
Browse files Browse the repository at this point in the history
  • Loading branch information
@rsafonseca
rsafonseca committed Sep 9, 2024
1 parent 427cfee commit caea388
Show file tree
Hide file tree
Showing 7 changed files with 219 additions and 163 deletions.
160 changes: 97 additions & 63 deletions controllers/controlplane/kopscontrolplane_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,13 @@ type KopsControlPlaneReconciler struct {
ControllerClass string
Recorder record.EventRecorder
TfExecPath string
DryRun bool
GetKopsClientSetFactory func(configBase string) (simple.Clientset, error)
BuildCloudFactory func(*kopsapi.Cluster) (fi.Cloud, error)
PopulateClusterSpecFactory func(ctx context.Context, kopsCluster *kopsapi.Cluster, kopsClientset simple.Clientset, cloud fi.Cloud) (*kopsapi.Cluster, error)
PrepareKopsCloudResourcesFactory func(ctx context.Context, kopsClientset simple.Clientset, kopsCluster *kopsapi.Cluster, terraformOutputDir string, cloud fi.Cloud) error
ApplyTerraformFactory func(ctx context.Context, terraformDir, tfExecPath string, credentials aws.Credentials) error
PlanTerraformFactory func(ctx context.Context, terraformDir, tfExecPath string, credentials aws.Credentials) error
DestroyTerraformFactory func(ctx context.Context, terraformDir, tfExecPath string, credentials aws.Credentials) error
KopsDeleteResourcesFactory func(ctx context.Context, cloud fi.Cloud, kopsClientset simple.Clientset, kopsCluster *kopsapi.Cluster) error
ValidateKopsClusterFactory func(kubeConfig *rest.Config, kopsCluster *kopsapi.Cluster, cloud fi.Cloud, igs *kopsapi.InstanceGroupList) (*validation.ValidationCluster, error)
Expand Down Expand Up @@ -136,6 +138,9 @@ func GetClusterStatus(kopsCluster *kopsapi.Cluster, cloud fi.Cloud) (*kopsapi.Cl
}

func (r *KopsControlPlaneReconciler) shouldDeleteCluster(kcp *controlplanev1alpha1.KopsControlPlane) bool {
if r.DryRun {
return false
}
if !kcp.ObjectMeta.DeletionTimestamp.IsZero() {
if kcp.Annotations[controlplanev1alpha1.ClusterDeleteProtectionAnnotation] == "true" {
r.Recorder.Eventf(kcp, corev1.EventTypeWarning, "ClusterDeleteProtectionEnabled", "cluster delete protection is enabled, skipping deletion")
Expand Down Expand Up @@ -355,7 +360,7 @@ func PrepareKopsCloudResources(ctx context.Context, kopsClientset simple.Clients
defer func() {
os.Stdout = stdout
}()
os.Stdout = nil
//os.Stdout = nil

if err := applyCmd.Run(ctx); err != nil {
return err
Expand Down Expand Up @@ -571,8 +576,27 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req

initTime := time.Now()
kopsControlPlane := &controlplanev1alpha1.KopsControlPlane{}
if err := r.Get(ctx, req.NamespacedName, kopsControlPlane); err != nil {
return resultError, client.IgnoreNotFound(err)
var kubeReader client.Reader
if r.DryRun {
cli, ok := ctx.Value("client").(client.Reader)
if !ok {
log.Info("failed to get kube client")
return ctrl.Result{}, nil
}
kubeReader = cli
kcp, ok := ctx.Value("kcp").(controlplanev1alpha1.KopsControlPlane)
if !ok {
log.Info("failed to get kcp")
return ctrl.Result{}, nil
} else {
log.Info("got kcp")
kopsControlPlane = &kcp
}
} else {
kubeReader = r.Client
if err := r.Get(ctx, req.NamespacedName, kopsControlPlane); err != nil {
return resultError, client.IgnoreNotFound(err)
}
}

reconciler := &KopsControlPlaneReconciliation{
Expand Down Expand Up @@ -600,7 +624,6 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToGetClusterMetadata", "could not get cluster with metadata: %s", err)
return resultError, err
}

var kmps []infrastructurev1alpha1.KopsMachinePool

terraformOutputDir := fmt.Sprintf("/tmp/%s", owner.GetName())
Expand Down Expand Up @@ -640,7 +663,7 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
}
}

if kopsControlPlane.Spec.TerraformConfig.CleanupTerraformDirectory {
if !r.DryRun && kopsControlPlane.Spec.TerraformConfig.CleanupTerraformDirectory {
err := utils.CleanupTerraformDirectory(terraformOutputDir)
if err != nil {
r.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedCleanupTerraformDirectory", "failed to cleanup terraform directory from cluster: %s", err)
Expand All @@ -659,7 +682,6 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req

reconciler.log.Info(fmt.Sprintf("finished reconcile loop for %s, took %s", kopsControlPlane.ObjectMeta.GetName(), time.Since(initTime)))
}()

if annotations.HasPaused(owner) {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeNormal, "ClusterPaused", "reconciliation is paused since cluster %s is paused", owner.GetName())
kopsControlPlane.Status.Paused = true
Expand All @@ -668,13 +690,12 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
kopsControlPlane.Status.Paused = false
kopsControlPlane.Status.Ready = false

awsCredentials, err := util.GetAWSCredentialsFromKopsControlPlaneSecret(ctx, r.Client, kopsControlPlane.Spec.IdentityRef.Name, kopsControlPlane.Spec.IdentityRef.Namespace)
awsCredentials, err := util.GetAWSCredentialsFromKopsControlPlaneSecret(ctx, kubeReader, kopsControlPlane.Spec.IdentityRef.Name, kopsControlPlane.Spec.IdentityRef.Namespace)
if err != nil {
reconciler.log.Error(err, "failed to get AWS credentials")
return resultError, err
}
reconciler.awsCredentials = *awsCredentials

kopsCluster := &kopsapi.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: owner.GetName(),
Expand All @@ -691,8 +712,7 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
return resultError, fmt.Errorf("error parsing NonMasqueradeCIDR %q: %v", kopsCluster.Spec.Networking.NonMasqueradeCIDR, err)
}
}

kmps, err = kopsutils.GetKopsMachinePoolsWithLabel(ctx, reconciler.Client, "cluster.x-k8s.io/cluster-name", kopsControlPlane.Name)
kmps, err = kopsutils.GetKopsMachinePoolsWithLabel(ctx, kubeReader, "cluster.x-k8s.io/cluster-name", kopsControlPlane.Name)
if err != nil {
return resultError, err
}
Expand Down Expand Up @@ -777,7 +797,6 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
if !controllerutil.ContainsFinalizer(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneFinalizer) {
controllerutil.AddFinalizer(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneFinalizer)
}

reconciler.Mux.Lock()
shouldUnlock = true

Expand Down Expand Up @@ -854,19 +873,24 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
return resultError, err
}

err = reconciler.createOrUpdateKopsCluster(ctx, kopsClientset, fullCluster, kopsControlPlane.Spec.SSHPublicKey, cloud)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToManageKopsState", "failed to manage Kops state: %s", err)
conditions.MarkFalse(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneStateReadyCondition, controlplanev1alpha1.KopsControlPlaneStateReconciliationFailedReason, clusterv1.ConditionSeverityError, "failed to create/update KopsCluster: %s", err.Error())
return resultError, err
}
conditions.MarkTrue(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneStateReadyCondition)
// Only Apply resources if DryRun isn't set from command line
if r.DryRun {

err = reconciler.ReconcileKopsSecrets(ctx, kopsClientset, kopsCluster)
if err != nil {
conditions.MarkFalse(reconciler.kcp, controlplanev1alpha1.KopsControlPlaneSecretsReadyCondition, controlplanev1alpha1.KopsControlPlaneSecretsReconciliationFailedReason, clusterv1.ConditionSeverityWarning, "failed to reconcile KopsSecrets: %s", err.Error())
} else {
err = reconciler.createOrUpdateKopsCluster(ctx, kopsClientset, fullCluster, kopsControlPlane.Spec.SSHPublicKey, cloud)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToManageKopsState", "failed to manage Kops state: %s", err)
conditions.MarkFalse(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneStateReadyCondition, controlplanev1alpha1.KopsControlPlaneStateReconciliationFailedReason, clusterv1.ConditionSeverityError, "failed to create/update KopsCluster: %s", err.Error())
return resultError, err
}
conditions.MarkTrue(kopsControlPlane, controlplanev1alpha1.KopsControlPlaneStateReadyCondition)

err = reconciler.ReconcileKopsSecrets(ctx, kopsClientset, kopsCluster)
if err != nil {
conditions.MarkFalse(reconciler.kcp, controlplanev1alpha1.KopsControlPlaneSecretsReadyCondition, controlplanev1alpha1.KopsControlPlaneSecretsReconciliationFailedReason, clusterv1.ConditionSeverityWarning, "failed to reconcile KopsSecrets: %s", err.Error())
}
conditions.MarkTrue(reconciler.kcp, controlplanev1alpha1.KopsControlPlaneSecretsReadyCondition)
}
conditions.MarkTrue(reconciler.kcp, controlplanev1alpha1.KopsControlPlaneSecretsReadyCondition)

var shouldIgnoreSG bool
if _, ok := owner.GetAnnotations()["kopscontrolplane.controlplane.wildlife.io/external-security-groups"]; ok {
Expand All @@ -889,57 +913,67 @@ func (r *KopsControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Req
return resultError, err
}

// TODO: This is needed because we are using a method from kops lib
// we should check alternatives
kubeConfig, err := utils.GetKubeconfigFromKopsState(ctx, kopsCluster, kopsClientset)
if err != nil {
return resultError, err
}
// Only Apply resources if DryRun isn't set from command line
if r.DryRun {
reconciler.log.Info(fmt.Sprintf("planning Terraform for %s", kopsControlPlane.ObjectMeta.GetName()))
err = reconciler.PlanTerraformFactory(ctx, terraformOutputDir, r.TfExecPath, reconciler.awsCredentials)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToPlanTerraform", "failed to plan terraform: %s", err)
return resultError, err
}
} else {
// TODO: This is needed because we are using a method from kops lib
// we should check alternatives
kubeConfig, err := utils.GetKubeconfigFromKopsState(ctx, kopsCluster, kopsClientset)
if err != nil {
return resultError, err
}

err = reconciler.reconcileKubeconfig(ctx, kubeConfig, kopsCluster, owner)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToReconcileKubeconfig", "failed to reconcile kubeconfig: %s", err)
return resultError, err
}
err = reconciler.reconcileKubeconfig(ctx, kubeConfig, kopsCluster, owner)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToReconcileKubeconfig", "failed to reconcile kubeconfig: %s", err)
return resultError, err
}

reconciler.Mux.Unlock()
shouldUnlock = false
reconciler.Mux.Unlock()
shouldUnlock = false

reconciler.log.Info(fmt.Sprintf("applying Terraform for %s", kopsControlPlane.ObjectMeta.GetName()))
reconciler.log.Info(fmt.Sprintf("applying Terraform for %s", kopsControlPlane.ObjectMeta.GetName()))

err = reconciler.ApplyTerraformFactory(ctx, terraformOutputDir, r.TfExecPath, reconciler.awsCredentials)
if err != nil {
conditions.MarkFalse(kopsControlPlane, controlplanev1alpha1.TerraformApplyReadyCondition, controlplanev1alpha1.TerraformApplyReconciliationFailedReason, clusterv1.ConditionSeverityError, "failed to apply Terraform: %s", err.Error())
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToApplyTerraform", "failed to apply terraform: %s", err)
return resultError, err
}
conditions.MarkTrue(kopsControlPlane, controlplanev1alpha1.TerraformApplyReadyCondition)
err = reconciler.ApplyTerraformFactory(ctx, terraformOutputDir, r.TfExecPath, reconciler.awsCredentials)
if err != nil {
conditions.MarkFalse(kopsControlPlane, controlplanev1alpha1.TerraformApplyReadyCondition, controlplanev1alpha1.TerraformApplyReconciliationFailedReason, clusterv1.ConditionSeverityError, "failed to apply Terraform: %s", err.Error())
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToApplyTerraform", "failed to apply terraform: %s", err)
return resultError, err
}
conditions.MarkTrue(kopsControlPlane, controlplanev1alpha1.TerraformApplyReadyCondition)

reconciler.log.Info(fmt.Sprintf("Terraform applied for %s", kopsControlPlane.ObjectMeta.GetName()))
reconciler.Recorder.Event(kopsControlPlane, corev1.EventTypeNormal, "TerraformApplied", "Terraform applied")
reconciler.log.Info(fmt.Sprintf("Terraform applied for %s", kopsControlPlane.ObjectMeta.GetName()))
reconciler.Recorder.Event(kopsControlPlane, corev1.EventTypeNormal, "TerraformApplied", "Terraform applied")

err = reconciler.updateKopsMachinePoolWithProviderIDList(kopsControlPlane, kmps, &reconciler.awsCredentials)
if err != nil {
if apierrors.IsNotFound(err) {
return requeue1min, nil
err = reconciler.updateKopsMachinePoolWithProviderIDList(kopsControlPlane, kmps, &reconciler.awsCredentials)
if err != nil {
if apierrors.IsNotFound(err) {
return requeue1min, nil
}
return resultError, err
}
return resultError, err
}

igList, err := kopsClientset.InstanceGroupsFor(kopsCluster).List(ctx, metav1.ListOptions{})
if err != nil || len(igList.Items) == 0 {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToGetIGs", "cannot get InstanceGroups: %s", err)
return resultError, fmt.Errorf("cannot get InstanceGroups for %q: %w", kopsCluster.ObjectMeta.Name, err)
}
igList, err := kopsClientset.InstanceGroupsFor(kopsCluster).List(ctx, metav1.ListOptions{})
if err != nil || len(igList.Items) == 0 {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToGetIGs", "cannot get InstanceGroups: %s", err)
return resultError, fmt.Errorf("cannot get InstanceGroups for %q: %w", kopsCluster.ObjectMeta.Name, err)
}

val, err := reconciler.ValidateKopsClusterFactory(kubeConfig, kopsCluster, cloud, igList)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToValidateKubernetesCluster", "failed trying to validate Kubernetes cluster: %v", err)
return resultError, err
}
val, err := reconciler.ValidateKopsClusterFactory(kubeConfig, kopsCluster, cloud, igList)
if err != nil {
reconciler.Recorder.Eventf(kopsControlPlane, corev1.EventTypeWarning, "FailedToValidateKubernetesCluster", "failed trying to validate Kubernetes cluster: %v", err)
return resultError, err
}

kopsControlPlane.Status.Ready = utils.KopsClusterValidation(kopsControlPlane, r.Recorder, reconciler.log, val)
reconciler.Recorder.Event(kopsControlPlane, corev1.EventTypeNormal, "ClusterReconciliationFinished", "cluster reconciliation finished")
kopsControlPlane.Status.Ready = utils.KopsClusterValidation(kopsControlPlane, r.Recorder, reconciler.log, val)
reconciler.Recorder.Event(kopsControlPlane, corev1.EventTypeNormal, "ClusterReconciliationFinished", "cluster reconciliation finished")
}
return resultDefault, nil
}

Expand Down
18 changes: 13 additions & 5 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ require (
github.com/crossplane-contrib/provider-aws v0.30.1
github.com/go-logr/logr v1.4.2
github.com/hashicorp/go-version v1.6.0
github.com/hashicorp/hc-install v0.4.0
github.com/hashicorp/terraform-exec v0.17.3
github.com/hashicorp/hc-install v0.6.4
github.com/hashicorp/terraform-exec v0.21.0
github.com/onsi/ginkgo/v2 v2.20.0
github.com/onsi/gomega v1.34.1
github.com/pkg/errors v0.9.1
Expand Down Expand Up @@ -43,6 +43,9 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/ProtonMail/go-crypto v1.1.0-alpha.2 // indirect
github.com/agext/levenshtein v1.2.1 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.8 // indirect
Expand All @@ -67,6 +70,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.29.1 // indirect
github.com/awslabs/operatorpkg v0.0.0-20240514175841-edb8fe5824b4 // indirect
github.com/cert-manager/cert-manager v1.15.0 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
Expand All @@ -88,6 +92,7 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/hashicorp/go-retryablehttp v0.7.6 // indirect
github.com/hashicorp/hcl/v2 v2.20.1 // indirect
github.com/hetznercloud/hcloud-go v1.56.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
Expand All @@ -96,6 +101,7 @@ require (
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
github.com/moby/sys/mountinfo v0.6.2 // indirect
github.com/moby/term v0.5.0 // indirect
Expand All @@ -121,6 +127,7 @@ require (
go.opentelemetry.io/otel/trace v1.27.0 // indirect
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/mod v0.20.0 // indirect
golang.org/x/tools v0.24.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect
gotest.tools/v3 v3.4.0 // indirect
Expand Down Expand Up @@ -170,7 +177,7 @@ require (
github.com/gophercloud/gophercloud v1.12.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/hcl v1.0.1-vault-5 // indirect
github.com/hashicorp/terraform-json v0.14.0 // indirect
github.com/hashicorp/terraform-json v0.22.1 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
Expand All @@ -190,14 +197,15 @@ require (
github.com/prometheus/client_model v0.6.1
github.com/prometheus/common v0.53.0 // indirect
github.com/prometheus/procfs v0.15.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spotinst/spotinst-sdk-go v1.171.0 // indirect
github.com/zclconf/go-cty v1.12.1 // indirect
github.com/zclconf/go-cty v1.14.4 // indirect
go.mercari.io/hcledit v0.0.17
go.opencensus.io v0.24.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0
Expand Down
Loading

0 comments on commit caea388

Please sign in to comment.