From a9cc1c6c138e6718a852386c0d7c13ba156519bb Mon Sep 17 00:00:00 2001 From: Jon Huhn Date: Fri, 22 Sep 2023 11:07:44 -0500 Subject: [PATCH] add transient error handling to AMMP delete --- .../azuremanagedmachinepool_controller.go | 11 +++++++++++ .../azuremanagedmachinepool_controller_test.go | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/controllers/azuremanagedmachinepool_controller.go b/controllers/azuremanagedmachinepool_controller.go index e7167fe21b6..be879dbdc12 100644 --- a/controllers/azuremanagedmachinepool_controller.go +++ b/controllers/azuremanagedmachinepool_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "fmt" "time" "github.com/pkg/errors" @@ -311,6 +312,16 @@ func (ammpr *AzureManagedMachinePoolReconciler) reconcileDelete(ctx context.Cont } if err := svc.Delete(ctx); err != nil { + // Handle transient errors + var reconcileError azure.ReconcileError + if errors.As(err, &reconcileError) && reconcileError.IsTransient() { + if azure.IsOperationNotDoneError(reconcileError) { + log.V(2).Info(fmt.Sprintf("AzureManagedMachinePool delete not done: %s", reconcileError.Error())) + } else { + log.V(2).Info("transient failure to delete AzureManagedMachinePool, retrying") + } + return reconcile.Result{RequeueAfter: reconcileError.RequeueAfter()}, nil + } return reconcile.Result{}, errors.Wrapf(err, "error deleting AzureManagedMachinePool %s/%s", scope.InfraMachinePool.Namespace, scope.InfraMachinePool.Name) } // Machine pool successfully deleted, remove the finalizer. diff --git a/controllers/azuremanagedmachinepool_controller_test.go b/controllers/azuremanagedmachinepool_controller_test.go index 6c0fae2b4a5..c1767f346fe 100644 --- a/controllers/azuremanagedmachinepool_controller_test.go +++ b/controllers/azuremanagedmachinepool_controller_test.go @@ -26,6 +26,7 @@ import ( "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -33,6 +34,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/pointer" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/mock_azure" "sigs.k8s.io/cluster-api-provider-azure/azure/scope" "sigs.k8s.io/cluster-api-provider-azure/azure/services/agentpools" @@ -95,6 +97,22 @@ func TestAzureManagedMachinePoolReconcile(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) }, }, + { + name: "Reconcile delete transient error", + Setup: func(cb *fake.ClientBuilder, reconciler *mock_azure.MockReconcilerMockRecorder, agentpools *mock_agentpools.MockAgentPoolScopeMockRecorder, _ *MockNodeListerMockRecorder) { + cluster, azManagedCluster, azManagedControlPlane, ammp, mp := newReadyAzureManagedMachinePoolCluster() + reconciler.Delete(gomock2.AContext()).Return(azure.WithTransientError(errors.New("transient"), 76*time.Second)) + agentpools.Name() + ammp.DeletionTimestamp = &metav1.Time{ + Time: time.Now(), + } + cb.WithObjects(cluster, azManagedCluster, azManagedControlPlane, ammp, mp) + }, + Verify: func(g *WithT, result ctrl.Result, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result.RequeueAfter).To(Equal(76 * time.Second)) + }, + }, } for _, c := range cases {