account for retryable errors (#422)

controller/linodemachine_controller.go

@@ -300,6 +300,9 @@ func (r *LinodeMachineReconciler) reconcileCreate(
 		createOpts, err := r.newCreateConfig(ctx, machineScope, tags, logger)
 		if err != nil {
 			logger.Error(err, "Failed to create Linode machine InstanceCreateOptions")
+			if util.IsTransientError(err) {
+				return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerRetryDelay}, nil
+			}
 
 			return ctrl.Result{}, err
 		}

util/helpers.go

@@ -2,6 +2,9 @@ package util
 
 import (
 	"errors"
+	"io"
+	"net/http"
+	"os"
 
 	"github.com/linode/linodego"
 )
@@ -30,3 +33,23 @@ func UnwrapError(err error) error {
 
 	return err
 }
+
+// IsTransientError determines if the error is transient, meaning a controller that
+// encounters this error should requeue reconciliation to try again later
+func IsTransientError(err error) bool {
+	if linodego.ErrHasStatus(
+		err,
+		http.StatusTooManyRequests,
+		http.StatusInternalServerError,
+		http.StatusBadGateway,
+		http.StatusGatewayTimeout,
+		http.StatusServiceUnavailable) {
+		return true
+	}
+
+	if errors.Is(err, http.ErrHandlerTimeout) || errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, io.ErrUnexpectedEOF) {
+		return true
+	}
+
+	return false
+}

util/helpers_test.go

@@ -2,6 +2,8 @@ package util
 
 import (
 	"errors"
+	"io"
+	"net/http"
 	"testing"
 
 	"github.com/linode/linodego"
@@ -49,3 +51,41 @@ func TestIgnoreLinodeAPIError(t *testing.T) {
 		})
 	}
 }
+
+func TestIsTransientError(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name        string
+		err         error
+		shouldRetry bool
+	}{{
+		name:        "unexpected EOF",
+		err:         io.ErrUnexpectedEOF,
+		shouldRetry: true,
+	}, {
+		name: "not found Linode API error",
+		err: &linodego.Error{
+			Response: nil,
+			Code:     http.StatusNotFound,
+			Message:  "not found",
+		},
+		shouldRetry: false,
+	}, {
+		name: "Rate limiting Linode API error",
+		err: &linodego.Error{
+			Response: nil,
+			Code:     http.StatusTooManyRequests,
+			Message:  "rate limited",
+		},
+		shouldRetry: true,
+	}}
+	for _, tt := range tests {
+		testcase := tt
+		t.Run(testcase.name, func(t *testing.T) {
+			t.Parallel()
+			if testcase.shouldRetry != IsTransientError(testcase.err) {
+				t.Errorf("wanted %v, got %v", testcase.shouldRetry, IsTransientError(testcase.err))
+			}
+		})
+	}
+}