csiaddonsnode: Add retry with exponential backoff for connections

This patch adds the functionality to retry for a maximum of
`maxRetries` to connect to the sidecar.

If the connection attempt is not successful, the object is considered
obsolete and is deleted.

The retry is tracked inside an annotation(`connRetryAnnotation`) and also
reflected in object's status.

These transient artifacts are cleaned up once a connection is
established.

Signed-off-by: Niraj Yadav <niryadav@redhat.com>

Author: black-dragon74

api/csiaddons/v1alpha1/csiaddonsnode_types.go

@@ -29,6 +29,11 @@ const (
 
 	// Failed represents the Connection Failed state.
 	CSIAddonsNodeStateFailed CSIAddonsNodeState = "Failed"
+
+	// Retrying represents the Connection Retrying state.
+	CSIAddonsNodeStateRetrying CSIAddonsNodeState = "Retrying"
+
+	CSIAddonsNodeStateRetryingFmtStr = "currentAttempt: %d"
 )
 
 type CSIAddonsNodeDriver struct {

internal/controller/csiaddons/csiaddonsnode_controller.go

@@ -21,9 +21,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math"
 	"net/url"
 	"slices"
+	"strconv"
 	"strings"
+	"time"
 
 	csiaddonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/csiaddons/v1alpha1"
 	"github.com/csi-addons/kubernetes-csi-addons/internal/connection"
@@ -42,6 +45,19 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
 
+const (
+	// The base duration to use for exponential backoff while retrying connection
+	baseRetryDelay = 2 * time.Second
+
+	// Maximum attempts to make while retrying the connection
+	maxRetries = 3
+
+	// The duration after which a new reconcile should be triggered
+	// to validate the cluster state. Used only when reconciliation
+	// completes without any errors.
+	baseRequeueAfter = 1 * time.Hour
+)
+
 var (
 	csiAddonsNodeFinalizer = csiaddonsv1alpha1.GroupVersion.Group + "/csiaddonsnode"
 )
@@ -128,20 +144,50 @@ func (r *CSIAddonsNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 	logger.Info("Connecting to sidecar")
 	newConn, err := connection.NewConnection(ctx, endPoint, nodeID, driverName, csiAddonsNode.Namespace, csiAddonsNode.Name, r.EnableAuth)
+
+	// If error occurs, we retry with exponential backoff until we reach `maxRetries`
+	// Store the transient state in annotation and also update the status to reflect the same
 	if err != nil {
-		logger.Error(err, "Failed to establish connection with sidecar")
+		// Only continue if we get a valid/initial retry count
+		currentRetries, e := getRetryCountFromReason(csiAddonsNode.Status.Reason)
+		if e != nil {
+			logger.Error(e, "failed to get the retry count from csiAddonsNode status", "csiAddonsNodeStatus", csiAddonsNode.Status)
+
+			return ctrl.Result{}, e
+		}
+		logger.Error(err, "Failed to establish connection with sidecar", "attempt", currentRetries+1)
+
+		// If reached max retries, abort and cleanup
+		if currentRetries >= maxRetries {
+			logger.Info(fmt.Sprintf("Failed to establish connection with sidecar after %d attempts, deleting the object", maxRetries))
+
+			if delErr := r.Delete(ctx, csiAddonsNode); client.IgnoreNotFound(delErr) != nil {
+				logger.Error(delErr, "failed to delete CSIAddonsNode object after max retries")
+
+				return ctrl.Result{}, delErr
+			}
+
+			// Object is deleted, stop the reconcile phase
+			logger.Info("successfully deleted CSIAddonsNode object due to reaching max reconnection attempts")
+			return ctrl.Result{}, nil
+		}
 
 		errMessage := util.GetErrorMessage(err)
-		csiAddonsNode.Status.State = csiaddonsv1alpha1.CSIAddonsNodeStateFailed
-		csiAddonsNode.Status.Message = fmt.Sprintf("Failed to establish connection with sidecar: %v", errMessage)
+		csiAddonsNode.Status.State = csiaddonsv1alpha1.CSIAddonsNodeStateRetrying
+		csiAddonsNode.Status.Message = fmt.Sprintf("Connection failed: %v. Retrying attempt %d/%d.", errMessage, currentRetries+1, maxRetries)
+		csiAddonsNode.Status.Reason = fmt.Sprintf(csiaddonsv1alpha1.CSIAddonsNodeStateRetryingFmtStr, currentRetries+1)
 		statusErr := r.Status().Update(ctx, csiAddonsNode)
 		if statusErr != nil {
 			logger.Error(statusErr, "Failed to update status")
 
 			return ctrl.Result{}, statusErr
 		}
 
-		return ctrl.Result{}, err
+		// Calculate backoff; baseRetryDelay * 1, 2, 4....
+		backoff := baseRetryDelay * time.Duration(math.Pow(2, float64(currentRetries)))
+		logger.Info("Requeuing request for attempting the connection again", "backoff", backoff)
+
+		return ctrl.Result{RequeueAfter: backoff}, nil
 	}
 
 	nfsc, err := r.getNetworkFenceClientStatus(ctx, &logger, newConn, csiAddonsNode)
@@ -157,6 +203,7 @@ func (r *CSIAddonsNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 	csiAddonsNode.Status.State = csiaddonsv1alpha1.CSIAddonsNodeStateConnected
 	csiAddonsNode.Status.Message = "Successfully established connection with sidecar"
+	csiAddonsNode.Status.Reason = ""
 	csiAddonsNode.Status.Capabilities = parseCapabilities(newConn.Capabilities)
 	err = r.Status().Update(ctx, csiAddonsNode)
 	if err != nil {
@@ -165,7 +212,8 @@ func (r *CSIAddonsNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, err
 	}
 
-	return ctrl.Result{}, nil
+	// Reconciled successfully, requeue to validate state periodically
+	return ctrl.Result{RequeueAfter: baseRequeueAfter}, nil
 }
 
 // getNetworkFenceClassesForDriver gets the networkfenceclasses for the driver.
@@ -341,6 +389,10 @@ func (r *CSIAddonsNodeReconciler) resolveEndpoint(ctx context.Context, rawURL st
 		Name:      podname,
 	}, pod)
 	if err != nil {
+		// do not return podname if the pod does not exist
+		if apierrors.IsNotFound(err) {
+			podname = ""
+		}
 		return podname, "", fmt.Errorf("failed to get pod %s/%s: %w", namespace, podname, err)
 	} else if pod.Status.PodIP == "" {
 		return podname, "", fmt.Errorf("pod %s/%s does not have an IP-address", namespace, podname)
@@ -416,3 +468,26 @@ func parseCapabilities(caps []*identity.Capability) []string {
 
 	return capabilities
 }
+
+// getRetryCountFromReason expects a string and tries to extract
+// and return the retry count from the string.
+// If the reason string is empty, it assumes the first attempt and returns 0.
+// An error is returned if the parsing is not successful.
+func getRetryCountFromReason(reason string) (int, error) {
+	// Might not be updated yet, likely the 1st attempt
+	if reason == "" {
+		return 0, nil
+	}
+
+	parts := strings.SplitN(reason, ":", 2)
+	if len(parts) < 2 {
+		return 0, errors.New("got an unexpected length after splitting the reason string")
+	}
+
+	// Parse
+	if c, err := strconv.Atoi(strings.TrimSpace(parts[1])); err == nil {
+		return c, nil
+	}
+
+	return 0, errors.New("failed to parse the reason string to an integer")
+}

internal/controller/csiaddons/csiaddonsnode_controller_test.go

@@ -124,3 +124,66 @@ func TestParseCapabilities(t *testing.T) {
 		})
 	}
 }
+func TestGetRetryCountFromReason(t *testing.T) {
+	tests := []struct {
+		name    string
+		reason  string
+		want    int
+		wantErr bool
+	}{
+		{
+			name:    "empty reason",
+			reason:  "",
+			want:    0,
+			wantErr: false,
+		},
+		{
+			name:    "valid reason",
+			reason:  "retry: 2",
+			want:    2,
+			wantErr: false,
+		},
+		{
+			name:    "valid with extra spaces",
+			reason:  "retry:    5",
+			want:    5,
+			wantErr: false,
+		},
+		{
+			name:    "valid with trailing spaces",
+			reason:  "something:  10  ",
+			want:    10,
+			wantErr: false,
+		},
+		{
+			name:    "no colon",
+			reason:  "retry 3",
+			want:    0,
+			wantErr: true,
+		},
+		{
+			name:    "non-integer value",
+			reason:  "retry: abc",
+			want:    0,
+			wantErr: true,
+		},
+		{
+			name:    "multiple colons",
+			reason:  "prefix: 7:extra",
+			want:    0,
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := getRetryCountFromReason(tt.reason)
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.want, got)
+			}
+		})
+	}
+}