STAC-25639: retry transient port-forward setup failures

internal/orchestration/portforward/portforward.go

@@ -2,11 +2,20 @@ package portforward
 
 import (
 	"fmt"
+	"time"
 
-	"github.com/stackvista/stackstate-backup-cli/internal/clients/k8s"
 	"github.com/stackvista/stackstate-backup-cli/internal/foundation/logger"
 )
 
+const (
+	portForwardMaxAttempts = 3
+	portForwardRetryDelay  = 2 * time.Second
+)
+
+type portForwardClient interface {
+	PortForwardService(namespace, serviceName string, remotePort int) (chan struct{}, int, error)
+}
+
 // Conn contains the channels needed to manage a port-forward connection
 type Conn struct {
 	StopChan  chan struct{}
@@ -18,23 +27,58 @@ type Conn struct {
 // It returns a Conn containing the stop channel and the actual local port.
 // The caller is responsible for closing the StopChan when done.
 func SetupPortForward(
-	k8sClient *k8s.Client,
+	k8sClient portForwardClient,
 	namespace string,
 	serviceName string,
 	remotePort int,
 	log *logger.Logger,
+) (*Conn, error) {
+	return setupPortForward(
+		k8sClient,
+		namespace,
+		serviceName,
+		remotePort,
+		log,
+		portForwardMaxAttempts,
+		portForwardRetryDelay,
+	)
+}
+
+func setupPortForward(
+	k8sClient portForwardClient,
+	namespace string,
+	serviceName string,
+	remotePort int,
+	log *logger.Logger,
+	maxAttempts int,
+	retryDelay time.Duration,
 ) (*Conn, error) {
 	log.Infof("Setting up port-forward to %s:%d in namespace %s...", serviceName, remotePort, namespace)
 
-	stopChan, actualLocalPort, err := k8sClient.PortForwardService(namespace, serviceName, remotePort)
-	if err != nil {
-		return nil, fmt.Errorf("failed to setup port-forward: %w", err)
-	}
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		stopChan, actualLocalPort, err := k8sClient.PortForwardService(namespace, serviceName, remotePort)
+		if err == nil {
+			log.Successf("Port-forward established on localhost:%d", actualLocalPort)
 
-	log.Successf("Port-forward established on localhost:%d", actualLocalPort)
+			return &Conn{
+				StopChan:  stopChan,
+				LocalPort: actualLocalPort,
+			}, nil
+		}
+
+		if attempt == maxAttempts {
+			return nil, fmt.Errorf("failed to setup port-forward after %d attempts: %w", attempt, err)
+		}
+
+		log.Warningf(
+			"Port-forward attempt %d/%d failed: %v; retrying in %s",
+			attempt,
+			maxAttempts,
+			err,
+			retryDelay,
+		)
+		time.Sleep(retryDelay)
+	}
 
-	return &Conn{
-		StopChan:  stopChan,
-		LocalPort: actualLocalPort,
-	}, nil
+	return nil, fmt.Errorf("failed to setup port-forward")
 }

internal/orchestration/portforward/portforward_test.go

@@ -1,83 +1,68 @@
 package portforward
 
 import (
+	"errors"
 	"testing"
 
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes/fake"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
-	"github.com/stackvista/stackstate-backup-cli/internal/clients/k8s"
 	"github.com/stackvista/stackstate-backup-cli/internal/foundation/logger"
 )
 
-func TestSetupPortForward_ServiceNotFound(t *testing.T) {
-	fakeClientset := fake.NewSimpleClientset()
-	client := k8s.NewTestClient(fakeClientset)
-	log := logger.New(true, false)
+type portForwardResult struct {
+	stopChan  chan struct{}
+	localPort int
+	err       error
+}
 
-	_, err := SetupPortForward(client, "default", "nonexistent-service", 9200, log)
-	if err == nil {
-		t.Fatal("expected error for nonexistent service, got nil")
-	}
+type fakeClient struct {
+	results []portForwardResult
+	calls   int
 }
 
-func TestSetupPortForward_NoPodsFound(t *testing.T) {
-	fakeClientset := fake.NewSimpleClientset(
-		&corev1.Service{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "test-service",
-				Namespace: "default",
-			},
-			Spec: corev1.ServiceSpec{
-				Selector: map[string]string{
-					"app": "test",
-				},
-			},
+func (f *fakeClient) PortForwardService(_, _ string, _ int) (chan struct{}, int, error) {
+	result := f.results[f.calls]
+	f.calls++
+	return result.stopChan, result.localPort, result.err
+}
+
+func TestSetupPortForward_RetriesTransientFailure(t *testing.T) {
+	stopChan := make(chan struct{})
+	client := &fakeClient{
+		results: []portForwardResult{
+			{err: errors.New("connection reset by peer")},
+			{err: errors.New("error upgrading connection")},
+			{stopChan: stopChan, localPort: 43210},
 		},
-	)
-	client := k8s.NewTestClient(fakeClientset)
+	}
 	log := logger.New(true, false)
 
-	_, err := SetupPortForward(client, "default", "test-service", 9200, log)
-	if err == nil {
-		t.Fatal("expected error for service with no pods, got nil")
-	}
+	result, err := setupPortForward(client, "default", "test-service", 9200, log, 3, 0)
+
+	require.NoError(t, err)
+	assert.Equal(t, 3, client.calls)
+	assert.Equal(t, stopChan, result.StopChan)
+	assert.Equal(t, 43210, result.LocalPort)
 }
 
-func TestSetupPortForward_NoRunningPods(t *testing.T) {
-	fakeClientset := fake.NewSimpleClientset(
-		&corev1.Service{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "test-service",
-				Namespace: "default",
-			},
-			Spec: corev1.ServiceSpec{
-				Selector: map[string]string{
-					"app": "test",
-				},
-			},
-		},
-		&corev1.Pod{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "test-pod",
-				Namespace: "default",
-				Labels: map[string]string{
-					"app": "test",
-				},
-			},
-			Status: corev1.PodStatus{
-				Phase: corev1.PodPending,
-			},
+func TestSetupPortForward_ReturnsLastErrorAfterRetries(t *testing.T) {
+	client := &fakeClient{
+		results: []portForwardResult{
+			{err: errors.New("first failure")},
+			{err: errors.New("second failure")},
+			{err: errors.New("last failure")},
 		},
-	)
-	client := k8s.NewTestClient(fakeClientset)
+	}
 	log := logger.New(true, false)
 
-	_, err := SetupPortForward(client, "default", "test-service", 9200, log)
-	if err == nil {
-		t.Fatal("expected error for service with no running pods, got nil")
-	}
+	result, err := setupPortForward(client, "default", "test-service", 9200, log, 3, 0)
+
+	require.Error(t, err)
+	assert.Nil(t, result)
+	assert.Equal(t, 3, client.calls)
+	assert.ErrorContains(t, err, "failed to setup port-forward after 3 attempts")
+	assert.ErrorContains(t, err, "last failure")
 }
 
 func TestConn_Structure(t *testing.T) {