From 2d523b6554666f7caa925ed6ff148683264cc073 Mon Sep 17 00:00:00 2001 From: Vladimir Iliakov Date: Fri, 28 Aug 2026 09:32:05 +0200 Subject: [PATCH] STAC-25639: retry transient port-forward setup failures --- .../orchestration/portforward/portforward.go | 66 +++++++++-- .../portforward/portforward_test.go | 105 ++++++++---------- 2 files changed, 100 insertions(+), 71 deletions(-) diff --git a/internal/orchestration/portforward/portforward.go b/internal/orchestration/portforward/portforward.go index 3225603..8c03138 100644 --- a/internal/orchestration/portforward/portforward.go +++ b/internal/orchestration/portforward/portforward.go @@ -2,11 +2,20 @@ package portforward import ( "fmt" + "time" - "github.com/stackvista/stackstate-backup-cli/internal/clients/k8s" "github.com/stackvista/stackstate-backup-cli/internal/foundation/logger" ) +const ( + portForwardMaxAttempts = 3 + portForwardRetryDelay = 2 * time.Second +) + +type portForwardClient interface { + PortForwardService(namespace, serviceName string, remotePort int) (chan struct{}, int, error) +} + // Conn contains the channels needed to manage a port-forward connection type Conn struct { StopChan chan struct{} @@ -18,23 +27,58 @@ type Conn struct { // It returns a Conn containing the stop channel and the actual local port. // The caller is responsible for closing the StopChan when done. func SetupPortForward( - k8sClient *k8s.Client, + k8sClient portForwardClient, namespace string, serviceName string, remotePort int, log *logger.Logger, +) (*Conn, error) { + return setupPortForward( + k8sClient, + namespace, + serviceName, + remotePort, + log, + portForwardMaxAttempts, + portForwardRetryDelay, + ) +} + +func setupPortForward( + k8sClient portForwardClient, + namespace string, + serviceName string, + remotePort int, + log *logger.Logger, + maxAttempts int, + retryDelay time.Duration, ) (*Conn, error) { log.Infof("Setting up port-forward to %s:%d in namespace %s...", serviceName, remotePort, namespace) - stopChan, actualLocalPort, err := k8sClient.PortForwardService(namespace, serviceName, remotePort) - if err != nil { - return nil, fmt.Errorf("failed to setup port-forward: %w", err) - } + for attempt := 1; attempt <= maxAttempts; attempt++ { + stopChan, actualLocalPort, err := k8sClient.PortForwardService(namespace, serviceName, remotePort) + if err == nil { + log.Successf("Port-forward established on localhost:%d", actualLocalPort) - log.Successf("Port-forward established on localhost:%d", actualLocalPort) + return &Conn{ + StopChan: stopChan, + LocalPort: actualLocalPort, + }, nil + } + + if attempt == maxAttempts { + return nil, fmt.Errorf("failed to setup port-forward after %d attempts: %w", attempt, err) + } + + log.Warningf( + "Port-forward attempt %d/%d failed: %v; retrying in %s", + attempt, + maxAttempts, + err, + retryDelay, + ) + time.Sleep(retryDelay) + } - return &Conn{ - StopChan: stopChan, - LocalPort: actualLocalPort, - }, nil + return nil, fmt.Errorf("failed to setup port-forward") } diff --git a/internal/orchestration/portforward/portforward_test.go b/internal/orchestration/portforward/portforward_test.go index af3cffe..f153b8c 100644 --- a/internal/orchestration/portforward/portforward_test.go +++ b/internal/orchestration/portforward/portforward_test.go @@ -1,83 +1,68 @@ package portforward import ( + "errors" "testing" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" - "github.com/stackvista/stackstate-backup-cli/internal/clients/k8s" "github.com/stackvista/stackstate-backup-cli/internal/foundation/logger" ) -func TestSetupPortForward_ServiceNotFound(t *testing.T) { - fakeClientset := fake.NewSimpleClientset() - client := k8s.NewTestClient(fakeClientset) - log := logger.New(true, false) +type portForwardResult struct { + stopChan chan struct{} + localPort int + err error +} - _, err := SetupPortForward(client, "default", "nonexistent-service", 9200, log) - if err == nil { - t.Fatal("expected error for nonexistent service, got nil") - } +type fakeClient struct { + results []portForwardResult + calls int } -func TestSetupPortForward_NoPodsFound(t *testing.T) { - fakeClientset := fake.NewSimpleClientset( - &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-service", - Namespace: "default", - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": "test", - }, - }, +func (f *fakeClient) PortForwardService(_, _ string, _ int) (chan struct{}, int, error) { + result := f.results[f.calls] + f.calls++ + return result.stopChan, result.localPort, result.err +} + +func TestSetupPortForward_RetriesTransientFailure(t *testing.T) { + stopChan := make(chan struct{}) + client := &fakeClient{ + results: []portForwardResult{ + {err: errors.New("connection reset by peer")}, + {err: errors.New("error upgrading connection")}, + {stopChan: stopChan, localPort: 43210}, }, - ) - client := k8s.NewTestClient(fakeClientset) + } log := logger.New(true, false) - _, err := SetupPortForward(client, "default", "test-service", 9200, log) - if err == nil { - t.Fatal("expected error for service with no pods, got nil") - } + result, err := setupPortForward(client, "default", "test-service", 9200, log, 3, 0) + + require.NoError(t, err) + assert.Equal(t, 3, client.calls) + assert.Equal(t, stopChan, result.StopChan) + assert.Equal(t, 43210, result.LocalPort) } -func TestSetupPortForward_NoRunningPods(t *testing.T) { - fakeClientset := fake.NewSimpleClientset( - &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-service", - Namespace: "default", - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": "test", - }, - }, - }, - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "default", - Labels: map[string]string{ - "app": "test", - }, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodPending, - }, +func TestSetupPortForward_ReturnsLastErrorAfterRetries(t *testing.T) { + client := &fakeClient{ + results: []portForwardResult{ + {err: errors.New("first failure")}, + {err: errors.New("second failure")}, + {err: errors.New("last failure")}, }, - ) - client := k8s.NewTestClient(fakeClientset) + } log := logger.New(true, false) - _, err := SetupPortForward(client, "default", "test-service", 9200, log) - if err == nil { - t.Fatal("expected error for service with no running pods, got nil") - } + result, err := setupPortForward(client, "default", "test-service", 9200, log, 3, 0) + + require.Error(t, err) + assert.Nil(t, result) + assert.Equal(t, 3, client.calls) + assert.ErrorContains(t, err, "failed to setup port-forward after 3 attempts") + assert.ErrorContains(t, err, "last failure") } func TestConn_Structure(t *testing.T) {