From 844a784edc8ee2fcbd6001efc9e9d16af7264ee0 Mon Sep 17 00:00:00 2001 From: Vladimir Iliakov Date: Fri, 4 Sep 2026 09:42:01 +0200 Subject: [PATCH 1/5] Fix Elasticsearch finalization after ILM cleanup Treat missing data-stream backing indices as lifecycle cleanup only when they form the oldest contiguous prefix of the snapshot generations. Keep ordinary indices and the newest backing generation mandatory so restore startup and incomplete restores cannot be mistaken for success. --- cmd/elasticsearch/check_and_finalize.go | 89 ++++++++++++-- cmd/elasticsearch/check_and_finalize_test.go | 119 +++++++++++++++++-- 2 files changed, 189 insertions(+), 19 deletions(-) diff --git a/cmd/elasticsearch/check_and_finalize.go b/cmd/elasticsearch/check_and_finalize.go index cd5cf5c..59cac50 100644 --- a/cmd/elasticsearch/check_and_finalize.go +++ b/cmd/elasticsearch/check_and_finalize.go @@ -2,6 +2,8 @@ package elasticsearch import ( "fmt" + "sort" + "strings" "time" "github.com/spf13/cobra" @@ -183,22 +185,61 @@ func expectedRestoredIndices(esClient es.Interface, appCtx *app.Context, reposit // cluster with fewer nodes than replicas keeps them unassigned forever, so requiring green would // never complete there. type restoreProgress struct { - // indicesRestored counts expected indices with every primary shard active, and decides completion. + // indicesRestored counts present expected indices with every primary shard active. indicesRestored int // primariesActive counts active primary shards, and drives stall detection. Whole indices are // too coarse for that: a large multi-shard index can restore for a long time without finishing. primariesActive int + // requiredExpected and requiredRestored track indices that ILM cannot remove. + requiredExpected int + requiredRestored int + // lifecycleExpected and lifecyclePresent track data-stream backing indices. ILM can remove an + // oldest contiguous prefix while the restore is running. + lifecycleExpected int + lifecyclePresent int + lifecycleMissing int + lifecycleGap bool } -func measureRestore(expected []string, health map[string]es.IndexHealth) restoreProgress { +func measureRestore(expected []string, datastreamPrefix string, health map[string]es.IndexHealth) restoreProgress { var progress restoreProgress + lifecycleIndices := make([]string, 0) for _, index := range expected { + if strings.HasPrefix(index, datastreamPrefix+"-") { + lifecycleIndices = append(lifecycleIndices, index) + continue + } + + progress.requiredExpected++ indexHealth, exists := health[index] if !exists { continue } + progress.primariesActive += indexHealth.ActivePrimaryShards + if indexHealth.NumberOfShards > 0 && indexHealth.ActivePrimaryShards == indexHealth.NumberOfShards { + progress.indicesRestored++ + progress.requiredRestored++ + } + } + + sort.Strings(lifecycleIndices) + progress.lifecycleExpected = len(lifecycleIndices) + + seenPresent := false + for _, index := range lifecycleIndices { + indexHealth, exists := health[index] + if !exists { + progress.lifecycleMissing++ + if seenPresent { + progress.lifecycleGap = true + } + continue + } + + seenPresent = true + progress.lifecyclePresent++ progress.primariesActive += indexHealth.ActivePrimaryShards if indexHealth.NumberOfShards > 0 && indexHealth.ActivePrimaryShards == indexHealth.NumberOfShards { progress.indicesRestored++ @@ -208,6 +249,22 @@ func measureRestore(expected []string, health map[string]es.IndexHealth) restore return progress } +func (p restoreProgress) complete() bool { + if p.requiredRestored != p.requiredExpected { + return false + } + + if p.lifecycleExpected == 0 { + return p.requiredExpected > 0 + } + + // The newest snapshot backing index must exist. Combined with no gap, this only permits missing + // indices at the oldest edge, matching the order in which ILM removes data-stream generations. + return p.lifecyclePresent > 0 && + !p.lifecycleGap && + p.indicesRestored == p.requiredRestored+p.lifecyclePresent +} + type indicesHealthGetter interface { GetIndicesHealth() (map[string]es.IndexHealth, error) } @@ -219,6 +276,7 @@ func newRestoreStatusFn( esClient indicesHealthGetter, log *logger.Logger, expected []string, + datastreamPrefix string, noProgressTimeout time.Duration, maxErrors int, ) func() (string, bool, error) { @@ -240,8 +298,14 @@ func newRestoreStatusFn( } errCount = 0 - progress := measureRestore(expected, health) - if progress.indicesRestored == len(expected) { + progress := measureRestore(expected, datastreamPrefix, health) + if progress.complete() { + if progress.lifecycleMissing > 0 { + log.Infof( + "Restore complete; %d oldest data-stream backing indices are no longer present, consistent with lifecycle retention", + progress.lifecycleMissing, + ) + } return es.StatusSuccess, true, nil } @@ -254,15 +318,15 @@ func newRestoreStatusFn( // that reads this as "failed" and retries would destroy a restore that is merely slow. return "", false, fmt.Errorf( "elasticsearch restore stalled: no primary shard finished restoring in %s "+ - "(%d of %d indices complete, %d primaries active); "+ + "(%d of %d indices present and complete, %d lifecycle-managed indices absent, %d primaries active); "+ "it may still be running server-side, so check before restarting it", - noProgressTimeout, progress.indicesRestored, len(expected), progress.primariesActive, + noProgressTimeout, progress.indicesRestored, len(expected), progress.lifecycleMissing, progress.primariesActive, ) } log.Debugf( - "Restored %d of %d indices (%d primaries active)", - progress.indicesRestored, len(expected), progress.primariesActive, + "Restored %d of %d indices (%d lifecycle-managed indices absent, %d primaries active)", + progress.indicesRestored, len(expected), progress.lifecycleMissing, progress.primariesActive, ) return es.StatusInProgress, false, nil @@ -291,7 +355,14 @@ func checkAndFinalize( maxErrors = 1 } - statusFn := newRestoreStatusFn(healthClient, appCtx.Logger, expected, noProgressTimeout, maxErrors) + statusFn := newRestoreStatusFn( + healthClient, + appCtx.Logger, + expected, + appCtx.Config.Elasticsearch.Restore.DatastreamIndexPrefix, + noProgressTimeout, + maxErrors, + ) // Get restore status appCtx.Logger.Infof("Checking restore status for snapshot: %s (%d indices)", snapshotName, len(expected)) diff --git a/cmd/elasticsearch/check_and_finalize_test.go b/cmd/elasticsearch/check_and_finalize_test.go index b55ae03..a6f2d92 100644 --- a/cmd/elasticsearch/check_and_finalize_test.go +++ b/cmd/elasticsearch/check_and_finalize_test.go @@ -38,6 +38,8 @@ func restored(shards int) es.IndexHealth { return es.IndexHealth{Status: "green", NumberOfShards: shards, ActivePrimaryShards: shards} } +const testDatastreamPrefix = ".ds-sts_k8s_logs" + func TestMeasureRestore(t *testing.T) { tests := []struct { name string @@ -94,6 +96,22 @@ func TestMeasureRestore(t *testing.T) { wantIndices: 2, wantPrimaries: 5, }, + { + name: "oldest lifecycle-managed indices can be absent", + expected: []string{ + ".ds-sts_k8s_logs-000001", + ".ds-sts_k8s_logs-000002", + ".ds-sts_k8s_logs-000003", + "sts_topology", + }, + health: map[string]es.IndexHealth{ + ".ds-sts_k8s_logs-000002": restored(2), + ".ds-sts_k8s_logs-000003": restored(2), + "sts_topology": restored(1), + }, + wantIndices: 3, + wantPrimaries: 5, + }, { name: "index reporting zero shards is not counted", expected: []string{"sts_topology"}, @@ -105,7 +123,7 @@ func TestMeasureRestore(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - progress := measureRestore(tt.expected, tt.health) + progress := measureRestore(tt.expected, testDatastreamPrefix, tt.health) assert.Equal(t, tt.wantIndices, progress.indicesRestored, "indicesRestored") assert.Equal(t, tt.wantPrimaries, progress.primariesActive, "primariesActive") }) @@ -121,7 +139,7 @@ func TestNewRestoreStatusFn_Progress(t *testing.T) { t.Run("accepted but not yet applied reports in progress", func(t *testing.T) { client := &fakeHealthClient{results: []healthResult{ok(map[string]es.IndexHealth{})}} - status, isComplete, err := newRestoreStatusFn(client, log, expected, time.Minute, 5)() + status, isComplete, err := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 5)() require.NoError(t, err) assert.Equal(t, es.StatusInProgress, status) @@ -133,7 +151,7 @@ func TestNewRestoreStatusFn_Progress(t *testing.T) { ok(map[string]es.IndexHealth{"sts_topology": restored(1)}), ok(map[string]es.IndexHealth{"sts_topology": restored(1), "sts_events": restored(1)}), }} - statusFn := newRestoreStatusFn(client, log, expected, time.Minute, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 5) status, isComplete, err := statusFn() require.NoError(t, err) @@ -147,6 +165,87 @@ func TestNewRestoreStatusFn_Progress(t *testing.T) { }) } +func TestNewRestoreStatusFn_ILMRemovedOldestBackingIndices(t *testing.T) { + expected := []string{ + "sts_topology", + ".ds-sts_k8s_logs-000003", + ".ds-sts_k8s_logs-000001", + ".ds-sts_k8s_logs-000002", + } + health := map[string]es.IndexHealth{ + "sts_topology": restored(1), + ".ds-sts_k8s_logs-000002": restored(1), + ".ds-sts_k8s_logs-000003": restored(1), + } + + status, isComplete, err := restoreStatus(expected, health) + + require.NoError(t, err) + assert.Equal(t, es.StatusSuccess, status) + assert.True(t, isComplete) +} + +func TestNewRestoreStatusFn_UnsafeMissingIndices(t *testing.T) { + tests := []struct { + name string + expected []string + health map[string]es.IndexHealth + }{ + { + name: "ordinary index", + expected: []string{"sts_topology", "sts_events", ".ds-sts_k8s_logs-000001"}, + health: map[string]es.IndexHealth{ + "sts_topology": restored(1), + ".ds-sts_k8s_logs-000001": restored(1), + }, + }, + { + name: "backing index after a present generation", + expected: []string{ + "sts_topology", + ".ds-sts_k8s_logs-000001", + ".ds-sts_k8s_logs-000002", + ".ds-sts_k8s_logs-000003", + }, + health: map[string]es.IndexHealth{ + "sts_topology": restored(1), + ".ds-sts_k8s_logs-000001": restored(1), + ".ds-sts_k8s_logs-000003": restored(1), + }, + }, + { + name: "newest backing index", + expected: []string{"sts_topology", ".ds-sts_k8s_logs-000001", ".ds-sts_k8s_logs-000002"}, + health: map[string]es.IndexHealth{ + "sts_topology": restored(1), + ".ds-sts_k8s_logs-000001": restored(1), + }, + }, + { + name: "all backing indices", + expected: []string{"sts_topology", ".ds-sts_k8s_logs-000001"}, + health: map[string]es.IndexHealth{"sts_topology": restored(1)}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status, isComplete, err := restoreStatus(tt.expected, tt.health) + + require.NoError(t, err) + assert.Equal(t, es.StatusInProgress, status) + assert.False(t, isComplete) + }) + } +} + +func restoreStatus(expected []string, health map[string]es.IndexHealth) (string, bool, error) { + client := &fakeHealthClient{results: []healthResult{ok(health)}} + log := logger.New(true, false) + + return newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 5)() +} + func TestNewRestoreStatusFn_Deadline(t *testing.T) { expected := statusFnExpected log := logger.New(true, false) @@ -155,7 +254,7 @@ func TestNewRestoreStatusFn_Deadline(t *testing.T) { client := &fakeHealthClient{results: []healthResult{ ok(map[string]es.IndexHealth{"sts_topology": restored(1)}), }} - statusFn := newRestoreStatusFn(client, log, expected, time.Nanosecond, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Nanosecond, 5) // First call records the progress it observed; the second sees no change past the deadline. _, isComplete, err := statusFn() @@ -179,7 +278,7 @@ func TestNewRestoreStatusFn_Deadline(t *testing.T) { ok(map[string]es.IndexHealth{"sts_topology": {NumberOfShards: 5, ActivePrimaryShards: 2}}), ok(map[string]es.IndexHealth{"sts_topology": {NumberOfShards: 5, ActivePrimaryShards: 3}}), }} - statusFn := newRestoreStatusFn(client, log, expected, time.Nanosecond, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Nanosecond, 5) for range 3 { status, isComplete, err := statusFn() @@ -194,7 +293,7 @@ func TestNewRestoreStatusFn_Deadline(t *testing.T) { ok(map[string]es.IndexHealth{}), ok(map[string]es.IndexHealth{"sts_topology": restored(1)}), }} - statusFn := newRestoreStatusFn(client, log, expected, time.Nanosecond, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Nanosecond, 5) _, _, err := statusFn() require.NoError(t, err) @@ -209,7 +308,7 @@ func TestNewRestoreStatusFn_Deadline(t *testing.T) { client := &fakeHealthClient{results: []healthResult{ ok(map[string]es.IndexHealth{"sts_topology": restored(1)}), }} - statusFn := newRestoreStatusFn(client, log, expected, 0, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, 0, 5) for range 3 { status, isComplete, err := statusFn() @@ -230,7 +329,7 @@ func TestNewRestoreStatusFn_TransientErrors(t *testing.T) { {err: errors.New("connection refused")}, ok(map[string]es.IndexHealth{"sts_topology": restored(1), "sts_events": restored(1)}), }} - statusFn := newRestoreStatusFn(client, log, expected, time.Minute, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 5) for range 2 { status, isComplete, err := statusFn() @@ -247,7 +346,7 @@ func TestNewRestoreStatusFn_TransientErrors(t *testing.T) { t.Run("errors surface once they stop being transient", func(t *testing.T) { client := &fakeHealthClient{results: []healthResult{{err: errors.New("connection refused")}}} - statusFn := newRestoreStatusFn(client, log, expected, time.Minute, 3) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 3) for range 2 { _, _, err := statusFn() @@ -266,7 +365,7 @@ func TestNewRestoreStatusFn_TransientErrors(t *testing.T) { {err: errors.New("blip")}, {err: errors.New("blip")}, }} - statusFn := newRestoreStatusFn(client, log, expected, time.Minute, 3) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 3) for range 4 { _, _, err := statusFn() From a3224b93026b6abdd4c3961f12d3975a35c9e5a7 Mon Sep 17 00:00:00 2001 From: Vladimir Iliakov Date: Fri, 4 Sep 2026 10:13:07 +0200 Subject: [PATCH 2/5] Address restore review and security findings Use ordinary STS indices as the positive restore anchor, allow all lifecycle-managed generations to age out, reject lifecycle-only snapshots, and keep stall progress monotonic. Add realistic backing-index coverage and bump x/crypto to v0.56.0 for the current HIGH CVE fixes. --- cmd/elasticsearch/check_and_finalize.go | 49 ++++++++++--- cmd/elasticsearch/check_and_finalize_test.go | 74 +++++++++++++++++--- go.mod | 2 +- go.sum | 4 +- 4 files changed, 106 insertions(+), 23 deletions(-) diff --git a/cmd/elasticsearch/check_and_finalize.go b/cmd/elasticsearch/check_and_finalize.go index 59cac50..a616ba0 100644 --- a/cmd/elasticsearch/check_and_finalize.go +++ b/cmd/elasticsearch/check_and_finalize.go @@ -178,6 +178,21 @@ func expectedRestoredIndices(esClient es.Interface, appCtx *app.Context, reposit return nil, fmt.Errorf("snapshot %s contains no indices matching the configured STS prefixes", snapshotName) } + hasRequiredIndex := false + for _, index := range expected { + if !isDatastreamBackingIndex(index, appCtx.Config.Elasticsearch.Restore.DatastreamIndexPrefix) { + hasRequiredIndex = true + break + } + } + if !hasRequiredIndex { + return nil, fmt.Errorf( + "snapshot %s contains only lifecycle-managed data-stream backing indices; "+ + "restore completion cannot be determined safely", + snapshotName, + ) + } + return expected, nil } @@ -203,10 +218,10 @@ type restoreProgress struct { func measureRestore(expected []string, datastreamPrefix string, health map[string]es.IndexHealth) restoreProgress { var progress restoreProgress - lifecycleIndices := make([]string, 0) + var lifecycleIndices []string for _, index := range expected { - if strings.HasPrefix(index, datastreamPrefix+"-") { + if isDatastreamBackingIndex(index, datastreamPrefix) { lifecycleIndices = append(lifecycleIndices, index) continue } @@ -249,19 +264,27 @@ func measureRestore(expected []string, datastreamPrefix string, health map[strin return progress } +func isDatastreamBackingIndex(index, datastreamPrefix string) bool { + return strings.HasPrefix(index, datastreamPrefix+"-") +} + func (p restoreProgress) complete() bool { + if p.requiredExpected == 0 { + return false + } + if p.requiredRestored != p.requiredExpected { return false } - if p.lifecycleExpected == 0 { - return p.requiredExpected > 0 + // Required indices prove the restore's cluster-state update has applied; target index metadata + // is created together before shard recovery starts. + if p.lifecycleExpected == 0 || p.lifecyclePresent == 0 { + return true } - // The newest snapshot backing index must exist. Combined with no gap, this only permits missing - // indices at the oldest edge, matching the order in which ILM removes data-stream generations. - return p.lifecyclePresent > 0 && - !p.lifecycleGap && + // With any backing indices left, only an oldest prefix may be absent. + return !p.lifecycleGap && p.indicesRestored == p.requiredRestored+p.lifecyclePresent } @@ -301,15 +324,19 @@ func newRestoreStatusFn( progress := measureRestore(expected, datastreamPrefix, health) if progress.complete() { if progress.lifecycleMissing > 0 { + indexWord := "indices" + if progress.lifecycleMissing == 1 { + indexWord = "index" + } log.Infof( - "Restore complete; %d oldest data-stream backing indices are no longer present, consistent with lifecycle retention", - progress.lifecycleMissing, + "Restore complete; %d oldest data-stream backing %s no longer present, consistent with lifecycle retention", + progress.lifecycleMissing, indexWord, ) } return es.StatusSuccess, true, nil } - if progress.primariesActive != lastPrimariesActive { + if progress.primariesActive > lastPrimariesActive { lastPrimariesActive = progress.primariesActive lastProgressAt = time.Now() } else if noProgressTimeout > 0 && time.Since(lastProgressAt) > noProgressTimeout { diff --git a/cmd/elasticsearch/check_and_finalize_test.go b/cmd/elasticsearch/check_and_finalize_test.go index a6f2d92..c8c197f 100644 --- a/cmd/elasticsearch/check_and_finalize_test.go +++ b/cmd/elasticsearch/check_and_finalize_test.go @@ -8,7 +8,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stackvista/stackstate-backup-cli/internal/app" es "github.com/stackvista/stackstate-backup-cli/internal/clients/elasticsearch" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/foundation/logger" ) @@ -23,6 +25,15 @@ type fakeHealthClient struct { calls int } +type snapshotOnlyClient struct { + es.Interface + snapshot *es.Snapshot +} + +func (c *snapshotOnlyClient) GetSnapshot(_, _ string) (*es.Snapshot, error) { + return c.snapshot, nil +} + func (f *fakeHealthClient) GetIndicesHealth() (map[string]es.IndexHealth, error) { result := f.results[min(f.calls, len(f.results)-1)] f.calls++ @@ -168,14 +179,14 @@ func TestNewRestoreStatusFn_Progress(t *testing.T) { func TestNewRestoreStatusFn_ILMRemovedOldestBackingIndices(t *testing.T) { expected := []string{ "sts_topology", - ".ds-sts_k8s_logs-000003", - ".ds-sts_k8s_logs-000001", - ".ds-sts_k8s_logs-000002", + ".ds-sts_k8s_logs-2026.08.28-012011", + ".ds-sts_k8s_logs-2026.08.26-011965", + ".ds-sts_k8s_logs-2026.08.27-011999", } health := map[string]es.IndexHealth{ - "sts_topology": restored(1), - ".ds-sts_k8s_logs-000002": restored(1), - ".ds-sts_k8s_logs-000003": restored(1), + "sts_topology": restored(1), + ".ds-sts_k8s_logs-2026.08.27-011999": restored(1), + ".ds-sts_k8s_logs-2026.08.28-012011": restored(1), } status, isComplete, err := restoreStatus(expected, health) @@ -185,6 +196,17 @@ func TestNewRestoreStatusFn_ILMRemovedOldestBackingIndices(t *testing.T) { assert.True(t, isComplete) } +func TestNewRestoreStatusFn_ILMRemovedAllBackingIndices(t *testing.T) { + expected := []string{"sts_topology", ".ds-sts_k8s_logs-2026.08.26-011965"} + health := map[string]es.IndexHealth{"sts_topology": restored(1)} + + status, isComplete, err := restoreStatus(expected, health) + + require.NoError(t, err) + assert.Equal(t, es.StatusSuccess, status) + assert.True(t, isComplete) +} + func TestNewRestoreStatusFn_UnsafeMissingIndices(t *testing.T) { tests := []struct { name string @@ -222,9 +244,9 @@ func TestNewRestoreStatusFn_UnsafeMissingIndices(t *testing.T) { }, }, { - name: "all backing indices", - expected: []string{"sts_topology", ".ds-sts_k8s_logs-000001"}, - health: map[string]es.IndexHealth{"sts_topology": restored(1)}, + name: "snapshot has no required anchor", + expected: []string{".ds-sts_k8s_logs-000001"}, + health: map[string]es.IndexHealth{".ds-sts_k8s_logs-000001": restored(1)}, }, } @@ -239,6 +261,24 @@ func TestNewRestoreStatusFn_UnsafeMissingIndices(t *testing.T) { } } +func TestExpectedRestoredIndices_RejectsLifecycleOnlySnapshot(t *testing.T) { + client := &snapshotOnlyClient{snapshot: &es.Snapshot{ + Indices: []string{".ds-sts_k8s_logs-2026.08.28-012011"}, + }} + appCtx := &app.Context{Config: &config.Config{ + Elasticsearch: config.ElasticsearchConfig{Restore: config.RestoreConfig{ + IndexPrefix: "sts", + DatastreamIndexPrefix: testDatastreamPrefix, + }}, + }} + + _, err := expectedRestoredIndices(client, appCtx, "repository", "snapshot") + + require.Error(t, err) + assert.ErrorContains(t, err, "only lifecycle-managed data-stream backing indices") + assert.ErrorContains(t, err, "cannot be determined safely") +} + func restoreStatus(expected []string, health map[string]es.IndexHealth) (string, bool, error) { client := &fakeHealthClient{results: []healthResult{ok(health)}} log := logger.New(true, false) @@ -304,6 +344,22 @@ func TestNewRestoreStatusFn_Deadline(t *testing.T) { assert.False(t, isComplete) }) + t.Run("decreasing primaries do not reset the deadline", func(t *testing.T) { + client := &fakeHealthClient{results: []healthResult{ + ok(map[string]es.IndexHealth{"sts_topology": {NumberOfShards: 4, ActivePrimaryShards: 3}}), + ok(map[string]es.IndexHealth{"sts_topology": {NumberOfShards: 4, ActivePrimaryShards: 2}}), + }} + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Nanosecond, 5) + + _, _, err := statusFn() + require.NoError(t, err) + + _, isComplete, err := statusFn() + require.Error(t, err) + assert.False(t, isComplete) + assert.ErrorContains(t, err, "stalled") + }) + t.Run("zero waits indefinitely", func(t *testing.T) { client := &fakeHealthClient{results: []healthResult{ ok(map[string]es.IndexHealth{"sts_topology": restored(1)}), diff --git a/go.mod b/go.mod index 10543f2..04f2e61 100644 --- a/go.mod +++ b/go.mod @@ -80,7 +80,7 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.55.0 // indirect + golang.org/x/crypto v0.56.0 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sys v0.47.0 // indirect diff --git a/go.sum b/go.sum index dade845..9cb0984 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= From d816ff300f72007330d028ee19a9d7cbe34de36c Mon Sep 17 00:00:00 2001 From: Vladimir Iliakov Date: Fri, 4 Sep 2026 11:51:17 +0200 Subject: [PATCH 3/5] Handle ILM deletions during restore progress --- cmd/elasticsearch/check_and_finalize.go | 27 +++++++-- cmd/elasticsearch/check_and_finalize_test.go | 59 +++++++++++++++++++- cmd/elasticsearch/restore.go | 10 +++- cmd/elasticsearch/restore_test.go | 11 ++++ 4 files changed, 98 insertions(+), 9 deletions(-) diff --git a/cmd/elasticsearch/check_and_finalize.go b/cmd/elasticsearch/check_and_finalize.go index a616ba0..be54f05 100644 --- a/cmd/elasticsearch/check_and_finalize.go +++ b/cmd/elasticsearch/check_and_finalize.go @@ -169,18 +169,22 @@ func expectedRestoredIndices(esClient es.Interface, appCtx *app.Context, reposit return nil, fmt.Errorf("failed to get snapshot details: %w", err) } + return restorableSnapshotIndices(snapshot, appCtx.Config.Elasticsearch.Restore) +} + +func restorableSnapshotIndices(snapshot *es.Snapshot, restoreConfig config.RestoreConfig) ([]string, error) { expected := filterSTSIndices( snapshot.Indices, - appCtx.Config.Elasticsearch.Restore.IndexPrefix, - appCtx.Config.Elasticsearch.Restore.DatastreamIndexPrefix, + restoreConfig.IndexPrefix, + restoreConfig.DatastreamIndexPrefix, ) if len(expected) == 0 { - return nil, fmt.Errorf("snapshot %s contains no indices matching the configured STS prefixes", snapshotName) + return nil, fmt.Errorf("snapshot %s contains no indices matching the configured STS prefixes", snapshot.Snapshot) } hasRequiredIndex := false for _, index := range expected { - if !isDatastreamBackingIndex(index, appCtx.Config.Elasticsearch.Restore.DatastreamIndexPrefix) { + if !isDatastreamBackingIndex(index, restoreConfig.DatastreamIndexPrefix) { hasRequiredIndex = true break } @@ -189,7 +193,7 @@ func expectedRestoredIndices(esClient es.Interface, appCtx *app.Context, reposit return nil, fmt.Errorf( "snapshot %s contains only lifecycle-managed data-stream backing indices; "+ "restore completion cannot be determined safely", - snapshotName, + snapshot.Snapshot, ) } @@ -304,6 +308,7 @@ func newRestoreStatusFn( maxErrors int, ) func() (string, bool, error) { lastPrimariesActive := -1 + lastLifecyclePresent := -1 lastProgressAt := time.Now() errCount := 0 @@ -336,10 +341,20 @@ func newRestoreStatusFn( return es.StatusSuccess, true, nil } + lifecycleRemoved := lastLifecyclePresent >= 0 && progress.lifecyclePresent < lastLifecyclePresent + lastLifecyclePresent = progress.lifecyclePresent + + if lifecycleRemoved && progress.primariesActive < lastPrimariesActive { + // An ILM deletion makes totals before and after this poll incomparable. Rebaseline + // without treating the deletion as progress or failing before a later shard increase + // can be observed. + lastPrimariesActive = progress.primariesActive + } + if progress.primariesActive > lastPrimariesActive { lastPrimariesActive = progress.primariesActive lastProgressAt = time.Now() - } else if noProgressTimeout > 0 && time.Since(lastProgressAt) > noProgressTimeout { + } else if !lifecycleRemoved && noProgressTimeout > 0 && time.Since(lastProgressAt) > noProgressTimeout { // Deliberately not reported as a failed restore: all this establishes is that no progress // was observed from here. Restarting a restore deletes every STS index first, so a caller // that reads this as "failed" and retries would destroy a restore that is merely slow. diff --git a/cmd/elasticsearch/check_and_finalize_test.go b/cmd/elasticsearch/check_and_finalize_test.go index c8c197f..a132d72 100644 --- a/cmd/elasticsearch/check_and_finalize_test.go +++ b/cmd/elasticsearch/check_and_finalize_test.go @@ -263,7 +263,8 @@ func TestNewRestoreStatusFn_UnsafeMissingIndices(t *testing.T) { func TestExpectedRestoredIndices_RejectsLifecycleOnlySnapshot(t *testing.T) { client := &snapshotOnlyClient{snapshot: &es.Snapshot{ - Indices: []string{".ds-sts_k8s_logs-2026.08.28-012011"}, + Snapshot: "snapshot", + Indices: []string{".ds-sts_k8s_logs-2026.08.28-012011"}, }} appCtx := &app.Context{Config: &config.Config{ Elasticsearch: config.ElasticsearchConfig{Restore: config.RestoreConfig{ @@ -375,6 +376,62 @@ func TestNewRestoreStatusFn_Deadline(t *testing.T) { }) } +func TestNewRestoreStatusFn_ILMDeletionRebasesHighWaterMark(t *testing.T) { + expected := []string{"sts_topology", ".ds-sts_k8s_logs-2026.08.28-012011"} + + t.Run("later shard progress can exceed the rebased total", func(t *testing.T) { + client := &fakeHealthClient{results: []healthResult{ + restoreHealth(3, true), + restoreHealth(5, false), + restoreHealth(6, false), + restoreHealth(6, false), + }} + statusFn := newRestoreStatusFn(client, logger.New(true, false), expected, testDatastreamPrefix, time.Nanosecond, 5) + + for range 3 { + status, isComplete, err := statusFn() + require.NoError(t, err) + assert.Equal(t, es.StatusInProgress, status) + assert.False(t, isComplete) + } + + _, isComplete, err := statusFn() + require.Error(t, err) + assert.False(t, isComplete) + assert.ErrorContains(t, err, "stalled") + }) + + t.Run("the deletion itself does not reset the deadline", func(t *testing.T) { + client := &fakeHealthClient{results: []healthResult{ + restoreHealth(3, true), + restoreHealth(5, false), + restoreHealth(5, false), + }} + statusFn := newRestoreStatusFn(client, logger.New(true, false), expected, testDatastreamPrefix, time.Nanosecond, 5) + + for range 2 { + _, _, err := statusFn() + require.NoError(t, err) + } + + _, isComplete, err := statusFn() + require.Error(t, err) + assert.False(t, isComplete) + assert.ErrorContains(t, err, "stalled") + }) +} + +func restoreHealth(topologyPrimaries int, includeBackingIndex bool) healthResult { + health := map[string]es.IndexHealth{ + "sts_topology": {NumberOfShards: 10, ActivePrimaryShards: topologyPrimaries}, + } + if includeBackingIndex { + health[".ds-sts_k8s_logs-2026.08.28-012011"] = restored(8) + } + + return ok(health) +} + func TestNewRestoreStatusFn_TransientErrors(t *testing.T) { expected := statusFnExpected log := logger.New(true, false) diff --git a/cmd/elasticsearch/restore.go b/cmd/elasticsearch/restore.go index 77868d4..680fa56 100644 --- a/cmd/elasticsearch/restore.go +++ b/cmd/elasticsearch/restore.go @@ -96,6 +96,12 @@ func runRestore(appCtx *app.Context) error { return err } + // Reject snapshots that cannot be monitored safely before scaling down workloads or deleting + // any indices. check-and-finalize repeats this validation for independently resumed operations. + if _, err := restorableSnapshotIndices(snapshotDetails, appCtx.Config.Elasticsearch.Restore); err != nil { + return err + } + // Confirm with user before starting destructive operation if !skipConfirmation { appCtx.Logger.Println() @@ -246,7 +252,7 @@ func deleteAllSTSIndices(esClient es.Interface, appCtx *app.Context) error { func filterSTSIndices(allIndices []string, indexPrefix, datastreamPrefix string) []string { var stsIndices []string for _, index := range allIndices { - if strings.HasPrefix(index, indexPrefix) || strings.HasPrefix(index, datastreamPrefix) { + if strings.HasPrefix(index, indexPrefix) || isDatastreamBackingIndex(index, datastreamPrefix) { stsIndices = append(stsIndices, index) } } @@ -256,7 +262,7 @@ func filterSTSIndices(allIndices []string, indexPrefix, datastreamPrefix string) // hasDatastreamIndices checks if any indices belong to a datastream func hasDatastreamIndices(indices []string, datastreamPrefix string) bool { for _, index := range indices { - if strings.HasPrefix(index, datastreamPrefix+"-") { + if isDatastreamBackingIndex(index, datastreamPrefix) { return true } } diff --git a/cmd/elasticsearch/restore_test.go b/cmd/elasticsearch/restore_test.go index e611100..d112d2a 100644 --- a/cmd/elasticsearch/restore_test.go +++ b/cmd/elasticsearch/restore_test.go @@ -180,6 +180,17 @@ func TestFilterSTSIndices(t *testing.T) { expectedCount: 2, expectedIndices: []string{"sts_k8s_logs-000001", "sts_k8s_logs-000002"}, }, + { + name: "sibling datastream is not included", + allIndices: []string{ + ".ds-sts_k8s_logs-2026.08.28-012011", + ".ds-sts_k8s_logs_archive-2026.08.28-000001", + }, + indexPrefix: "sts", + datastreamPrefix: ".ds-sts_k8s_logs", + expectedCount: 1, + expectedIndices: []string{".ds-sts_k8s_logs-2026.08.28-012011"}, + }, } for _, tt := range tests { From 34db3b3129727296394141b7f0b89fbb6c8e3ae3 Mon Sep 17 00:00:00 2001 From: Vladimir Iliakov Date: Fri, 4 Sep 2026 12:17:33 +0200 Subject: [PATCH 4/5] Align Elasticsearch restore index selection --- cmd/elasticsearch/check_and_finalize_test.go | 26 ++++++++++++++++++++ cmd/elasticsearch/restore.go | 5 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/cmd/elasticsearch/check_and_finalize_test.go b/cmd/elasticsearch/check_and_finalize_test.go index a132d72..1c162a3 100644 --- a/cmd/elasticsearch/check_and_finalize_test.go +++ b/cmd/elasticsearch/check_and_finalize_test.go @@ -2,6 +2,7 @@ package elasticsearch import ( "errors" + "strings" "testing" "time" @@ -280,6 +281,31 @@ func TestExpectedRestoredIndices_RejectsLifecycleOnlySnapshot(t *testing.T) { assert.ErrorContains(t, err, "cannot be determined safely") } +func TestRestorableSnapshotIndices_ExcludesSiblingDatastream(t *testing.T) { + restoreConfig := config.RestoreConfig{ + IndexPrefix: "sts", + DatastreamIndexPrefix: testDatastreamPrefix, + IndicesPattern: "sts*,.ds-sts_k8s_logs*", + } + snapshot := &es.Snapshot{ + Snapshot: "snapshot", + Indices: []string{ + "sts_topology", + ".ds-sts_k8s_logs-2026.08.28-012011", + ".ds-sts_k8s_logs_archive-2026.08.28-000001", + }, + } + + indices, err := restorableSnapshotIndices(snapshot, restoreConfig) + + require.NoError(t, err) + assert.Equal(t, []string{ + "sts_topology", + ".ds-sts_k8s_logs-2026.08.28-012011", + }, indices) + assert.Equal(t, "sts_topology,.ds-sts_k8s_logs-2026.08.28-012011", strings.Join(indices, ",")) +} + func restoreStatus(expected []string, health map[string]es.IndexHealth) (string, bool, error) { client := &fakeHealthClient{results: []healthResult{ok(health)}} log := logger.New(true, false) diff --git a/cmd/elasticsearch/restore.go b/cmd/elasticsearch/restore.go index 680fa56..47631c8 100644 --- a/cmd/elasticsearch/restore.go +++ b/cmd/elasticsearch/restore.go @@ -98,7 +98,8 @@ func runRestore(appCtx *app.Context) error { // Reject snapshots that cannot be monitored safely before scaling down workloads or deleting // any indices. check-and-finalize repeats this validation for independently resumed operations. - if _, err := restorableSnapshotIndices(snapshotDetails, appCtx.Config.Elasticsearch.Restore); err != nil { + restoreIndices, err := restorableSnapshotIndices(snapshotDetails, appCtx.Config.Elasticsearch.Restore) + if err != nil { return err } @@ -143,7 +144,7 @@ func runRestore(appCtx *app.Context) error { appCtx.Logger.Println() isPartial := snapshotDetails.State == "PARTIAL" appCtx.Logger.Infof("Triggering restore for snapshot: %s", selectedSnapshot) - if err := esClient.RestoreSnapshot(repository, selectedSnapshot, appCtx.Config.Elasticsearch.Restore.IndicesPattern, isPartial); err != nil { + if err := esClient.RestoreSnapshot(repository, selectedSnapshot, strings.Join(restoreIndices, ","), isPartial); err != nil { return fmt.Errorf("failed to trigger restore: %w", err) } appCtx.Logger.Successf("Restore triggered successfully") From 93e8ca7620b129a457f7df0322276f5b8a6ae151 Mon Sep 17 00:00:00 2001 From: Vladimir Iliakov Date: Fri, 4 Sep 2026 12:51:15 +0200 Subject: [PATCH 5/5] Honor configured restore index patterns --- cmd/elasticsearch/check_and_finalize.go | 11 ++- cmd/elasticsearch/check_and_finalize_test.go | 6 +- cmd/elasticsearch/restore.go | 67 +++++++++++++++ cmd/elasticsearch/restore_test.go | 88 ++++++++++++++++++++ 4 files changed, 168 insertions(+), 4 deletions(-) diff --git a/cmd/elasticsearch/check_and_finalize.go b/cmd/elasticsearch/check_and_finalize.go index be54f05..c6b6197 100644 --- a/cmd/elasticsearch/check_and_finalize.go +++ b/cmd/elasticsearch/check_and_finalize.go @@ -173,13 +173,20 @@ func expectedRestoredIndices(esClient es.Interface, appCtx *app.Context, reposit } func restorableSnapshotIndices(snapshot *es.Snapshot, restoreConfig config.RestoreConfig) ([]string, error) { - expected := filterSTSIndices( + candidates := filterSTSIndices( snapshot.Indices, restoreConfig.IndexPrefix, restoreConfig.DatastreamIndexPrefix, ) + expected, err := filterIndicesByPattern(candidates, restoreConfig.IndicesPattern) + if err != nil { + return nil, fmt.Errorf("invalid Elasticsearch restore indicesPattern: %w", err) + } if len(expected) == 0 { - return nil, fmt.Errorf("snapshot %s contains no indices matching the configured STS prefixes", snapshot.Snapshot) + return nil, fmt.Errorf( + "snapshot %s contains no indices matching the configured STS prefixes and indicesPattern", + snapshot.Snapshot, + ) } hasRequiredIndex := false diff --git a/cmd/elasticsearch/check_and_finalize_test.go b/cmd/elasticsearch/check_and_finalize_test.go index 1c162a3..ad1c185 100644 --- a/cmd/elasticsearch/check_and_finalize_test.go +++ b/cmd/elasticsearch/check_and_finalize_test.go @@ -271,6 +271,7 @@ func TestExpectedRestoredIndices_RejectsLifecycleOnlySnapshot(t *testing.T) { Elasticsearch: config.ElasticsearchConfig{Restore: config.RestoreConfig{ IndexPrefix: "sts", DatastreamIndexPrefix: testDatastreamPrefix, + IndicesPattern: "sts*,.ds-sts_k8s_logs*", }}, }} @@ -281,16 +282,17 @@ func TestExpectedRestoredIndices_RejectsLifecycleOnlySnapshot(t *testing.T) { assert.ErrorContains(t, err, "cannot be determined safely") } -func TestRestorableSnapshotIndices_ExcludesSiblingDatastream(t *testing.T) { +func TestRestorableSnapshotIndices_AppliesPatternAndExcludesSiblingDatastream(t *testing.T) { restoreConfig := config.RestoreConfig{ IndexPrefix: "sts", DatastreamIndexPrefix: testDatastreamPrefix, - IndicesPattern: "sts*,.ds-sts_k8s_logs*", + IndicesPattern: "sts_topology,.ds-sts_k8s_logs-*", } snapshot := &es.Snapshot{ Snapshot: "snapshot", Indices: []string{ "sts_topology", + "sts_metrics", ".ds-sts_k8s_logs-2026.08.28-012011", ".ds-sts_k8s_logs_archive-2026.08.28-000001", }, diff --git a/cmd/elasticsearch/restore.go b/cmd/elasticsearch/restore.go index 47631c8..4c25f98 100644 --- a/cmd/elasticsearch/restore.go +++ b/cmd/elasticsearch/restore.go @@ -2,6 +2,7 @@ package elasticsearch import ( "fmt" + "path" "sort" "strings" "time" @@ -218,6 +219,10 @@ func deleteAllSTSIndices(esClient es.Interface, appCtx *app.Context) error { } stsIndices := filterSTSIndices(allIndices, appCtx.Config.Elasticsearch.Restore.IndexPrefix, appCtx.Config.Elasticsearch.Restore.DatastreamIndexPrefix) + stsIndices, err = filterIndicesByPattern(stsIndices, appCtx.Config.Elasticsearch.Restore.IndicesPattern) + if err != nil { + return fmt.Errorf("invalid Elasticsearch restore indicesPattern: %w", err) + } if len(stsIndices) == 0 { appCtx.Logger.Infof("No STS indices found to delete") @@ -260,6 +265,68 @@ func filterSTSIndices(allIndices []string, indexPrefix, datastreamPrefix string) return stsIndices } +func filterIndicesByPattern(indices []string, expression string) ([]string, error) { + type pattern struct { + value string + exclude bool + } + + var patterns []pattern + hasInclude := false + for _, value := range strings.Split(expression, ",") { + value = strings.TrimSpace(value) + if value == "" { + return nil, fmt.Errorf("contains an empty pattern") + } + + exclude := strings.HasPrefix(value, "-") + if exclude { + value = strings.TrimPrefix(value, "-") + if value == "" { + return nil, fmt.Errorf("contains an empty exclusion") + } + } else { + hasInclude = true + } + + if value == "_all" { + value = "*" + } + if _, err := path.Match(value, ""); err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", value, err) + } + patterns = append(patterns, pattern{value: value, exclude: exclude}) + } + if !hasInclude { + return nil, fmt.Errorf("must contain at least one inclusion pattern") + } + + var matched []string + for _, index := range indices { + included := false + excluded := false + for _, candidate := range patterns { + isMatch, err := path.Match(candidate.value, index) + if err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", candidate.value, err) + } + if !isMatch { + continue + } + if candidate.exclude { + excluded = true + } else { + included = true + } + } + if included && !excluded { + matched = append(matched, index) + } + } + + return matched, nil +} + // hasDatastreamIndices checks if any indices belong to a datastream func hasDatastreamIndices(indices []string, datastreamPrefix string) bool { for _, index := range indices { diff --git a/cmd/elasticsearch/restore_test.go b/cmd/elasticsearch/restore_test.go index d112d2a..33531e5 100644 --- a/cmd/elasticsearch/restore_test.go +++ b/cmd/elasticsearch/restore_test.go @@ -90,6 +90,10 @@ func (m *mockESClientForRestore) ConfigureSnapshotRepository(_, _, _, _, _, _ st return fmt.Errorf("not implemented") } +func (m *mockESClientForRestore) DeleteSnapshotRepository(_ string) error { + return fmt.Errorf("not implemented") +} + func (m *mockESClientForRestore) ConfigureSLMPolicy(_, _, _, _, _, _ string, _, _ int) error { return fmt.Errorf("not implemented") } @@ -207,6 +211,90 @@ func TestFilterSTSIndices(t *testing.T) { } } +func TestFilterIndicesByPattern(t *testing.T) { + indices := []string{ + "sts_topology", + "sts_metrics", + ".ds-sts_k8s_logs-2026.08.28-012011", + } + + tests := []struct { + name string + expression string + expected []string + expectErr string + }{ + { + name: "narrowed restore", + expression: "sts_topology,.ds-sts_k8s_logs-*", + expected: []string{ + "sts_topology", + ".ds-sts_k8s_logs-2026.08.28-012011", + }, + }, + { + name: "exclusion", + expression: "sts*,-sts_metrics", + expected: []string{"sts_topology"}, + }, + { + name: "all", + expression: "_all", + expected: indices, + }, + { + name: "invalid pattern", + expression: "sts[", + expectErr: "invalid pattern", + }, + { + name: "exclusion only", + expression: "-sts_metrics", + expectErr: "at least one inclusion", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := filterIndicesByPattern(indices, tt.expression) + + if tt.expectErr != "" { + require.ErrorContains(t, err, tt.expectErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestDeleteAllSTSIndicesHonorsIndicesPattern(t *testing.T) { + client := &mockESClientForRestore{ + indices: []string{"sts_topology", "sts_metrics", "other"}, + indexExistsMap: map[string]bool{ + "sts_topology": true, + "sts_metrics": true, + }, + } + appCtx := &app.Context{ + Config: &config.Config{ + Elasticsearch: config.ElasticsearchConfig{ + Restore: config.RestoreConfig{ + IndexPrefix: "sts", + DatastreamIndexPrefix: ".ds-sts_k8s_logs", + IndicesPattern: "sts_topology", + }, + }, + }, + Logger: logger.New(true, false), + } + + err := deleteAllSTSIndices(client, appCtx) + + require.NoError(t, err) + assert.Equal(t, []string{"sts_topology"}, client.deletedIndices) +} + // TestHasDatastreamIndices tests datastream detection func TestHasDatastreamIndices(t *testing.T) { tests := []struct {