diff --git a/cmd/elasticsearch/check_and_finalize.go b/cmd/elasticsearch/check_and_finalize.go index cd5cf5c..c6b6197 100644 --- a/cmd/elasticsearch/check_and_finalize.go +++ b/cmd/elasticsearch/check_and_finalize.go @@ -2,6 +2,8 @@ package elasticsearch import ( "fmt" + "sort" + "strings" "time" "github.com/spf13/cobra" @@ -167,13 +169,39 @@ func expectedRestoredIndices(esClient es.Interface, appCtx *app.Context, reposit return nil, fmt.Errorf("failed to get snapshot details: %w", err) } - expected := filterSTSIndices( + return restorableSnapshotIndices(snapshot, appCtx.Config.Elasticsearch.Restore) +} + +func restorableSnapshotIndices(snapshot *es.Snapshot, restoreConfig config.RestoreConfig) ([]string, error) { + candidates := filterSTSIndices( snapshot.Indices, - appCtx.Config.Elasticsearch.Restore.IndexPrefix, - appCtx.Config.Elasticsearch.Restore.DatastreamIndexPrefix, + restoreConfig.IndexPrefix, + restoreConfig.DatastreamIndexPrefix, ) + expected, err := filterIndicesByPattern(candidates, restoreConfig.IndicesPattern) + if err != nil { + return nil, fmt.Errorf("invalid Elasticsearch restore indicesPattern: %w", err) + } if len(expected) == 0 { - return nil, fmt.Errorf("snapshot %s contains no indices matching the configured STS prefixes", snapshotName) + return nil, fmt.Errorf( + "snapshot %s contains no indices matching the configured STS prefixes and indicesPattern", + snapshot.Snapshot, + ) + } + + hasRequiredIndex := false + for _, index := range expected { + if !isDatastreamBackingIndex(index, restoreConfig.DatastreamIndexPrefix) { + hasRequiredIndex = true + break + } + } + if !hasRequiredIndex { + return nil, fmt.Errorf( + "snapshot %s contains only lifecycle-managed data-stream backing indices; "+ + "restore completion cannot be determined safely", + snapshot.Snapshot, + ) } return expected, nil @@ -183,22 +211,61 @@ func expectedRestoredIndices(esClient es.Interface, appCtx *app.Context, reposit // cluster with fewer nodes than replicas keeps them unassigned forever, so requiring green would // never complete there. type restoreProgress struct { - // indicesRestored counts expected indices with every primary shard active, and decides completion. + // indicesRestored counts present expected indices with every primary shard active. indicesRestored int // primariesActive counts active primary shards, and drives stall detection. Whole indices are // too coarse for that: a large multi-shard index can restore for a long time without finishing. primariesActive int + // requiredExpected and requiredRestored track indices that ILM cannot remove. + requiredExpected int + requiredRestored int + // lifecycleExpected and lifecyclePresent track data-stream backing indices. ILM can remove an + // oldest contiguous prefix while the restore is running. + lifecycleExpected int + lifecyclePresent int + lifecycleMissing int + lifecycleGap bool } -func measureRestore(expected []string, health map[string]es.IndexHealth) restoreProgress { +func measureRestore(expected []string, datastreamPrefix string, health map[string]es.IndexHealth) restoreProgress { var progress restoreProgress + var lifecycleIndices []string for _, index := range expected { + if isDatastreamBackingIndex(index, datastreamPrefix) { + lifecycleIndices = append(lifecycleIndices, index) + continue + } + + progress.requiredExpected++ indexHealth, exists := health[index] if !exists { continue } + progress.primariesActive += indexHealth.ActivePrimaryShards + if indexHealth.NumberOfShards > 0 && indexHealth.ActivePrimaryShards == indexHealth.NumberOfShards { + progress.indicesRestored++ + progress.requiredRestored++ + } + } + + sort.Strings(lifecycleIndices) + progress.lifecycleExpected = len(lifecycleIndices) + + seenPresent := false + for _, index := range lifecycleIndices { + indexHealth, exists := health[index] + if !exists { + progress.lifecycleMissing++ + if seenPresent { + progress.lifecycleGap = true + } + continue + } + + seenPresent = true + progress.lifecyclePresent++ progress.primariesActive += indexHealth.ActivePrimaryShards if indexHealth.NumberOfShards > 0 && indexHealth.ActivePrimaryShards == indexHealth.NumberOfShards { progress.indicesRestored++ @@ -208,6 +275,30 @@ func measureRestore(expected []string, health map[string]es.IndexHealth) restore return progress } +func isDatastreamBackingIndex(index, datastreamPrefix string) bool { + return strings.HasPrefix(index, datastreamPrefix+"-") +} + +func (p restoreProgress) complete() bool { + if p.requiredExpected == 0 { + return false + } + + if p.requiredRestored != p.requiredExpected { + return false + } + + // Required indices prove the restore's cluster-state update has applied; target index metadata + // is created together before shard recovery starts. + if p.lifecycleExpected == 0 || p.lifecyclePresent == 0 { + return true + } + + // With any backing indices left, only an oldest prefix may be absent. + return !p.lifecycleGap && + p.indicesRestored == p.requiredRestored+p.lifecyclePresent +} + type indicesHealthGetter interface { GetIndicesHealth() (map[string]es.IndexHealth, error) } @@ -219,10 +310,12 @@ func newRestoreStatusFn( esClient indicesHealthGetter, log *logger.Logger, expected []string, + datastreamPrefix string, noProgressTimeout time.Duration, maxErrors int, ) func() (string, bool, error) { lastPrimariesActive := -1 + lastLifecyclePresent := -1 lastProgressAt := time.Now() errCount := 0 @@ -240,29 +333,49 @@ func newRestoreStatusFn( } errCount = 0 - progress := measureRestore(expected, health) - if progress.indicesRestored == len(expected) { + progress := measureRestore(expected, datastreamPrefix, health) + if progress.complete() { + if progress.lifecycleMissing > 0 { + indexWord := "indices" + if progress.lifecycleMissing == 1 { + indexWord = "index" + } + log.Infof( + "Restore complete; %d oldest data-stream backing %s no longer present, consistent with lifecycle retention", + progress.lifecycleMissing, indexWord, + ) + } return es.StatusSuccess, true, nil } - if progress.primariesActive != lastPrimariesActive { + lifecycleRemoved := lastLifecyclePresent >= 0 && progress.lifecyclePresent < lastLifecyclePresent + lastLifecyclePresent = progress.lifecyclePresent + + if lifecycleRemoved && progress.primariesActive < lastPrimariesActive { + // An ILM deletion makes totals before and after this poll incomparable. Rebaseline + // without treating the deletion as progress or failing before a later shard increase + // can be observed. + lastPrimariesActive = progress.primariesActive + } + + if progress.primariesActive > lastPrimariesActive { lastPrimariesActive = progress.primariesActive lastProgressAt = time.Now() - } else if noProgressTimeout > 0 && time.Since(lastProgressAt) > noProgressTimeout { + } else if !lifecycleRemoved && noProgressTimeout > 0 && time.Since(lastProgressAt) > noProgressTimeout { // Deliberately not reported as a failed restore: all this establishes is that no progress // was observed from here. Restarting a restore deletes every STS index first, so a caller // that reads this as "failed" and retries would destroy a restore that is merely slow. return "", false, fmt.Errorf( "elasticsearch restore stalled: no primary shard finished restoring in %s "+ - "(%d of %d indices complete, %d primaries active); "+ + "(%d of %d indices present and complete, %d lifecycle-managed indices absent, %d primaries active); "+ "it may still be running server-side, so check before restarting it", - noProgressTimeout, progress.indicesRestored, len(expected), progress.primariesActive, + noProgressTimeout, progress.indicesRestored, len(expected), progress.lifecycleMissing, progress.primariesActive, ) } log.Debugf( - "Restored %d of %d indices (%d primaries active)", - progress.indicesRestored, len(expected), progress.primariesActive, + "Restored %d of %d indices (%d lifecycle-managed indices absent, %d primaries active)", + progress.indicesRestored, len(expected), progress.lifecycleMissing, progress.primariesActive, ) return es.StatusInProgress, false, nil @@ -291,7 +404,14 @@ func checkAndFinalize( maxErrors = 1 } - statusFn := newRestoreStatusFn(healthClient, appCtx.Logger, expected, noProgressTimeout, maxErrors) + statusFn := newRestoreStatusFn( + healthClient, + appCtx.Logger, + expected, + appCtx.Config.Elasticsearch.Restore.DatastreamIndexPrefix, + noProgressTimeout, + maxErrors, + ) // Get restore status appCtx.Logger.Infof("Checking restore status for snapshot: %s (%d indices)", snapshotName, len(expected)) diff --git a/cmd/elasticsearch/check_and_finalize_test.go b/cmd/elasticsearch/check_and_finalize_test.go index b55ae03..ad1c185 100644 --- a/cmd/elasticsearch/check_and_finalize_test.go +++ b/cmd/elasticsearch/check_and_finalize_test.go @@ -2,13 +2,16 @@ package elasticsearch import ( "errors" + "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stackvista/stackstate-backup-cli/internal/app" es "github.com/stackvista/stackstate-backup-cli/internal/clients/elasticsearch" + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" "github.com/stackvista/stackstate-backup-cli/internal/foundation/logger" ) @@ -23,6 +26,15 @@ type fakeHealthClient struct { calls int } +type snapshotOnlyClient struct { + es.Interface + snapshot *es.Snapshot +} + +func (c *snapshotOnlyClient) GetSnapshot(_, _ string) (*es.Snapshot, error) { + return c.snapshot, nil +} + func (f *fakeHealthClient) GetIndicesHealth() (map[string]es.IndexHealth, error) { result := f.results[min(f.calls, len(f.results)-1)] f.calls++ @@ -38,6 +50,8 @@ func restored(shards int) es.IndexHealth { return es.IndexHealth{Status: "green", NumberOfShards: shards, ActivePrimaryShards: shards} } +const testDatastreamPrefix = ".ds-sts_k8s_logs" + func TestMeasureRestore(t *testing.T) { tests := []struct { name string @@ -94,6 +108,22 @@ func TestMeasureRestore(t *testing.T) { wantIndices: 2, wantPrimaries: 5, }, + { + name: "oldest lifecycle-managed indices can be absent", + expected: []string{ + ".ds-sts_k8s_logs-000001", + ".ds-sts_k8s_logs-000002", + ".ds-sts_k8s_logs-000003", + "sts_topology", + }, + health: map[string]es.IndexHealth{ + ".ds-sts_k8s_logs-000002": restored(2), + ".ds-sts_k8s_logs-000003": restored(2), + "sts_topology": restored(1), + }, + wantIndices: 3, + wantPrimaries: 5, + }, { name: "index reporting zero shards is not counted", expected: []string{"sts_topology"}, @@ -105,7 +135,7 @@ func TestMeasureRestore(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - progress := measureRestore(tt.expected, tt.health) + progress := measureRestore(tt.expected, testDatastreamPrefix, tt.health) assert.Equal(t, tt.wantIndices, progress.indicesRestored, "indicesRestored") assert.Equal(t, tt.wantPrimaries, progress.primariesActive, "primariesActive") }) @@ -121,7 +151,7 @@ func TestNewRestoreStatusFn_Progress(t *testing.T) { t.Run("accepted but not yet applied reports in progress", func(t *testing.T) { client := &fakeHealthClient{results: []healthResult{ok(map[string]es.IndexHealth{})}} - status, isComplete, err := newRestoreStatusFn(client, log, expected, time.Minute, 5)() + status, isComplete, err := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 5)() require.NoError(t, err) assert.Equal(t, es.StatusInProgress, status) @@ -133,7 +163,7 @@ func TestNewRestoreStatusFn_Progress(t *testing.T) { ok(map[string]es.IndexHealth{"sts_topology": restored(1)}), ok(map[string]es.IndexHealth{"sts_topology": restored(1), "sts_events": restored(1)}), }} - statusFn := newRestoreStatusFn(client, log, expected, time.Minute, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 5) status, isComplete, err := statusFn() require.NoError(t, err) @@ -147,6 +177,144 @@ func TestNewRestoreStatusFn_Progress(t *testing.T) { }) } +func TestNewRestoreStatusFn_ILMRemovedOldestBackingIndices(t *testing.T) { + expected := []string{ + "sts_topology", + ".ds-sts_k8s_logs-2026.08.28-012011", + ".ds-sts_k8s_logs-2026.08.26-011965", + ".ds-sts_k8s_logs-2026.08.27-011999", + } + health := map[string]es.IndexHealth{ + "sts_topology": restored(1), + ".ds-sts_k8s_logs-2026.08.27-011999": restored(1), + ".ds-sts_k8s_logs-2026.08.28-012011": restored(1), + } + + status, isComplete, err := restoreStatus(expected, health) + + require.NoError(t, err) + assert.Equal(t, es.StatusSuccess, status) + assert.True(t, isComplete) +} + +func TestNewRestoreStatusFn_ILMRemovedAllBackingIndices(t *testing.T) { + expected := []string{"sts_topology", ".ds-sts_k8s_logs-2026.08.26-011965"} + health := map[string]es.IndexHealth{"sts_topology": restored(1)} + + status, isComplete, err := restoreStatus(expected, health) + + require.NoError(t, err) + assert.Equal(t, es.StatusSuccess, status) + assert.True(t, isComplete) +} + +func TestNewRestoreStatusFn_UnsafeMissingIndices(t *testing.T) { + tests := []struct { + name string + expected []string + health map[string]es.IndexHealth + }{ + { + name: "ordinary index", + expected: []string{"sts_topology", "sts_events", ".ds-sts_k8s_logs-000001"}, + health: map[string]es.IndexHealth{ + "sts_topology": restored(1), + ".ds-sts_k8s_logs-000001": restored(1), + }, + }, + { + name: "backing index after a present generation", + expected: []string{ + "sts_topology", + ".ds-sts_k8s_logs-000001", + ".ds-sts_k8s_logs-000002", + ".ds-sts_k8s_logs-000003", + }, + health: map[string]es.IndexHealth{ + "sts_topology": restored(1), + ".ds-sts_k8s_logs-000001": restored(1), + ".ds-sts_k8s_logs-000003": restored(1), + }, + }, + { + name: "newest backing index", + expected: []string{"sts_topology", ".ds-sts_k8s_logs-000001", ".ds-sts_k8s_logs-000002"}, + health: map[string]es.IndexHealth{ + "sts_topology": restored(1), + ".ds-sts_k8s_logs-000001": restored(1), + }, + }, + { + name: "snapshot has no required anchor", + expected: []string{".ds-sts_k8s_logs-000001"}, + health: map[string]es.IndexHealth{".ds-sts_k8s_logs-000001": restored(1)}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status, isComplete, err := restoreStatus(tt.expected, tt.health) + + require.NoError(t, err) + assert.Equal(t, es.StatusInProgress, status) + assert.False(t, isComplete) + }) + } +} + +func TestExpectedRestoredIndices_RejectsLifecycleOnlySnapshot(t *testing.T) { + client := &snapshotOnlyClient{snapshot: &es.Snapshot{ + Snapshot: "snapshot", + Indices: []string{".ds-sts_k8s_logs-2026.08.28-012011"}, + }} + appCtx := &app.Context{Config: &config.Config{ + Elasticsearch: config.ElasticsearchConfig{Restore: config.RestoreConfig{ + IndexPrefix: "sts", + DatastreamIndexPrefix: testDatastreamPrefix, + IndicesPattern: "sts*,.ds-sts_k8s_logs*", + }}, + }} + + _, err := expectedRestoredIndices(client, appCtx, "repository", "snapshot") + + require.Error(t, err) + assert.ErrorContains(t, err, "only lifecycle-managed data-stream backing indices") + assert.ErrorContains(t, err, "cannot be determined safely") +} + +func TestRestorableSnapshotIndices_AppliesPatternAndExcludesSiblingDatastream(t *testing.T) { + restoreConfig := config.RestoreConfig{ + IndexPrefix: "sts", + DatastreamIndexPrefix: testDatastreamPrefix, + IndicesPattern: "sts_topology,.ds-sts_k8s_logs-*", + } + snapshot := &es.Snapshot{ + Snapshot: "snapshot", + Indices: []string{ + "sts_topology", + "sts_metrics", + ".ds-sts_k8s_logs-2026.08.28-012011", + ".ds-sts_k8s_logs_archive-2026.08.28-000001", + }, + } + + indices, err := restorableSnapshotIndices(snapshot, restoreConfig) + + require.NoError(t, err) + assert.Equal(t, []string{ + "sts_topology", + ".ds-sts_k8s_logs-2026.08.28-012011", + }, indices) + assert.Equal(t, "sts_topology,.ds-sts_k8s_logs-2026.08.28-012011", strings.Join(indices, ",")) +} + +func restoreStatus(expected []string, health map[string]es.IndexHealth) (string, bool, error) { + client := &fakeHealthClient{results: []healthResult{ok(health)}} + log := logger.New(true, false) + + return newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 5)() +} + func TestNewRestoreStatusFn_Deadline(t *testing.T) { expected := statusFnExpected log := logger.New(true, false) @@ -155,7 +323,7 @@ func TestNewRestoreStatusFn_Deadline(t *testing.T) { client := &fakeHealthClient{results: []healthResult{ ok(map[string]es.IndexHealth{"sts_topology": restored(1)}), }} - statusFn := newRestoreStatusFn(client, log, expected, time.Nanosecond, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Nanosecond, 5) // First call records the progress it observed; the second sees no change past the deadline. _, isComplete, err := statusFn() @@ -179,7 +347,7 @@ func TestNewRestoreStatusFn_Deadline(t *testing.T) { ok(map[string]es.IndexHealth{"sts_topology": {NumberOfShards: 5, ActivePrimaryShards: 2}}), ok(map[string]es.IndexHealth{"sts_topology": {NumberOfShards: 5, ActivePrimaryShards: 3}}), }} - statusFn := newRestoreStatusFn(client, log, expected, time.Nanosecond, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Nanosecond, 5) for range 3 { status, isComplete, err := statusFn() @@ -194,7 +362,7 @@ func TestNewRestoreStatusFn_Deadline(t *testing.T) { ok(map[string]es.IndexHealth{}), ok(map[string]es.IndexHealth{"sts_topology": restored(1)}), }} - statusFn := newRestoreStatusFn(client, log, expected, time.Nanosecond, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Nanosecond, 5) _, _, err := statusFn() require.NoError(t, err) @@ -205,11 +373,48 @@ func TestNewRestoreStatusFn_Deadline(t *testing.T) { assert.False(t, isComplete) }) + t.Run("decreasing primaries do not reset the deadline", func(t *testing.T) { + client := &fakeHealthClient{results: []healthResult{ + ok(map[string]es.IndexHealth{"sts_topology": {NumberOfShards: 4, ActivePrimaryShards: 3}}), + ok(map[string]es.IndexHealth{"sts_topology": {NumberOfShards: 4, ActivePrimaryShards: 2}}), + }} + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Nanosecond, 5) + + _, _, err := statusFn() + require.NoError(t, err) + + _, isComplete, err := statusFn() + require.Error(t, err) + assert.False(t, isComplete) + assert.ErrorContains(t, err, "stalled") + }) + t.Run("zero waits indefinitely", func(t *testing.T) { client := &fakeHealthClient{results: []healthResult{ ok(map[string]es.IndexHealth{"sts_topology": restored(1)}), }} - statusFn := newRestoreStatusFn(client, log, expected, 0, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, 0, 5) + + for range 3 { + status, isComplete, err := statusFn() + require.NoError(t, err) + assert.Equal(t, es.StatusInProgress, status) + assert.False(t, isComplete) + } + }) +} + +func TestNewRestoreStatusFn_ILMDeletionRebasesHighWaterMark(t *testing.T) { + expected := []string{"sts_topology", ".ds-sts_k8s_logs-2026.08.28-012011"} + + t.Run("later shard progress can exceed the rebased total", func(t *testing.T) { + client := &fakeHealthClient{results: []healthResult{ + restoreHealth(3, true), + restoreHealth(5, false), + restoreHealth(6, false), + restoreHealth(6, false), + }} + statusFn := newRestoreStatusFn(client, logger.New(true, false), expected, testDatastreamPrefix, time.Nanosecond, 5) for range 3 { status, isComplete, err := statusFn() @@ -217,9 +422,44 @@ func TestNewRestoreStatusFn_Deadline(t *testing.T) { assert.Equal(t, es.StatusInProgress, status) assert.False(t, isComplete) } + + _, isComplete, err := statusFn() + require.Error(t, err) + assert.False(t, isComplete) + assert.ErrorContains(t, err, "stalled") + }) + + t.Run("the deletion itself does not reset the deadline", func(t *testing.T) { + client := &fakeHealthClient{results: []healthResult{ + restoreHealth(3, true), + restoreHealth(5, false), + restoreHealth(5, false), + }} + statusFn := newRestoreStatusFn(client, logger.New(true, false), expected, testDatastreamPrefix, time.Nanosecond, 5) + + for range 2 { + _, _, err := statusFn() + require.NoError(t, err) + } + + _, isComplete, err := statusFn() + require.Error(t, err) + assert.False(t, isComplete) + assert.ErrorContains(t, err, "stalled") }) } +func restoreHealth(topologyPrimaries int, includeBackingIndex bool) healthResult { + health := map[string]es.IndexHealth{ + "sts_topology": {NumberOfShards: 10, ActivePrimaryShards: topologyPrimaries}, + } + if includeBackingIndex { + health[".ds-sts_k8s_logs-2026.08.28-012011"] = restored(8) + } + + return ok(health) +} + func TestNewRestoreStatusFn_TransientErrors(t *testing.T) { expected := statusFnExpected log := logger.New(true, false) @@ -230,7 +470,7 @@ func TestNewRestoreStatusFn_TransientErrors(t *testing.T) { {err: errors.New("connection refused")}, ok(map[string]es.IndexHealth{"sts_topology": restored(1), "sts_events": restored(1)}), }} - statusFn := newRestoreStatusFn(client, log, expected, time.Minute, 5) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 5) for range 2 { status, isComplete, err := statusFn() @@ -247,7 +487,7 @@ func TestNewRestoreStatusFn_TransientErrors(t *testing.T) { t.Run("errors surface once they stop being transient", func(t *testing.T) { client := &fakeHealthClient{results: []healthResult{{err: errors.New("connection refused")}}} - statusFn := newRestoreStatusFn(client, log, expected, time.Minute, 3) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 3) for range 2 { _, _, err := statusFn() @@ -266,7 +506,7 @@ func TestNewRestoreStatusFn_TransientErrors(t *testing.T) { {err: errors.New("blip")}, {err: errors.New("blip")}, }} - statusFn := newRestoreStatusFn(client, log, expected, time.Minute, 3) + statusFn := newRestoreStatusFn(client, log, expected, testDatastreamPrefix, time.Minute, 3) for range 4 { _, _, err := statusFn() diff --git a/cmd/elasticsearch/restore.go b/cmd/elasticsearch/restore.go index 77868d4..4c25f98 100644 --- a/cmd/elasticsearch/restore.go +++ b/cmd/elasticsearch/restore.go @@ -2,6 +2,7 @@ package elasticsearch import ( "fmt" + "path" "sort" "strings" "time" @@ -96,6 +97,13 @@ func runRestore(appCtx *app.Context) error { return err } + // Reject snapshots that cannot be monitored safely before scaling down workloads or deleting + // any indices. check-and-finalize repeats this validation for independently resumed operations. + restoreIndices, err := restorableSnapshotIndices(snapshotDetails, appCtx.Config.Elasticsearch.Restore) + if err != nil { + return err + } + // Confirm with user before starting destructive operation if !skipConfirmation { appCtx.Logger.Println() @@ -137,7 +145,7 @@ func runRestore(appCtx *app.Context) error { appCtx.Logger.Println() isPartial := snapshotDetails.State == "PARTIAL" appCtx.Logger.Infof("Triggering restore for snapshot: %s", selectedSnapshot) - if err := esClient.RestoreSnapshot(repository, selectedSnapshot, appCtx.Config.Elasticsearch.Restore.IndicesPattern, isPartial); err != nil { + if err := esClient.RestoreSnapshot(repository, selectedSnapshot, strings.Join(restoreIndices, ","), isPartial); err != nil { return fmt.Errorf("failed to trigger restore: %w", err) } appCtx.Logger.Successf("Restore triggered successfully") @@ -211,6 +219,10 @@ func deleteAllSTSIndices(esClient es.Interface, appCtx *app.Context) error { } stsIndices := filterSTSIndices(allIndices, appCtx.Config.Elasticsearch.Restore.IndexPrefix, appCtx.Config.Elasticsearch.Restore.DatastreamIndexPrefix) + stsIndices, err = filterIndicesByPattern(stsIndices, appCtx.Config.Elasticsearch.Restore.IndicesPattern) + if err != nil { + return fmt.Errorf("invalid Elasticsearch restore indicesPattern: %w", err) + } if len(stsIndices) == 0 { appCtx.Logger.Infof("No STS indices found to delete") @@ -246,17 +258,79 @@ func deleteAllSTSIndices(esClient es.Interface, appCtx *app.Context) error { func filterSTSIndices(allIndices []string, indexPrefix, datastreamPrefix string) []string { var stsIndices []string for _, index := range allIndices { - if strings.HasPrefix(index, indexPrefix) || strings.HasPrefix(index, datastreamPrefix) { + if strings.HasPrefix(index, indexPrefix) || isDatastreamBackingIndex(index, datastreamPrefix) { stsIndices = append(stsIndices, index) } } return stsIndices } +func filterIndicesByPattern(indices []string, expression string) ([]string, error) { + type pattern struct { + value string + exclude bool + } + + var patterns []pattern + hasInclude := false + for _, value := range strings.Split(expression, ",") { + value = strings.TrimSpace(value) + if value == "" { + return nil, fmt.Errorf("contains an empty pattern") + } + + exclude := strings.HasPrefix(value, "-") + if exclude { + value = strings.TrimPrefix(value, "-") + if value == "" { + return nil, fmt.Errorf("contains an empty exclusion") + } + } else { + hasInclude = true + } + + if value == "_all" { + value = "*" + } + if _, err := path.Match(value, ""); err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", value, err) + } + patterns = append(patterns, pattern{value: value, exclude: exclude}) + } + if !hasInclude { + return nil, fmt.Errorf("must contain at least one inclusion pattern") + } + + var matched []string + for _, index := range indices { + included := false + excluded := false + for _, candidate := range patterns { + isMatch, err := path.Match(candidate.value, index) + if err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", candidate.value, err) + } + if !isMatch { + continue + } + if candidate.exclude { + excluded = true + } else { + included = true + } + } + if included && !excluded { + matched = append(matched, index) + } + } + + return matched, nil +} + // hasDatastreamIndices checks if any indices belong to a datastream func hasDatastreamIndices(indices []string, datastreamPrefix string) bool { for _, index := range indices { - if strings.HasPrefix(index, datastreamPrefix+"-") { + if isDatastreamBackingIndex(index, datastreamPrefix) { return true } } diff --git a/cmd/elasticsearch/restore_test.go b/cmd/elasticsearch/restore_test.go index e611100..33531e5 100644 --- a/cmd/elasticsearch/restore_test.go +++ b/cmd/elasticsearch/restore_test.go @@ -90,6 +90,10 @@ func (m *mockESClientForRestore) ConfigureSnapshotRepository(_, _, _, _, _, _ st return fmt.Errorf("not implemented") } +func (m *mockESClientForRestore) DeleteSnapshotRepository(_ string) error { + return fmt.Errorf("not implemented") +} + func (m *mockESClientForRestore) ConfigureSLMPolicy(_, _, _, _, _, _ string, _, _ int) error { return fmt.Errorf("not implemented") } @@ -180,6 +184,17 @@ func TestFilterSTSIndices(t *testing.T) { expectedCount: 2, expectedIndices: []string{"sts_k8s_logs-000001", "sts_k8s_logs-000002"}, }, + { + name: "sibling datastream is not included", + allIndices: []string{ + ".ds-sts_k8s_logs-2026.08.28-012011", + ".ds-sts_k8s_logs_archive-2026.08.28-000001", + }, + indexPrefix: "sts", + datastreamPrefix: ".ds-sts_k8s_logs", + expectedCount: 1, + expectedIndices: []string{".ds-sts_k8s_logs-2026.08.28-012011"}, + }, } for _, tt := range tests { @@ -196,6 +211,90 @@ func TestFilterSTSIndices(t *testing.T) { } } +func TestFilterIndicesByPattern(t *testing.T) { + indices := []string{ + "sts_topology", + "sts_metrics", + ".ds-sts_k8s_logs-2026.08.28-012011", + } + + tests := []struct { + name string + expression string + expected []string + expectErr string + }{ + { + name: "narrowed restore", + expression: "sts_topology,.ds-sts_k8s_logs-*", + expected: []string{ + "sts_topology", + ".ds-sts_k8s_logs-2026.08.28-012011", + }, + }, + { + name: "exclusion", + expression: "sts*,-sts_metrics", + expected: []string{"sts_topology"}, + }, + { + name: "all", + expression: "_all", + expected: indices, + }, + { + name: "invalid pattern", + expression: "sts[", + expectErr: "invalid pattern", + }, + { + name: "exclusion only", + expression: "-sts_metrics", + expectErr: "at least one inclusion", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := filterIndicesByPattern(indices, tt.expression) + + if tt.expectErr != "" { + require.ErrorContains(t, err, tt.expectErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestDeleteAllSTSIndicesHonorsIndicesPattern(t *testing.T) { + client := &mockESClientForRestore{ + indices: []string{"sts_topology", "sts_metrics", "other"}, + indexExistsMap: map[string]bool{ + "sts_topology": true, + "sts_metrics": true, + }, + } + appCtx := &app.Context{ + Config: &config.Config{ + Elasticsearch: config.ElasticsearchConfig{ + Restore: config.RestoreConfig{ + IndexPrefix: "sts", + DatastreamIndexPrefix: ".ds-sts_k8s_logs", + IndicesPattern: "sts_topology", + }, + }, + }, + Logger: logger.New(true, false), + } + + err := deleteAllSTSIndices(client, appCtx) + + require.NoError(t, err) + assert.Equal(t, []string{"sts_topology"}, client.deletedIndices) +} + // TestHasDatastreamIndices tests datastream detection func TestHasDatastreamIndices(t *testing.T) { tests := []struct { diff --git a/go.mod b/go.mod index 10543f2..04f2e61 100644 --- a/go.mod +++ b/go.mod @@ -80,7 +80,7 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.55.0 // indirect + golang.org/x/crypto v0.56.0 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sys v0.47.0 // indirect diff --git a/go.sum b/go.sum index dade845..9cb0984 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=