Fix whole-block plan misclassification via bidirectional rule matching

.nextchanges/bundles/6589.md

@@ -0,0 +1 @@
+* direct: Fix pipelines recreation when the whole `ingestion_definition` block is added or removed. ([#6589](https://github.com/databricks/cli/pull/6589))

acceptance/bundle/resources/pipelines/recreate-add-whole-block/out.plan.direct.json

@@ -5,7 +5,7 @@
   "serial": 1,
   "plan": {
     "resources.pipelines.my": {
-      "action": "update",
+      "action": "recreate",
       "new_state": {
         "value": {
           "channel": "CURRENT",
@@ -57,7 +57,8 @@
       },
       "changes": {
         "ingestion_definition": {
-          "action": "update",
+          "action": "recreate",
+          "reason": "immutable",
           "new": {
             "connection_name": "my_connection",
             "objects": [

acceptance/bundle/resources/pipelines/recreate-add-whole-block/script

@@ -9,9 +9,9 @@ trace $CLI bundle deploy
 # markers. connection_name is immutable, so adding it must recreate the pipeline.
 trace update_file.py databricks.yml "#TO_ADD: " ""
 
-# The direct engine records the whole ingestion_definition block as one 'update'
-# entry (missing the recreate rule on the connection_name leaf); terraform records
-# 'recreate'. See the per-engine "changes" / "action" in the plan JSON.
+# Both engines recreate. structdiff records the added block as one change at
+# "ingestion_definition"; bidirectional rule matching relates that to the recreate
+# rule on the connection_name leaf. See the per-engine "changes"/"action" in the JSON.
 $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json
 
 rm -f out.requests.txt

acceptance/bundle/resources/pipelines/recreate-add-whole-block/test.toml

@@ -1,5 +1,5 @@
-# Runs on both engines to show the difference: the direct engine plans this as
-# 'update' (see Badness), while terraform correctly recreates. The plan verdict
-# is the only divergent output, so it is routed to per-engine out.plan.*.txt.
-Badness = "On the direct engine, adding the whole ingestion_definition block is planned as 'update' instead of 'recreate': structdiff records one change at path 'ingestion_definition' instead of descending to 'ingestion_definition.connection_name', so the immutable recreate rule never matches. Terraform recreates as expected."
+# Runs on both engines: both recreate (connection_name is immutable). The direct
+# plan's per-engine "changes" map (out.plan.direct.json) shows the recreate keyed on
+# the ingestion_definition block via bidirectional rule matching; terraform's plan
+# JSON just carries the action.
 Ignore = ["foo.py", ".databricks"]

acceptance/bundle/resources/pipelines/recreate-remove-whole-block/out.plan.direct.json

@@ -5,7 +5,7 @@
   "serial": 1,
   "plan": {
     "resources.pipelines.my": {
-      "action": "update",
+      "action": "recreate",
       "new_state": {
         "value": {
           "channel": "CURRENT",
@@ -57,7 +57,8 @@
       },
       "changes": {
         "ingestion_definition": {
-          "action": "update",
+          "action": "recreate",
+          "reason": "immutable",
           "old": {
             "connection_name": "my_connection",
             "objects": [

acceptance/bundle/resources/pipelines/recreate-remove-whole-block/script

@@ -9,9 +9,9 @@ trace $CLI bundle deploy
 # lines. connection_name is immutable, so removing it must recreate the pipeline.
 grep -v '# TO_DELETE' databricks.yml > databricks.yml.tmp && mv databricks.yml.tmp databricks.yml
 
-# The direct engine records the whole ingestion_definition block as one 'update'
-# entry (missing the recreate rule on the connection_name leaf); terraform records
-# 'recreate'. See the per-engine "changes" / "action" in the plan JSON.
+# Both engines recreate. structdiff records the removed block as one change at
+# "ingestion_definition"; bidirectional rule matching relates that to the recreate
+# rule on the connection_name leaf. See the per-engine "changes"/"action" in the JSON.
 $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json
 
 rm -f out.requests.txt

acceptance/bundle/resources/pipelines/recreate-remove-whole-block/test.toml

@@ -1,5 +1,5 @@
-# Runs on both engines to show the difference: the direct engine plans this as
-# 'update' (see Badness), while terraform correctly recreates. The plan verdict
-# is the only divergent output, so it is routed to per-engine out.plan.*.txt.
-Badness = "On the direct engine, removing the whole ingestion_definition block is planned as 'update' instead of 'recreate': structdiff records one change at path 'ingestion_definition' instead of descending to 'ingestion_definition.connection_name', so the immutable recreate rule never matches. Terraform recreates as expected."
+# Runs on both engines: both recreate (connection_name is immutable). The direct
+# plan's per-engine "changes" map (out.plan.direct.json) shows the recreate keyed on
+# the ingestion_definition block via bidirectional rule matching; terraform's plan
+# JSON just carries the action.
 Ignore = ["foo.py", ".databricks"]

bundle/direct/bundle_plan.go

@@ -482,10 +482,10 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change
 		} else if isFieldMissingInRemote(adapter, path) && structdiff.IsEqual(ch.Old, ch.New) {
 			ch.Action = deployplan.Skip
 			ch.Reason = deployplan.ReasonMissingInRemote
-		} else if reason, ok := findMatchingRule(path, cfg.RecreateOnChanges); ok {
+		} else if reason, ok := findMatchingRuleBidirectional(path, cfg.RecreateOnChanges); ok {
 			ch.Action = deployplan.Recreate
 			ch.Reason = reason
-		} else if reason, ok := findMatchingRule(path, generatedCfg.RecreateOnChanges); ok {
+		} else if reason, ok := findMatchingRuleBidirectional(path, generatedCfg.RecreateOnChanges); ok {
 			ch.Action = deployplan.Recreate
 			ch.Reason = reason
 		} else {
@@ -557,6 +557,32 @@ func findMatchingRule(path *structpath.PathNode, rules []dresources.FieldRule) (
 	return "", false
 }
 
+// findMatchingRuleBidirectional matches rules in both directions: the usual
+// descendant match, plus a rule on foo.bar matching a change recorded at foo,
+// because a whole block added or removed is one block-level change and the field
+// the rule names is part of it. Callers must only use this for escalating actions
+// (currently recreate): a whole block that merely contains a leaf named by a
+// suppressing rule (ignore_remote/ignore_local, backend_default, normalize) is
+// still a real change, so those keep the descendant-only findMatchingRule.
+func findMatchingRuleBidirectional(path *structpath.PathNode, rules []dresources.FieldRule) (string, bool) {
+	for _, r := range rules {
+		if matchesFieldRuleBidirectional(path, r.Field) {
+			return r.Reason, true
+		}
+	}
+	return "", false
+}
+
+func matchesFieldRuleBidirectional(path *structpath.PathNode, pattern *structpath.PatternNode) bool {
+	if path.HasPatternPrefix(pattern) {
+		return true
+	}
+	if path.Len() < pattern.Len() {
+		return path.HasPatternPrefix(pattern.Prefix(path.Len()))
+	}
+	return false
+}
+
 func shouldSkip(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, ch *deployplan.ChangeDesc) (string, bool) {
 	if cfg == nil {
 		return "", false