From 942c2e4fbd16d2a6f224b9819320bc97566318b7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 17:58:53 +0200 Subject: [PATCH 01/25] Do not report cluster fields supplied by a cluster policy as drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes - New `ignore_remote_additions` rule in `bundle/direct/dresources/resources.yml`: an object plus a `when_set` field that gates it. Inside a gated object, a field absent from both state and config but present in the remote is skipped with reason `policy_managed`. A disagreement between config and remote, and a field the user removed from config, still report an update. - The gate is `policy_id`, applied to every place a cluster spec appears: `clusters`, `jobs.tasks[*].new_cluster`, `jobs.tasks[*].for_each_task.task.new_cluster`, `jobs.job_clusters[*].new_cluster` and `pipelines.clusters[*]`. - `libs/testserver` now applies cluster policies: `fixed` elements always, `defaultValue` elements only when the request sets `apply_policy_default_values`, never overriding a value the request supplied. Without this the new tests would pass locally for the wrong reason. - `jobFixUps` dropped `apply_policy_default_values` by zeroing it but left it in `ForceSendFields`, so it serialized as an explicit `false` where the Jobs API returns nothing. Fixed. ## Why A cluster policy supplies settings server-side, so the remote spec is a superset of what the bundle declares and every deploy planned an update that never converged. The gate is `policy_id` rather than `apply_policy_default_values`, because `fixed` policy elements are applied whether or not that flag is set — both reporters hit the bug with the flag unset. `acceptance/bundle/resources/cluster_policies/policy_value_semantics` records the measured backend behaviour for all four combinations. Fixes #5179 Fixes #6512 ## Tests - `policy_value_semantics` pins {`fixed`, `defaultValue`} x {flag, no flag} against a real workspace; `fixed_values_applied` shows a `fixed` element supplying `spark_version` and a tag the config never declares; `fixed_value_conflict` shows it rejecting a contradicting value with 400. - `policy_drift` covers the classification matrix in one golden: policy addition skipped, out-of-band change to a config-owned tag still an update, removal from config still an update, no `policy_id` still an update. - `policy_no_drift_variants` covers all five cluster-spec locations. - Unit tests for the classifier and for the testserver policy application; a config test validates every rule's pattern and gate against the state type. Co-authored-by: Isaac --- .../bundles/cluster-policy-no-drift.md | 1 + .../fixed_value_conflict/databricks.yml.tmpl | 36 ++++ .../fixed_value_conflict/hello_world.py | 1 + .../fixed_value_conflict/out.test.toml | 3 + .../fixed_value_conflict/output.txt | 23 +++ .../fixed_value_conflict/script | 9 + .../fixed_value_conflict/test.toml | 5 + .../fixed_values_applied/databricks.yml.tmpl | 38 ++++ .../fixed_values_applied/hello_world.py | 1 + .../fixed_values_applied/out.test.toml | 3 + .../fixed_values_applied/output.txt | 30 ++++ .../fixed_values_applied/script | 20 +++ .../fixed_values_applied/test.toml | 5 + .../policy_drift/databricks.yml.tmpl | 68 ++++++++ .../policy_drift/hello_world.py | 1 + .../policy_drift/out.test.toml | 3 + .../cluster_policies/policy_drift/output.txt | 64 +++++++ .../cluster_policies/policy_drift/script | 35 ++++ .../cluster_policies/policy_drift/test.toml | 5 + .../databricks.yml.tmpl | 68 ++++++++ .../policy_no_drift_variants/hello_world.py | 1 + .../policy_no_drift_variants/out.test.toml | 3 + .../policy_no_drift_variants/output.txt | 85 +++++++++ .../policy_no_drift_variants/script | 17 ++ .../policy_no_drift_variants/test.toml | 5 + .../databricks.yml.tmpl | 76 ++++++++ .../policy_value_semantics/hello_world.py | 1 + .../policy_value_semantics/out.test.toml | 3 + .../policy_value_semantics/output.txt | 53 ++++++ .../policy_value_semantics/script | 21 +++ .../policy_value_semantics/test.toml | 5 + bundle/deployplan/plan.go | 4 + bundle/direct/bundle_plan.go | 48 +++++- bundle/direct/bundle_plan_test.go | 110 +++++++++++- bundle/direct/dresources/config.go | 33 +++- bundle/direct/dresources/config_test.go | 31 ++++ bundle/direct/dresources/resources.yml | 31 ++++ libs/testserver/cluster_policies.go | 163 ++++++++++++++++++ libs/testserver/cluster_policies_test.go | 86 +++++++++ libs/testserver/clusters.go | 6 + libs/testserver/jobs.go | 19 +- libs/testserver/pipelines.go | 2 + 42 files changed, 1207 insertions(+), 15 deletions(-) create mode 100644 .nextchanges/bundles/cluster-policy-no-drift.md create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_value_conflict/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_value_conflict/hello_world.py create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_value_conflict/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_value_conflict/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_value_conflict/script create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_value_conflict/test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_values_applied/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_values_applied/hello_world.py create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_values_applied/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_values_applied/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_values_applied/script create mode 100644 acceptance/bundle/resources/cluster_policies/fixed_values_applied/test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/policy_drift/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/cluster_policies/policy_drift/hello_world.py create mode 100644 acceptance/bundle/resources/cluster_policies/policy_drift/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/policy_drift/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/policy_drift/script create mode 100644 acceptance/bundle/resources/cluster_policies/policy_drift/test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/hello_world.py create mode 100644 acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script create mode 100644 acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/policy_value_semantics/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/cluster_policies/policy_value_semantics/hello_world.py create mode 100644 acceptance/bundle/resources/cluster_policies/policy_value_semantics/out.test.toml create mode 100644 acceptance/bundle/resources/cluster_policies/policy_value_semantics/output.txt create mode 100644 acceptance/bundle/resources/cluster_policies/policy_value_semantics/script create mode 100644 acceptance/bundle/resources/cluster_policies/policy_value_semantics/test.toml create mode 100644 libs/testserver/cluster_policies_test.go diff --git a/.nextchanges/bundles/cluster-policy-no-drift.md b/.nextchanges/bundles/cluster-policy-no-drift.md new file mode 100644 index 00000000000..3bbcce1ef73 --- /dev/null +++ b/.nextchanges/bundles/cluster-policy-no-drift.md @@ -0,0 +1 @@ +* `bundle plan` no longer reports a permanent update on cluster fields supplied by a cluster policy; a field the config never declares is no longer drift when the cluster spec has a `policy_id`. diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/databricks.yml.tmpl new file mode 100644 index 00000000000..c80fed9f81c --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/databricks.yml.tmpl @@ -0,0 +1,36 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + pol: + name: test-policy-$UNIQUE_NAME + definition: |- + { + "custom_tags.CostCenter": { + "type": "fixed", + "value": "from-fixed" + } + } + + jobs: + # The config sets the fixed attribute to a value the policy forbids. + j: + name: test-job-$UNIQUE_NAME + job_clusters: + - job_cluster_key: small + new_cluster: + policy_id: ${resources.cluster_policies.pol.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + CostCenter: not-what-the-policy-says + tasks: + - task_key: t + job_cluster_key: small + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/hello_world.py b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/hello_world.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/hello_world.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/out.test.toml b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/out.test.toml new file mode 100644 index 00000000000..ae5c7bd798f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/output.txt b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/output.txt new file mode 100644 index 00000000000..097e5a7e98e --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/output.txt @@ -0,0 +1,23 @@ + +=== Deploy a cluster whose config contradicts a fixed policy value + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Error: cannot create resources.jobs.j: Cluster validation error: Validation failed for custom_tags, CostCenter must be from-fixed (is "not-what-the-policy-says") (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Cluster validation error: Validation failed for custom_tags, CostCenter must be from-fixed (is "not-what-the-policy-says") + +Files: 6 uploaded, 0 deleted + +Exit code: 1 + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/script b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/script new file mode 100644 index 00000000000..bf454016241 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/script @@ -0,0 +1,9 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy a cluster whose config contradicts a fixed policy value\n" +errcode trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/test.toml b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/test.toml new file mode 100644 index 00000000000..ae35e046923 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/test.toml @@ -0,0 +1,5 @@ +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/databricks.yml.tmpl new file mode 100644 index 00000000000..8fb232538eb --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/databricks.yml.tmpl @@ -0,0 +1,38 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + pol: + name: test-policy-$UNIQUE_NAME + definition: |- + { + "spark_version": { + "type": "fixed", + "value": "$DEFAULT_SPARK_VERSION" + }, + "custom_tags.CostCenter": { + "type": "fixed", + "value": "policy-supplied" + } + } + + jobs: + # new_cluster deliberately omits spark_version and custom_tags: the policy is + # expected to supply both. apply_policy_default_values is NOT set. + j: + name: test-job-$UNIQUE_NAME + job_clusters: + - job_cluster_key: small + new_cluster: + policy_id: ${resources.cluster_policies.pol.id} + node_type_id: $NODE_TYPE_ID + num_workers: 1 + tasks: + - task_key: t + job_cluster_key: small + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/hello_world.py b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/hello_world.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/hello_world.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/out.test.toml b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/out.test.toml new file mode 100644 index 00000000000..ae5c7bd798f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/output.txt b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/output.txt new file mode 100644 index 00000000000..7ef978595af --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/output.txt @@ -0,0 +1,30 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.pol +Created jobs.j +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Did the fixed policy SUPPLY spark_version and custom_tags, or only validate them? +The bundle declares neither, and does not set apply_policy_default_values. +{ + "spark_version": "13.3.x-snapshot-scala2.12", + "custom_tags": { + "CostCenter": "policy-supplied" + } +} + +=== Is the policy-supplied value reported as drift? + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/script b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/script new file mode 100644 index 00000000000..2a8f815ddac --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +# Mask the server-assigned policy id (real backend returns a hex id the built-in +# UUID replacement misses). +read_id.py pol > /dev/null +job_id=$(read_id.py j) + +title "Did the fixed policy SUPPLY spark_version and custom_tags, or only validate them?\n" +echo "The bundle declares neither, and does not set apply_policy_default_values." +$CLI jobs get "$job_id" | jq '.settings.job_clusters[0].new_cluster | {spark_version, custom_tags}' + +title "Is the policy-supplied value reported as drift?\n" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/test.toml b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/test.toml new file mode 100644 index 00000000000..ae35e046923 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/test.toml @@ -0,0 +1,5 @@ +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/policy_drift/databricks.yml.tmpl new file mode 100644 index 00000000000..1c988bcdb8f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_drift/databricks.yml.tmpl @@ -0,0 +1,68 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + pol: + name: test-policy-$UNIQUE_NAME + definition: |- + { + "custom_tags.CostCenter": { + "type": "fixed", + "value": "from-policy" + } + } + + jobs: + # policy attached: the tag the policy adds must not be drift. + with_policy: + name: test-with-policy-$UNIQUE_NAME + job_clusters: + - job_cluster_key: small + new_cluster: + policy_id: ${resources.cluster_policies.pol.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + tasks: + - task_key: t + job_cluster_key: small + spark_python_task: + python_file: ./hello_world.py + + # policy attached, and the config owns a tag of its own. An out-of-band change to that + # tag must still be reported: a policy suppresses additions, never disagreements. + owned_tag: + name: test-owned-tag-$UNIQUE_NAME + job_clusters: + - job_cluster_key: small + new_cluster: + policy_id: ${resources.cluster_policies.pol.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + Mine: mine + tasks: + - task_key: t + job_cluster_key: small + spark_python_task: + python_file: ./hello_world.py + + # no policy attached: a remote-only tag is still drift. edit_resource.py injects it below. + no_policy: + name: test-no-policy-$UNIQUE_NAME + job_clusters: + - job_cluster_key: small + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + tasks: + - task_key: t + job_cluster_key: small + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/hello_world.py b/acceptance/bundle/resources/cluster_policies/policy_drift/hello_world.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_drift/hello_world.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_drift/out.test.toml new file mode 100644 index 00000000000..ae5c7bd798f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_drift/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/output.txt b/acceptance/bundle/resources/cluster_policies/policy_drift/output.txt new file mode 100644 index 00000000000..2242def2f4c --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_drift/output.txt @@ -0,0 +1,64 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.pol +Created jobs.no_policy +Created jobs.owned_tag +Created jobs.with_policy +Files: 6 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +=== A tag the cluster policy added is not drift + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +=== Without a policy_id the same remote-only tag IS drift + +>>> [CLI] bundle plan +update jobs.no_policy + +Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged + +=== A policy does not mask a change to a tag the config owns + +>>> [CLI] bundle plan +update jobs.no_policy +update jobs.owned_tag + +Plan: 0 to add, 2 to change, 0 to delete, 2 unchanged + +=== Removing a tag from config is a change the user asked for, not an addition + +>>> update_file.py databricks.yml Mine: mine Other: other + +>>> [CLI] bundle plan -o json +[ + { + "job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter']": { + "action": "skip", + "reason": "policy_managed", + "remote": "from-policy" + }, + "job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine']": { + "action": "update", + "old": "mine", + "remote": "changed-out-of-band" + }, + "job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Other']": { + "action": "update", + "new": "other" + } + } +] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + delete resources.jobs.no_policy + delete resources.jobs.owned_tag + delete resources.jobs.with_policy + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/script b/acceptance/bundle/resources/cluster_policies/policy_drift/script new file mode 100644 index 00000000000..5d927fd49ab --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_drift/script @@ -0,0 +1,35 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +read_id.py pol > /dev/null +no_policy_id="$(read_id.py no_policy)" +owned_tag_id="$(read_id.py owned_tag)" + +title "A tag the cluster policy added is not drift\n" +trace $CLI bundle plan | contains.py "0 to change" + +title "Without a policy_id the same remote-only tag IS drift\n" +edit_resource.py jobs "$no_policy_id" <<'EOF' +for jc in r["job_clusters"]: + jc["new_cluster"]["custom_tags"] = {"CostCenter": "added-out-of-band"} +EOF +trace $CLI bundle plan | contains.py "1 to change" + +title "A policy does not mask a change to a tag the config owns\n" +edit_resource.py jobs "$owned_tag_id" <<'EOF' +for jc in r["job_clusters"]: + jc["new_cluster"]["custom_tags"]["Mine"] = "changed-out-of-band" +EOF +trace $CLI bundle plan | contains.py "2 to change" + +title "Removing a tag from config is a change the user asked for, not an addition\n" +trace update_file.py databricks.yml "Mine: mine" "Other: other" +trace $CLI bundle plan -o json | jq -S ' + [.plan | to_entries[] + | select(.key | endswith("owned_tag")) + | .value.changes | with_entries(select(.key | contains("custom_tags")))]' diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/test.toml b/acceptance/bundle/resources/cluster_policies/policy_drift/test.toml new file mode 100644 index 00000000000..ae35e046923 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_drift/test.toml @@ -0,0 +1,5 @@ +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl new file mode 100644 index 00000000000..f60e0f83a2b --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl @@ -0,0 +1,68 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + pol: + name: test-policy-$UNIQUE_NAME + definition: |- + { + "custom_tags.CostCenter": { + "type": "fixed", + "value": "from-policy" + } + } + + # Every place a cluster spec can appear must be covered by a rule; a gap here is how + # https://github.com/databricks/cli/issues/5179 stayed open after a partial fix. + clusters: + standalone: + cluster_name: test-cluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + policy_id: ${resources.cluster_policies.pol.id} + + jobs: + task_cluster: + name: test-task-cluster-$UNIQUE_NAME + tasks: + - task_key: t + new_cluster: + policy_id: ${resources.cluster_policies.pol.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + spark_python_task: + python_file: ./hello_world.py + + for_each_cluster: + name: test-for-each-cluster-$UNIQUE_NAME + tasks: + - task_key: outer + for_each_task: + inputs: "[1,2]" + task: + task_key: inner + new_cluster: + policy_id: ${resources.cluster_policies.pol.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + spark_python_task: + python_file: ./hello_world.py + + pipelines: + pipe: + name: test-pipeline-$UNIQUE_NAME + clusters: + - label: default + policy_id: ${resources.cluster_policies.pol.id} + node_type_id: $NODE_TYPE_ID + num_workers: 1 + libraries: + - file: + path: ./hello_world.py diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/hello_world.py b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/hello_world.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/hello_world.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml new file mode 100644 index 00000000000..ae5c7bd798f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt new file mode 100644 index 00000000000..1960a2d1f89 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt @@ -0,0 +1,85 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.pol +Created clusters.standalone +Created jobs.for_each_cluster +Created jobs.task_cluster +Created pipelines.pipe +Files: 6 uploaded, 0 deleted +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged + +=== Every cluster-spec location converges: the policy tag is not drift anywhere + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged + +=== Confirm the tag really is present remotely and was classified, not just absent + +>>> [CLI] bundle plan -o json +[ + { + "node": "resources.clusters.standalone", + "tags": { + "custom_tags": { + "action": "skip", + "reason": "policy_managed", + "remote": { + "CostCenter": "from-policy" + } + } + } + }, + { + "node": "resources.jobs.for_each_cluster", + "tags": { + "tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags": { + "action": "skip", + "reason": "policy_managed", + "remote": { + "CostCenter": "from-policy" + } + } + } + }, + { + "node": "resources.jobs.task_cluster", + "tags": { + "tasks[task_key='t'].new_cluster.custom_tags": { + "action": "skip", + "reason": "policy_managed", + "remote": { + "CostCenter": "from-policy" + } + } + } + }, + { + "node": "resources.pipelines.pipe", + "tags": { + "clusters[0].custom_tags": { + "action": "skip", + "reason": "policy_managed", + "remote": { + "CostCenter": "from-policy" + } + } + } + } +] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + delete resources.clusters.standalone + delete resources.jobs.for_each_cluster + delete resources.jobs.task_cluster + delete resources.pipelines.pipe + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.pipe + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 5 deleted diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script new file mode 100644 index 00000000000..c12e8caaf7f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Every cluster-spec location converges: the policy tag is not drift anywhere\n" +trace $CLI bundle plan + +title "Confirm the tag really is present remotely and was classified, not just absent\n" +trace $CLI bundle plan -o json | jq -S ' + [.plan | to_entries[] + | {node: .key, tags: ((.value.changes // {}) | with_entries(select(.key | contains("custom_tags"))))} + | select(.tags | length > 0)]' diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml new file mode 100644 index 00000000000..ae35e046923 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml @@ -0,0 +1,5 @@ +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/databricks.yml.tmpl new file mode 100644 index 00000000000..747138e1661 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/databricks.yml.tmpl @@ -0,0 +1,76 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + # A "fixed" element. Docs say it "limits the attribute to the specified value", + # which is validation language; whether it also SUPPLIES the value to a request + # that omits the attribute is what this test pins down. + fixed: + name: test-policy-fixed-$UNIQUE_NAME + definition: |- + { + "custom_tags.FixedTag": { + "type": "fixed", + "value": "from-fixed" + } + } + + # A "defaultValue" element. Documented to apply only when the request sets + # apply_policy_default_values=true. + defaulted: + name: test-policy-default-$UNIQUE_NAME + definition: |- + { + "custom_tags.DefaultTag": { + "type": "unlimited", + "defaultValue": "from-default", + "isOptional": true + } + } + + jobs: + j: + name: test-job-$UNIQUE_NAME + job_clusters: + # A: fixed element, apply_policy_default_values NOT set + - job_cluster_key: a_fixed_noflag + new_cluster: + policy_id: ${resources.cluster_policies.fixed.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + + # B: defaultValue element, apply_policy_default_values NOT set + - job_cluster_key: b_default_noflag + new_cluster: + policy_id: ${resources.cluster_policies.defaulted.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + + # C: defaultValue element, apply_policy_default_values = true + - job_cluster_key: c_default_flag + new_cluster: + policy_id: ${resources.cluster_policies.defaulted.id} + apply_policy_default_values: true + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + + # D: fixed element, apply_policy_default_values = true + - job_cluster_key: d_fixed_flag + new_cluster: + policy_id: ${resources.cluster_policies.fixed.id} + apply_policy_default_values: true + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + tasks: + - task_key: t + job_cluster_key: a_fixed_noflag + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/hello_world.py b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/hello_world.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/hello_world.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/out.test.toml new file mode 100644 index 00000000000..ae5c7bd798f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/output.txt b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/output.txt new file mode 100644 index 00000000000..30a4ff46b97 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/output.txt @@ -0,0 +1,53 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.defaulted +Created cluster_policies.fixed +Created jobs.j +Files: 6 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Which policy elements did the backend SUPPLY into each job_cluster? +[ + { + "apply_policy_default_values": null, + "custom_tags": { + "FixedTag": "from-fixed" + }, + "job_cluster_key": "a_fixed_noflag" + }, + { + "apply_policy_default_values": null, + "custom_tags": null, + "job_cluster_key": "b_default_noflag" + }, + { + "apply_policy_default_values": null, + "custom_tags": { + "DefaultTag": "from-default" + }, + "job_cluster_key": "c_default_flag" + }, + { + "apply_policy_default_values": null, + "custom_tags": { + "FixedTag": "from-fixed" + }, + "job_cluster_key": "d_fixed_flag" + } +] + +=== Are the supplied values reported as drift? + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.defaulted + delete resources.cluster_policies.fixed + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/script b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/script new file mode 100644 index 00000000000..300ee0780c6 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/script @@ -0,0 +1,21 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +read_id.py fixed > /dev/null +read_id.py defaulted > /dev/null +job_id=$(read_id.py j) + +title "Which policy elements did the backend SUPPLY into each job_cluster?\n" +$CLI jobs get "$job_id" | jq -S '[.settings.job_clusters[] + | {job_cluster_key, + apply_policy_default_values: .new_cluster.apply_policy_default_values, + custom_tags: .new_cluster.custom_tags}]' + +title "Are the supplied values reported as drift?\n" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/test.toml b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/test.toml new file mode 100644 index 00000000000..ae35e046923 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/test.toml @@ -0,0 +1,5 @@ +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index c9f5bec3516..3ffbb50ae27 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -147,6 +147,10 @@ const ( // ReasonMissingInRemote: field is not present in RemoteType (write-only / input-only). // Remote always appears nil, so treat the absence as a no-op when there is no local change. ReasonMissingInRemote = "missing_in_remote" + // ReasonPolicyManaged: the field is inside a cluster spec that has a policy_id, and the + // config never declared it. A cluster policy supplies values server-side, so the backend + // legitimately extends the spec beyond what the bundle asks for; that is not drift. + ReasonPolicyManaged = "policy_managed" // Special reason that results in removing this change from the plan ReasonDrop = "!drop" diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 739ca3bd8e7..0ab971dfc1e 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -281,7 +281,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } - err = addPerFieldActions(ctx, adapter, entry.Changes, remoteState) + err = addPerFieldActions(ctx, adapter, entry.Changes, sv.Value, remoteState) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) return false @@ -382,7 +382,7 @@ func prepareChanges(ctx context.Context, adapter *dresources.Adapter, localDiff, return m, nil } -func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, changes deployplan.Changes, remoteState any) error { +func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, changes deployplan.Changes, newState, remoteState any) error { cfg := adapter.ResourceConfig() generatedCfg := adapter.GeneratedResourceConfig() @@ -418,6 +418,9 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change } else if action, reason, ok := classifyIDField(generatedCfg, path, ch); ok { ch.Action = action ch.Reason = reason + } else if reason, ok := shouldSkipRemoteAddition(cfg, path, ch, newState); ok { + ch.Action = deployplan.Skip + ch.Reason = reason } else if reason, ok := shouldSkipBackendDefault(cfg, path, ch); ok { ch.Action = deployplan.Skip ch.Reason = reason @@ -571,6 +574,47 @@ func shouldSkipNormalized(cfg *dresources.ResourceLifecycleConfig, path *structp return "", false } +// shouldSkipRemoteAddition skips a field the backend added to an object it co-owns. +// +// It fires only on an addition: absent from both old state and new config, present in the +// remote. A disagreement between config and remote (New != nil) is left alone and still +// reports an update, and so does a field the user removed from config (Old != nil) — that +// is a deletion the user asked for, not a backend addition. +// +// The rule is gated on a field within the same object (ignore_remote_additions.when_set). +// For cluster specs that gate is policy_id: an attached cluster policy supplies values +// server-side — "fixed" elements always, "defaultValue" elements when the request sets +// apply_policy_default_values — so the remote spec is legitimately a superset of what the +// bundle declares. See acceptance/bundle/resources/cluster_policies/policy_value_semantics +// for the measured backend behavior. +// +// Suppressed values are never echoed back on write: an update sends the config spec as-is +// and the backend re-supplies the policy values. +func shouldSkipRemoteAddition(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, ch *deployplan.ChangeDesc, newState any) (string, bool) { + if cfg == nil || ch.Old != nil || ch.New != nil || ch.Remote == nil { + return "", false + } + for _, rule := range cfg.IgnoreRemoteAdditions { + if !path.HasPatternPrefix(rule.Field) { + continue + } + gate := structpath.NewDotString(path.Prefix(rule.Field.Len()), rule.WhenSet) + value, err := structaccess.Get(newState, gate) + if err != nil { + // The gated object is absent from the config entirely (e.g. the remote grew a + // whole new_cluster the bundle does not declare), so there is no policy to gate + // on and the addition is real drift. Rule typos cannot reach here: the patterns + // are validated against the state type by TestResourcesYMLRemoteAdditionGates. + continue + } + if allEmpty(value) { + continue + } + return deployplan.ReasonPolicyManaged, true + } + return "", false +} + // shouldSkipBackendDefault checks if a change should be skipped because the remote value // is a known backend default. Applies when old and new are nil but remote is set. // If the rule has allowed values, the remote value must match one of them. diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index bf875680c2b..ba75e1953ed 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" @@ -256,7 +257,7 @@ func TestRemoteAlreadySetGuards(t *testing.T) { adapter, ok := adapters[tt.resource] require.True(t, ok) changes := deployplan.Changes{tt.field: tt.ch} - err := addPerFieldActions(t.Context(), adapter, changes, nil) + err := addPerFieldActions(t.Context(), adapter, changes, nil, nil) require.NoError(t, err) assert.Equal(t, tt.expectedAction, tt.ch.Action) if tt.expectedReason != "" { @@ -323,7 +324,7 @@ func jobRunResultStateAction(t *testing.T, state *jobs.RunState) *deployplan.Cha Remote: state.ResultState, }} - require.NoError(t, addPerFieldActions(t.Context(), adapters["job_runs"], changes, remote)) + require.NoError(t, addPerFieldActions(t.Context(), adapters["job_runs"], changes, nil, remote)) return changes["result_state"] } @@ -401,3 +402,108 @@ func bundleWithSkippedJobRun(t *testing.T, remote *dresources.JobRunRemote) *Dep b.RemoteStateCache.Store(jobRunKey, remote) return b } + +func TestShouldSkipRemoteAddition(t *testing.T) { + // Rules mirror clusters/jobs ignore_remote_additions in resources.yml, but the test is + // deliberately self-contained so edits to resources.yml don't break it. The real wiring + // is covered by acceptance/bundle/resources/cluster_policies/*. + jobCluster, err := structpath.ParsePattern("job_clusters[*].new_cluster") + require.NoError(t, err) + cfg := &dresources.ResourceLifecycleConfig{ + IgnoreRemoteAdditions: []dresources.RemoteAdditionRule{ + {Field: jobCluster, WhenSet: "policy_id"}, + }, + } + + withPolicy := &jobs.JobSettings{JobClusters: []jobs.JobCluster{{ + JobClusterKey: "small", + NewCluster: &compute.ClusterSpec{PolicyId: "p1"}, + }}} + withoutPolicy := &jobs.JobSettings{JobClusters: []jobs.JobCluster{{ + JobClusterKey: "small", + NewCluster: &compute.ClusterSpec{}, + }}} + + const tagPath = "job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter']" + + tests := []struct { + name string + path string + state *jobs.JobSettings + change deployplan.ChangeDesc + expected bool + }{ + { + name: "policy attached, backend added a tag", + path: tagPath, + state: withPolicy, + change: deployplan.ChangeDesc{Remote: "dev-1234"}, + expected: true, + }, + { + name: "policy attached, whole map added by backend", + path: "job_clusters[job_cluster_key='small'].new_cluster.custom_tags", + state: withPolicy, + change: deployplan.ChangeDesc{Remote: map[string]string{"CostCenter": "dev-1234"}}, + expected: true, + }, + { + name: "no policy attached: an addition is still drift", + path: tagPath, + state: withoutPolicy, + change: deployplan.ChangeDesc{Remote: "dev-1234"}, + expected: false, + }, + { + name: "config disagrees with remote: still an update", + path: tagPath, + state: withPolicy, + change: deployplan.ChangeDesc{New: "mine", Remote: "dev-1234"}, + expected: false, + }, + { + name: "user removed the field from config: still an update", + path: tagPath, + state: withPolicy, + change: deployplan.ChangeDesc{Old: "mine", Remote: "dev-1234"}, + expected: false, + }, + { + name: "remote has nothing: not an addition", + path: tagPath, + state: withPolicy, + change: deployplan.ChangeDesc{}, + expected: false, + }, + { + name: "outside the gated object: not covered", + path: "tags['CostCenter']", + state: withPolicy, + change: deployplan.ChangeDesc{Remote: "dev-1234"}, + expected: false, + }, + { + // The remote grew a whole cluster the config does not declare: there is no + // policy_id to gate on, so this is real drift rather than a policy addition. + name: "gated object absent from config: still drift", + path: "job_clusters[job_cluster_key='other'].new_cluster.custom_tags['CostCenter']", + state: withPolicy, + change: deployplan.ChangeDesc{Remote: "dev-1234"}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path, err := structpath.ParsePath(tt.path) + require.NoError(t, err) + + change := tt.change + reason, ok := shouldSkipRemoteAddition(cfg, path, &change, tt.state) + assert.Equal(t, tt.expected, ok) + if tt.expected { + assert.Equal(t, deployplan.ReasonPolicyManaged, reason) + } + }) + } +} diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 33a00e0a2f7..0fab4e24e8b 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -45,6 +45,17 @@ func (b *BackendDefaultRule) UnmarshalYAML(unmarshal func(any) error) error { return nil } +// RemoteAdditionRule marks a sub-object whose contents the backend co-owns whenever the +// object's WhenSet field is set. Inside such an object, a field the config never declared +// coming back set from the remote is an addition by the backend, not drift. +// +// Field is a prefix pattern selecting the object (omitted = the resource root); WhenSet is +// the name of a field within that object which gates the rule. +type RemoteAdditionRule struct { + Field *structpath.PatternNode `yaml:"field"` + WhenSet string `yaml:"when_set"` +} + // ResourceLifecycleConfig defines lifecycle behavior for a resource type. type ResourceLifecycleConfig struct { // IgnoreRemoteChanges: field patterns where remote changes are ignored (output-only, policy-set). @@ -76,6 +87,11 @@ type ResourceLifecycleConfig struct { // A change is skipped when local and remote differ only by trailing slashes. NormalizeSlash []FieldRule `yaml:"normalize_slash,omitempty"` + // IgnoreRemoteAdditions: objects whose fields the backend may add to when a gate field + // is set. A field that is absent from both old and new state but present in the remote + // is skipped; a disagreement between config and remote is still an update. + IgnoreRemoteAdditions []RemoteAdditionRule `yaml:"ignore_remote_additions,omitempty"` + // BackendDefaults: fields where the backend may set defaults. // When old and new are nil but remote is set, and the remote value matches allowed values (if specified), the change is skipped. BackendDefaults []BackendDefaultRule `yaml:"backend_defaults,omitempty"` @@ -96,14 +112,15 @@ var resourcesYAML []byte var resourcesGeneratedYAML []byte var empty = ResourceLifecycleConfig{ - IgnoreRemoteChanges: nil, - IgnoreLocalChanges: nil, - RecreateOnChanges: nil, - ProvidedIDFields: nil, - UpdatableIDFields: nil, - NormalizeSlash: nil, - BackendDefaults: nil, - SensitiveFields: nil, + IgnoreRemoteChanges: nil, + IgnoreLocalChanges: nil, + RecreateOnChanges: nil, + ProvidedIDFields: nil, + UpdatableIDFields: nil, + NormalizeSlash: nil, + IgnoreRemoteAdditions: nil, + BackendDefaults: nil, + SensitiveFields: nil, } func mustParseConfig(data []byte) func() *Config { diff --git a/bundle/direct/dresources/config_test.go b/bundle/direct/dresources/config_test.go index 33345306b0b..1bcae5f2e27 100644 --- a/bundle/direct/dresources/config_test.go +++ b/bundle/direct/dresources/config_test.go @@ -148,3 +148,34 @@ func TestResourcesYMLActionCategoriesExclusive(t *testing.T) { } } } + +// TestResourcesYMLRemoteAdditionGates validates every ignore_remote_additions rule against +// the resource's state type: the field pattern must resolve, and so must the when_set gate +// relative to it. Without this a typo produces a rule that silently never matches. +func TestResourcesYMLRemoteAdditionGates(t *testing.T) { + for resourceType, rc := range MustLoadConfig().Resources { + adapter, err := NewAdapter(SupportedResources[resourceType], resourceType, nil) + require.NoError(t, err) + + for _, rule := range rc.IgnoreRemoteAdditions { + require.NotEmpty(t, rule.WhenSet, "%s: ignore_remote_additions entry %q has no when_set", resourceType, rule.Field.String()) + + gate := rule.Field + if !gate.IsRoot() { + assert.NoError(t, structaccess.ValidatePattern(adapter.StateType(), gate), + "%s: ignore_remote_additions field %q does not resolve in the state type", resourceType, gate.String()) + } + gatePattern, err := structpath.ParsePattern(joinPattern(gate, rule.WhenSet)) + require.NoError(t, err) + assert.NoError(t, structaccess.ValidatePattern(adapter.StateType(), gatePattern), + "%s: ignore_remote_additions when_set %q does not resolve under %q", resourceType, rule.WhenSet, gate.String()) + } + } +} + +func joinPattern(prefix *structpath.PatternNode, field string) string { + if prefix.IsRoot() { + return field + } + return prefix.String() + "." + field +} diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 044fbc4fddb..4e457457756 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -85,6 +85,21 @@ resources: - field: triggers[*].table_update.condition reason: input_only + ignore_remote_additions: + # A cluster policy supplies cluster settings server-side: "fixed" elements always, + # "defaultValue" elements when the request sets apply_policy_default_values. The bundle + # is not the source for those, so the remote spec is legitimately a superset of what the + # config declares and an added field is not drift. Measured backend behavior is pinned by + # acceptance/bundle/resources/cluster_policies/policy_value_semantics. + # https://github.com/databricks/cli/issues/5179 + # https://github.com/databricks/cli/issues/6512 + - field: tasks[*].new_cluster + when_set: policy_id + - field: tasks[*].for_each_task.task.new_cluster + when_set: policy_id + - field: job_clusters[*].new_cluster + when_set: policy_id + backend_defaults: # Same as clusters.enable_elastic_disk — see clusters/resource_cluster.go#L331 # s.SchemaPath("enable_elastic_disk").SetComputed() @@ -199,6 +214,12 @@ resources: - reason: immutable pipelines: + # Same as clusters/jobs: a cluster policy supplies settings server-side, so a pipeline + # cluster the config never declared coming back set is not drift. + ignore_remote_additions: + - field: clusters[*] + when_set: policy_id + recreate_on_changes: - field: storage reason: immutable @@ -593,6 +614,16 @@ resources: - field: effective_value clusters: + # A cluster policy supplies cluster settings server-side: "fixed" elements always, + # "defaultValue" elements when the request sets apply_policy_default_values. The bundle + # is not the source for those, so the remote spec is legitimately a superset of what the + # config declares and an added field is not drift. Measured backend behavior is pinned by + # acceptance/bundle/resources/cluster_policies/policy_value_semantics. + # https://github.com/databricks/cli/issues/5179 + # https://github.com/databricks/cli/issues/6512 + ignore_remote_additions: + - when_set: policy_id + ignore_remote_changes: # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/clusters/resource_cluster.go#L361-L363 # s.SchemaPath("aws_attributes").SetSuppressDiff() diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go index 13c48f0770e..ce01d04cc5f 100644 --- a/libs/testserver/cluster_policies.go +++ b/libs/testserver/cluster_policies.go @@ -4,8 +4,11 @@ import ( "encoding/json" "fmt" "slices" + "strings" "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" ) // policyFamilyDefinition mimics the real backend: a policy created from a policy @@ -123,3 +126,163 @@ func (s *FakeWorkspace) ClusterPoliciesDelete(req Request) any { return Response{} } + +// policyElement is the part of a cluster policy element the fake applies. The backend +// supports more element types, but only these two fields materialize a value: +// "fixed" elements set Value, every limiting type ("allowlist", "blocklist", "regex", +// "range", "unlimited") may carry DefaultValue. "forbidden" only rejects and is ignored here. +type policyElement struct { + Type string `json:"type"` + Value any `json:"value"` + DefaultValue any `json:"defaultValue"` +} + +// effectiveValue returns the value this element materializes into a spec that omits the +// attribute, or nil if it materializes none. Mirrors the backend: "fixed" applies whether +// or not the request sets apply_policy_default_values, "defaultValue" only when it does. +// Pinned against a real workspace by +// acceptance/bundle/resources/cluster_policies/policy_value_semantics. +func (e policyElement) effectiveValue(applyDefaults bool) any { + if e.Type == "fixed" { + return e.Value + } + if applyDefaults { + return e.DefaultValue + } + return nil +} + +// clusterPolicyValues returns the values the policy supplies, keyed by the policy's +// attribute path (e.g. "spark_version", "custom_tags.CostCenter"). +func (s *FakeWorkspace) clusterPolicyValues(policyID string, applyDefaults bool) map[string]any { + policy, ok := s.ClusterPolicies[policyID] + if !ok { + return nil + } + var elements map[string]policyElement + if err := json.Unmarshal([]byte(policy.Definition), &elements); err != nil { + return nil + } + values := make(map[string]any, len(elements)) + for path, element := range elements { + if value := element.effectiveValue(applyDefaults); value != nil { + values[path] = value + } + } + return values +} + +// applyClusterPolicy fills in attributes the request omitted from the policy attached via +// policyID. Callers must hold the workspace lock. +// +// spec is a pointer to any struct with cluster-spec JSON tags (compute.ClusterDetails, +// compute.ClusterSpec, pipelines.PipelineCluster); the policy's attribute paths are applied +// against its JSON shape so one implementation covers all three. Attributes already present +// are left alone — a policy value never overrides what the request supplied (the real +// backend rejects a conflicting "fixed" value instead, which is not modeled here). +func (s *FakeWorkspace) applyClusterPolicy(spec any, policyID string, applyDefaults bool) { + if policyID == "" { + return + } + values := s.clusterPolicyValues(policyID, applyDefaults) + if len(values) == 0 { + return + } + + raw, err := json.Marshal(spec) + if err != nil { + return + } + // doc is only read, to test whether an attribute is already present; the spec itself is + // updated from patch below, so decoded numbers are never written back. + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + return + } + + // Collect only the attributes the request omitted, then unmarshal just those back onto + // the spec. Unmarshaling the whole document instead would rebuild ForceSendFields from + // every key present, which makes fields the backend omits (e.g. the Jobs API dropping + // apply_policy_default_values) serialize as explicit zeros. + patch := map[string]any{} + for path, value := range values { + setIfAbsent(doc, patch, strings.Split(path, "."), value) + } + if len(patch) == 0 { + return + } + + if raw, err = json.Marshal(patch); err == nil { + _ = json.Unmarshal(raw, spec) + } +} + +// setIfAbsent records value in patch at the given key path, unless doc already has an +// attribute there — a policy value never overrides what the request supplied. doc mirrors +// the current spec; patch collects only what is missing from it. +func setIfAbsent(doc, patch map[string]any, path []string, value any) { + for _, key := range path[:len(path)-1] { + child, ok := doc[key].(map[string]any) + if !ok { + if _, exists := doc[key]; exists { + // Attribute exists but is not an object: the policy path does not apply. + return + } + child = map[string]any{} + } + doc = child + + next, ok := patch[key].(map[string]any) + if !ok { + next = map[string]any{} + patch[key] = next + } + patch = next + } + leaf := path[len(path)-1] + if _, exists := doc[leaf]; exists { + return + } + patch[leaf] = value +} + +// applyPolicyDefaultValues reads apply_policy_default_values from a raw cluster request body. +// compute.ClusterDetails has no such field, so it cannot be read off the decoded request. +func applyPolicyDefaultValues(body []byte) bool { + var spec compute.ClusterSpec + if err := json.Unmarshal(body, &spec); err != nil { + return false + } + return spec.ApplyPolicyDefaultValues +} + +// applyJobClusterPolicies applies attached cluster policies to every cluster spec a job can +// carry. Callers must hold the workspace lock. +func (s *FakeWorkspace) applyJobClusterPolicies(settings *jobs.JobSettings) { + for i := range settings.JobClusters { + s.applyClusterSpecPolicy(settings.JobClusters[i].NewCluster) + } + for i := range settings.Tasks { + task := &settings.Tasks[i] + s.applyClusterSpecPolicy(task.NewCluster) + if task.ForEachTask != nil { + s.applyClusterSpecPolicy(task.ForEachTask.Task.NewCluster) + } + } +} + +func (s *FakeWorkspace) applyClusterSpecPolicy(spec *compute.ClusterSpec) { + if spec == nil { + return + } + s.applyClusterPolicy(spec, spec.PolicyId, spec.ApplyPolicyDefaultValues) +} + +// applyPipelineClusterPolicies applies attached cluster policies to a pipeline's clusters. +// Callers must hold the workspace lock. +func (s *FakeWorkspace) applyPipelineClusterPolicies(spec *pipelines.PipelineSpec) { + for i := range spec.Clusters { + cluster := &spec.Clusters[i] + s.applyClusterPolicy(cluster, cluster.PolicyId, cluster.ApplyPolicyDefaultValues) + } +} diff --git a/libs/testserver/cluster_policies_test.go b/libs/testserver/cluster_policies_test.go new file mode 100644 index 00000000000..c9852a942d7 --- /dev/null +++ b/libs/testserver/cluster_policies_test.go @@ -0,0 +1,86 @@ +package testserver + +import ( + "encoding/json" + "testing" + + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/stretchr/testify/assert" +) + +const testPolicyDefinition = `{ + "spark_version": {"type": "fixed", "value": "policy-version"}, + "custom_tags.Fixed": {"type": "fixed", "value": "f"}, + "custom_tags.Default": {"type": "unlimited", "defaultValue": "d"}, + "node_type_id": {"type": "forbidden"} +}` + +func policyWorkspace() *FakeWorkspace { + return &FakeWorkspace{ClusterPolicies: map[string]compute.Policy{ + "p1": {PolicyId: "p1", Definition: testPolicyDefinition}, + }} +} + +func TestApplyClusterPolicy(t *testing.T) { + tests := []struct { + name string + policyID string + spec compute.ClusterSpec + want string + wantForceSend []string + }{ + { + name: "fixed applies without apply_policy_default_values, defaultValue does not", + policyID: "p1", + spec: compute.ClusterSpec{PolicyId: "p1"}, + want: `{"custom_tags":{"Fixed":"f"},"policy_id":"p1","spark_version":"policy-version"}`, + wantForceSend: []string{"SparkVersion"}, + }, + { + name: "defaultValue applies with apply_policy_default_values", + policyID: "p1", + spec: compute.ClusterSpec{PolicyId: "p1", ApplyPolicyDefaultValues: true}, + want: `{"apply_policy_default_values":true,"custom_tags":{"Default":"d","Fixed":"f"},"policy_id":"p1","spark_version":"policy-version"}`, + wantForceSend: []string{"SparkVersion"}, + }, + { + name: "a value the request supplied is not overridden", + policyID: "p1", + spec: compute.ClusterSpec{PolicyId: "p1", SparkVersion: "user-version"}, + want: `{"custom_tags":{"Fixed":"f"},"policy_id":"p1","spark_version":"user-version"}`, + }, + { + name: "policy tags merge into tags the request supplied", + policyID: "p1", + spec: compute.ClusterSpec{PolicyId: "p1", SparkVersion: "v", CustomTags: map[string]string{"Mine": "yes"}}, + want: `{"custom_tags":{"Fixed":"f","Mine":"yes"},"policy_id":"p1","spark_version":"v"}`, + }, + { + name: "no policy attached leaves the spec untouched", + policyID: "", + spec: compute.ClusterSpec{SparkVersion: "v"}, + want: `{"spark_version":"v"}`, + }, + { + name: "unknown policy leaves the spec untouched", + policyID: "missing", + spec: compute.ClusterSpec{PolicyId: "missing", SparkVersion: "v"}, + want: `{"policy_id":"missing","spark_version":"v"}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := tt.spec + policyWorkspace().applyClusterPolicy(&spec, tt.policyID, spec.ApplyPolicyDefaultValues) + + got, err := json.Marshal(&spec) + assert.NoError(t, err) + assert.JSONEq(t, tt.want, string(got)) + // Only attributes the policy actually supplied may become force-sent. Anything + // else would serialize as an explicit zero and break the fake's ability to model + // fields the real API drops (the Jobs API drops apply_policy_default_values). + assert.Equal(t, tt.wantForceSend, spec.ForceSendFields) + }) + } +} diff --git a/libs/testserver/clusters.go b/libs/testserver/clusters.go index 80991b0091a..288ace64f25 100644 --- a/libs/testserver/clusters.go +++ b/libs/testserver/clusters.go @@ -32,6 +32,9 @@ func (s *FakeWorkspace) ClustersCreate(req Request) any { request.SingleUserName = s.CurrentUser().UserName } + // Apply the attached cluster policy before computing defaults, matching the backend: + // policy values feed the defaults below (e.g. driver_node_type_id from node_type_id). + s.applyClusterPolicy(&request, request.PolicyId, applyPolicyDefaultValues(req.Body)) clusterFixUps(&request) // The cluster GET API returns apply_policy_default_values only under .spec, not at the top @@ -113,6 +116,9 @@ func (s *FakeWorkspace) ClustersEdit(req Request) any { // Preserve runtime-only fields that the Edit API request doesn't include. request.State = existing.State request.ClusterId = existing.ClusterId + // Apply the attached cluster policy before computing defaults, matching the backend: + // policy values feed the defaults below (e.g. driver_node_type_id from node_type_id). + s.applyClusterPolicy(&request, request.PolicyId, applyPolicyDefaultValues(req.Body)) clusterFixUps(&request) // Refresh the .spec snapshot from the new settings, matching cloud behavior on edit. request.Spec = specSnapshot(req.Body) diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index 3b16c61b023..2f887290e35 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -101,6 +101,7 @@ func (s *FakeWorkspace) JobsCreate(req Request) Response { } } + s.applyJobClusterPolicies(&jobSettings) jobFixUps(&jobSettings) // CreatorUserName field is used by TF to check if the resource exists or not. CreatorUserName should be non-empty for the resource to be considered as "exists" @@ -130,6 +131,7 @@ func (s *FakeWorkspace) JobsReset(req Request) Response { defer s.LockUnlock()() + s.applyJobClusterPolicies(&request.NewSettings) jobFixUps(&request.NewSettings) jobId := request.JobId @@ -223,23 +225,34 @@ func jobFixUps(jobSettings *jobs.JobSettings) { // The real Jobs API consumes apply_policy_default_values but does not // return it in GET responses; clear it so testserver matches cloud. - task.NewCluster.ApplyPolicyDefaultValues = false + clearApplyPolicyDefaultValues(task.NewCluster) } // Handle for_each_task inner cluster. if task.ForEachTask != nil && task.ForEachTask.Task.NewCluster != nil { // Same as above: not returned in GET responses. - task.ForEachTask.Task.NewCluster.ApplyPolicyDefaultValues = false + clearApplyPolicyDefaultValues(task.ForEachTask.Task.NewCluster) } } // Handle job cluster new_clusters. for i := range jobSettings.JobClusters { // Same as above: not returned in GET responses. - jobSettings.JobClusters[i].NewCluster.ApplyPolicyDefaultValues = false + clearApplyPolicyDefaultValues(jobSettings.JobClusters[i].NewCluster) } } +// clearApplyPolicyDefaultValues drops apply_policy_default_values from a job's cluster spec. +// Zeroing the value alone is not enough: decoding the request populates ForceSendFields from +// the keys it carried, so a request that set the flag would still serialize it as an explicit +// false instead of omitting it the way the Jobs API does. +func clearApplyPolicyDefaultValues(spec *compute.ClusterSpec) { + spec.ApplyPolicyDefaultValues = false + spec.ForceSendFields = slices.DeleteFunc(spec.ForceSendFields, func(field string) bool { + return field == "ApplyPolicyDefaultValues" + }) +} + // jobsGetTasksPageSize matches the real Databricks API limit of 100 tasks per jobs.get response. // https://docs.databricks.com/api/workspace/jobs/get const jobsGetTasksPageSize = 100 diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 0aedc44c5d3..b46c438fae8 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -78,6 +78,7 @@ func (s *FakeWorkspace) PipelineCreate(req Request) Response { r.State = "IDLE" r.EffectivePublishingMode = pipelines.PublishingModeDefaultPublishingMode + s.applyPipelineClusterPolicies(&spec) setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = r @@ -137,6 +138,7 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response if edit.RunAs != nil { item.RunAs = edit.RunAs } + s.applyPipelineClusterPolicies(&spec) setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = item From 780e7eceeed456d1d92e761f733467a7443be3e7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 18:14:52 +0200 Subject: [PATCH 02/25] Match cloud: no policy expansion for pipelines, enforce fixed values Follow-up from re-recording the new tests against a real workspace. - The Pipelines API does not expand cluster policies into the stored spec: a pipeline cluster with a policy_id reads back exactly as authored. So there is nothing to suppress; the `pipelines.clusters[*]` rule is dropped and the testserver no longer applies policies there. `policy_no_drift_variants` keeps the pipeline case to record that. - `libs/testserver` now rejects a value that contradicts a `fixed` element, reproducing the backend's validation message. Without it `fixed_value_conflict` could not pass both locally and in cloud. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 3 + .../policy_no_drift_variants/output.txt | 12 -- bundle/direct/dresources/resources.yml | 6 - libs/testserver/cluster_policies.go | 116 +++++++++++------- libs/testserver/cluster_policies_test.go | 42 +++++-- libs/testserver/clusters.go | 20 ++- libs/testserver/jobs.go | 20 ++- libs/testserver/pipelines.go | 2 - 8 files changed, 143 insertions(+), 78 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl index f60e0f83a2b..d5670422794 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl @@ -55,6 +55,9 @@ resources: spark_python_task: python_file: ./hello_world.py + # A pipeline cluster carries policy_id too, but the Pipelines API does not expand the + # policy into the stored spec: it reads back exactly as authored, so there is no addition + # to suppress and no ignore_remote_additions rule for pipelines. Kept here to record that. pipelines: pipe: name: test-pipeline-$UNIQUE_NAME diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt index 1960a2d1f89..585eeb57771 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt @@ -53,18 +53,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged } } } - }, - { - "node": "resources.pipelines.pipe", - "tags": { - "clusters[0].custom_tags": { - "action": "skip", - "reason": "policy_managed", - "remote": { - "CostCenter": "from-policy" - } - } - } } ] diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 4e457457756..ce9ee2a4dcf 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -214,12 +214,6 @@ resources: - reason: immutable pipelines: - # Same as clusters/jobs: a cluster policy supplies settings server-side, so a pipeline - # cluster the config never declared coming back set is not drift. - ignore_remote_additions: - - field: clusters[*] - when_set: policy_id - recreate_on_changes: - field: storage reason: immutable diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go index ce01d04cc5f..f2871870594 100644 --- a/libs/testserver/cluster_policies.go +++ b/libs/testserver/cluster_policies.go @@ -3,12 +3,13 @@ package testserver import ( "encoding/json" "fmt" + "maps" + "reflect" "slices" "strings" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/databricks/databricks-sdk-go/service/pipelines" ) // policyFamilyDefinition mimics the real backend: a policy created from a policy @@ -173,31 +174,32 @@ func (s *FakeWorkspace) clusterPolicyValues(policyID string, applyDefaults bool) } // applyClusterPolicy fills in attributes the request omitted from the policy attached via -// policyID. Callers must hold the workspace lock. +// policyID. It returns the backend's validation message when a supplied value contradicts a +// "fixed" element, or "" when the spec is acceptable; callers must hold the workspace lock and +// surface a non-empty message as a 400. // -// spec is a pointer to any struct with cluster-spec JSON tags (compute.ClusterDetails, -// compute.ClusterSpec, pipelines.PipelineCluster); the policy's attribute paths are applied -// against its JSON shape so one implementation covers all three. Attributes already present -// are left alone — a policy value never overrides what the request supplied (the real -// backend rejects a conflicting "fixed" value instead, which is not modeled here). -func (s *FakeWorkspace) applyClusterPolicy(spec any, policyID string, applyDefaults bool) { +// spec is a pointer to any struct with cluster-spec JSON tags (compute.ClusterDetails or +// compute.ClusterSpec); the policy's attribute paths are applied against its JSON shape so +// one implementation covers both. +func (s *FakeWorkspace) applyClusterPolicy(spec any, policyID string, applyDefaults bool) string { if policyID == "" { - return + return "" } values := s.clusterPolicyValues(policyID, applyDefaults) if len(values) == 0 { - return + return "" } + // spec came from a successful decode of the request, so re-encoding it cannot fail. raw, err := json.Marshal(spec) if err != nil { - return + return "" } // doc is only read, to test whether an attribute is already present; the spec itself is // updated from patch below, so decoded numbers are never written back. var doc map[string]any if err := json.Unmarshal(raw, &doc); err != nil { - return + return "" } // Collect only the attributes the request omitted, then unmarshal just those back onto @@ -205,33 +207,58 @@ func (s *FakeWorkspace) applyClusterPolicy(spec any, policyID string, applyDefau // every key present, which makes fields the backend omits (e.g. the Jobs API dropping // apply_policy_default_values) serialize as explicit zeros. patch := map[string]any{} - for path, value := range values { - setIfAbsent(doc, patch, strings.Split(path, "."), value) + for _, path := range slices.Sorted(maps.Keys(values)) { + value := values[path] + segments := strings.Split(path, ".") + if existing, ok := lookup(doc, segments); ok { + // A "fixed" element enforces its value as well as supplying it; anything else + // only supplies a default, which the request is free to override. + if s.isFixed(policyID, path) && !reflect.DeepEqual(existing, value) { + // Message shape copied from the real backend for a nested attribute + // ("custom_tags, CostCenter must be ..."); the wording for a top-level + // attribute has not been observed, so it is only approximated here. + return fmt.Sprintf("Cluster validation error: Validation failed for %s must be %v (is %q)", + strings.Join(segments, ", "), value, existing) + } + continue + } + setPatch(patch, segments, value) } if len(patch) == 0 { - return + return "" } if raw, err = json.Marshal(patch); err == nil { _ = json.Unmarshal(raw, spec) } + return "" +} + +// isFixed reports whether the policy's element at path is a "fixed" element. +func (s *FakeWorkspace) isFixed(policyID, path string) bool { + var elements map[string]policyElement + if err := json.Unmarshal([]byte(s.ClusterPolicies[policyID].Definition), &elements); err != nil { + return false + } + return elements[path].Type == "fixed" } -// setIfAbsent records value in patch at the given key path, unless doc already has an -// attribute there — a policy value never overrides what the request supplied. doc mirrors -// the current spec; patch collects only what is missing from it. -func setIfAbsent(doc, patch map[string]any, path []string, value any) { +// lookup returns the value at the given key path in doc. +func lookup(doc map[string]any, path []string) (any, bool) { for _, key := range path[:len(path)-1] { child, ok := doc[key].(map[string]any) if !ok { - if _, exists := doc[key]; exists { - // Attribute exists but is not an object: the policy path does not apply. - return - } - child = map[string]any{} + return nil, false } doc = child + } + value, ok := doc[path[len(path)-1]] + return value, ok +} +// setPatch records value in patch at the given key path, creating intermediate maps. +func setPatch(patch map[string]any, path []string, value any) { + for _, key := range path[:len(path)-1] { next, ok := patch[key].(map[string]any) if !ok { next = map[string]any{} @@ -239,11 +266,7 @@ func setIfAbsent(doc, patch map[string]any, path []string, value any) { } patch = next } - leaf := path[len(path)-1] - if _, exists := doc[leaf]; exists { - return - } - patch[leaf] = value + patch[path[len(path)-1]] = value } // applyPolicyDefaultValues reads apply_policy_default_values from a raw cluster request body. @@ -258,31 +281,34 @@ func applyPolicyDefaultValues(body []byte) bool { // applyJobClusterPolicies applies attached cluster policies to every cluster spec a job can // carry. Callers must hold the workspace lock. -func (s *FakeWorkspace) applyJobClusterPolicies(settings *jobs.JobSettings) { +// +// The Pipelines API does not expand cluster policies into the stored spec: a pipeline cluster +// with a policy_id reads back exactly as authored, verified against a real workspace by +// acceptance/bundle/resources/cluster_policies/policy_no_drift_variants. So there is +// deliberately no pipeline equivalent. +func (s *FakeWorkspace) applyJobClusterPolicies(settings *jobs.JobSettings) string { for i := range settings.JobClusters { - s.applyClusterSpecPolicy(settings.JobClusters[i].NewCluster) + if msg := s.applyClusterSpecPolicy(settings.JobClusters[i].NewCluster); msg != "" { + return msg + } } for i := range settings.Tasks { task := &settings.Tasks[i] - s.applyClusterSpecPolicy(task.NewCluster) + if msg := s.applyClusterSpecPolicy(task.NewCluster); msg != "" { + return msg + } if task.ForEachTask != nil { - s.applyClusterSpecPolicy(task.ForEachTask.Task.NewCluster) + if msg := s.applyClusterSpecPolicy(task.ForEachTask.Task.NewCluster); msg != "" { + return msg + } } } + return "" } -func (s *FakeWorkspace) applyClusterSpecPolicy(spec *compute.ClusterSpec) { +func (s *FakeWorkspace) applyClusterSpecPolicy(spec *compute.ClusterSpec) string { if spec == nil { - return - } - s.applyClusterPolicy(spec, spec.PolicyId, spec.ApplyPolicyDefaultValues) -} - -// applyPipelineClusterPolicies applies attached cluster policies to a pipeline's clusters. -// Callers must hold the workspace lock. -func (s *FakeWorkspace) applyPipelineClusterPolicies(spec *pipelines.PipelineSpec) { - for i := range spec.Clusters { - cluster := &spec.Clusters[i] - s.applyClusterPolicy(cluster, cluster.PolicyId, cluster.ApplyPolicyDefaultValues) + return "" } + return s.applyClusterPolicy(spec, spec.PolicyId, spec.ApplyPolicyDefaultValues) } diff --git a/libs/testserver/cluster_policies_test.go b/libs/testserver/cluster_policies_test.go index c9852a942d7..83ef663d3ed 100644 --- a/libs/testserver/cluster_policies_test.go +++ b/libs/testserver/cluster_policies_test.go @@ -6,13 +6,16 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// A "fixed" element both supplies and enforces its value; a "defaultValue" element only +// supplies one, and only when the request sets apply_policy_default_values. const testPolicyDefinition = `{ "spark_version": {"type": "fixed", "value": "policy-version"}, "custom_tags.Fixed": {"type": "fixed", "value": "f"}, "custom_tags.Default": {"type": "unlimited", "defaultValue": "d"}, - "node_type_id": {"type": "forbidden"} + "node_type_id": {"type": "unlimited", "defaultValue": "policy-node"} }` func policyWorkspace() *FakeWorkspace { @@ -28,6 +31,7 @@ func TestApplyClusterPolicy(t *testing.T) { spec compute.ClusterSpec want string wantForceSend []string + wantErr string }{ { name: "fixed applies without apply_policy_default_values, defaultValue does not", @@ -40,20 +44,34 @@ func TestApplyClusterPolicy(t *testing.T) { name: "defaultValue applies with apply_policy_default_values", policyID: "p1", spec: compute.ClusterSpec{PolicyId: "p1", ApplyPolicyDefaultValues: true}, - want: `{"apply_policy_default_values":true,"custom_tags":{"Default":"d","Fixed":"f"},"policy_id":"p1","spark_version":"policy-version"}`, + want: `{"apply_policy_default_values":true,"custom_tags":{"Default":"d","Fixed":"f"},"node_type_id":"policy-node","policy_id":"p1","spark_version":"policy-version"}`, + wantForceSend: []string{"NodeTypeId", "SparkVersion"}, + }, + { + name: "a defaultValue does not override what the request supplied", + policyID: "p1", + spec: compute.ClusterSpec{PolicyId: "p1", ApplyPolicyDefaultValues: true, NodeTypeId: "user-node"}, + want: `{"apply_policy_default_values":true,"custom_tags":{"Default":"d","Fixed":"f"},"node_type_id":"user-node","policy_id":"p1","spark_version":"policy-version"}`, + wantForceSend: []string{"SparkVersion"}, + }, + { + name: "policy tags merge into tags the request supplied", + policyID: "p1", + spec: compute.ClusterSpec{PolicyId: "p1", CustomTags: map[string]string{"Mine": "yes"}}, + want: `{"custom_tags":{"Fixed":"f","Mine":"yes"},"policy_id":"p1","spark_version":"policy-version"}`, wantForceSend: []string{"SparkVersion"}, }, { - name: "a value the request supplied is not overridden", + name: "a value contradicting a fixed element is rejected", policyID: "p1", spec: compute.ClusterSpec{PolicyId: "p1", SparkVersion: "user-version"}, - want: `{"custom_tags":{"Fixed":"f"},"policy_id":"p1","spark_version":"user-version"}`, + wantErr: `Cluster validation error: Validation failed for spark_version must be policy-version (is "user-version")`, }, { - name: "policy tags merge into tags the request supplied", + name: "a tag contradicting a fixed element is rejected", policyID: "p1", - spec: compute.ClusterSpec{PolicyId: "p1", SparkVersion: "v", CustomTags: map[string]string{"Mine": "yes"}}, - want: `{"custom_tags":{"Fixed":"f","Mine":"yes"},"policy_id":"p1","spark_version":"v"}`, + spec: compute.ClusterSpec{PolicyId: "p1", CustomTags: map[string]string{"Fixed": "mine"}}, + wantErr: `Cluster validation error: Validation failed for custom_tags, Fixed must be f (is "mine")`, }, { name: "no policy attached leaves the spec untouched", @@ -72,10 +90,16 @@ func TestApplyClusterPolicy(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { spec := tt.spec - policyWorkspace().applyClusterPolicy(&spec, tt.policyID, spec.ApplyPolicyDefaultValues) + msg := policyWorkspace().applyClusterPolicy(&spec, tt.policyID, spec.ApplyPolicyDefaultValues) + + if tt.wantErr != "" { + assert.Equal(t, tt.wantErr, msg) + return + } + require.Empty(t, msg) got, err := json.Marshal(&spec) - assert.NoError(t, err) + require.NoError(t, err) assert.JSONEq(t, tt.want, string(got)) // Only attributes the policy actually supplied may become force-sent. Anything // else would serialize as an explicit zero and break the fake's ability to model diff --git a/libs/testserver/clusters.go b/libs/testserver/clusters.go index 288ace64f25..e0a046031e6 100644 --- a/libs/testserver/clusters.go +++ b/libs/testserver/clusters.go @@ -34,7 +34,15 @@ func (s *FakeWorkspace) ClustersCreate(req Request) any { // Apply the attached cluster policy before computing defaults, matching the backend: // policy values feed the defaults below (e.g. driver_node_type_id from node_type_id). - s.applyClusterPolicy(&request, request.PolicyId, applyPolicyDefaultValues(req.Body)) + if msg := s.applyClusterPolicy(&request, request.PolicyId, applyPolicyDefaultValues(req.Body)); msg != "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": msg, + }, + } + } clusterFixUps(&request) // The cluster GET API returns apply_policy_default_values only under .spec, not at the top @@ -118,7 +126,15 @@ func (s *FakeWorkspace) ClustersEdit(req Request) any { request.ClusterId = existing.ClusterId // Apply the attached cluster policy before computing defaults, matching the backend: // policy values feed the defaults below (e.g. driver_node_type_id from node_type_id). - s.applyClusterPolicy(&request, request.PolicyId, applyPolicyDefaultValues(req.Body)) + if msg := s.applyClusterPolicy(&request, request.PolicyId, applyPolicyDefaultValues(req.Body)); msg != "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": msg, + }, + } + } clusterFixUps(&request) // Refresh the .spec snapshot from the new settings, matching cloud behavior on edit. request.Spec = specSnapshot(req.Body) diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index 2f887290e35..29c023e4038 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -101,7 +101,15 @@ func (s *FakeWorkspace) JobsCreate(req Request) Response { } } - s.applyJobClusterPolicies(&jobSettings) + if msg := s.applyJobClusterPolicies(&jobSettings); msg != "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": msg, + }, + } + } jobFixUps(&jobSettings) // CreatorUserName field is used by TF to check if the resource exists or not. CreatorUserName should be non-empty for the resource to be considered as "exists" @@ -131,7 +139,15 @@ func (s *FakeWorkspace) JobsReset(req Request) Response { defer s.LockUnlock()() - s.applyJobClusterPolicies(&request.NewSettings) + if msg := s.applyJobClusterPolicies(&request.NewSettings); msg != "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": msg, + }, + } + } jobFixUps(&request.NewSettings) jobId := request.JobId diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index b46c438fae8..0aedc44c5d3 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -78,7 +78,6 @@ func (s *FakeWorkspace) PipelineCreate(req Request) Response { r.State = "IDLE" r.EffectivePublishingMode = pipelines.PublishingModeDefaultPublishingMode - s.applyPipelineClusterPolicies(&spec) setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = r @@ -138,7 +137,6 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response if edit.RunAs != nil { item.RunAs = edit.RunAs } - s.applyPipelineClusterPolicies(&spec) setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = item From 3896c2f7f8b250f15912dbf2cf700b176175eb48 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 18:17:37 +0200 Subject: [PATCH 03/25] Add PR link to changelog fragment Co-authored-by: Isaac --- .nextchanges/bundles/cluster-policy-no-drift.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/cluster-policy-no-drift.md b/.nextchanges/bundles/cluster-policy-no-drift.md index 3bbcce1ef73..0caa1d030c1 100644 --- a/.nextchanges/bundles/cluster-policy-no-drift.md +++ b/.nextchanges/bundles/cluster-policy-no-drift.md @@ -1 +1 @@ -* `bundle plan` no longer reports a permanent update on cluster fields supplied by a cluster policy; a field the config never declares is no longer drift when the cluster spec has a `policy_id`. +* `bundle plan` no longer reports a permanent update on cluster fields supplied by a cluster policy; a field the config never declares is no longer drift when the cluster spec has a `policy_id`. ([#6531](https://github.com/databricks/cli/pull/6531)) From 84e8eea328e26caa6e6d23782e80db6a24927f5a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 18:18:00 +0200 Subject: [PATCH 04/25] acc: show the spark_version verdict in fixed_values_applied The test proved the plan was clean but not why. spark_version is the field from issue 6512 that no backend_defaults rule covers, so print its classification to assert the suppression comes from policy_managed. Co-authored-by: Isaac --- .../fixed_values_applied/output.txt | 20 +++++++++++++++++++ .../fixed_values_applied/script | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/output.txt b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/output.txt index 7ef978595af..92a3a9b4e23 100644 --- a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/output.txt +++ b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/output.txt @@ -20,6 +20,26 @@ The bundle declares neither, and does not set apply_policy_default_values. >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +=== spark_version is not covered by any backend_defaults rule, so show its verdict + +>>> [CLI] bundle plan -o json +[ + { + "job_clusters[job_cluster_key='small'].new_cluster.custom_tags": { + "action": "skip", + "reason": "policy_managed", + "remote": { + "CostCenter": "policy-supplied" + } + }, + "job_clusters[job_cluster_key='small'].new_cluster.spark_version": { + "action": "skip", + "reason": "policy_managed", + "remote": "13.3.x-snapshot-scala2.12" + } + } +] + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.cluster_policies.pol diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/script b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/script index 2a8f815ddac..a20c901153d 100644 --- a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/script +++ b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/script @@ -18,3 +18,9 @@ $CLI jobs get "$job_id" | jq '.settings.job_clusters[0].new_cluster | {spark_ver title "Is the policy-supplied value reported as drift?\n" trace $CLI bundle plan + +title "spark_version is not covered by any backend_defaults rule, so show its verdict\n" +trace $CLI bundle plan -o json | jq -S ' + [.plan | to_entries[] | (.value.changes // {}) + | with_entries(select(.key | endswith("spark_version") or endswith("custom_tags")))] + | map(select(length > 0))' From 1ffb64a9e9917d9ebe514f5f60ad946aa43b586b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 19:26:50 +0200 Subject: [PATCH 05/25] acc: keep policy_no_drift_variants out of short cloud runs Creating a real cluster made it the slowest test in the integration suite, at 6-8 minutes per env across six envs. It still runs locally on every test run and in full cloud runs; the cheap job-based tests in the same directory keep the core behaviour in every cloud run. Co-authored-by: Isaac --- .../policy_no_drift_variants/databricks.yml.tmpl | 3 +++ .../cluster_policies/policy_no_drift_variants/out.test.toml | 1 + .../cluster_policies/policy_no_drift_variants/test.toml | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl index d5670422794..9c7520a8f5f 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl @@ -23,6 +23,9 @@ resources: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID + # Use the shared instance pool: a cold cluster boot made this the slowest test in the + # cloud suite (6-8 minutes per env), and the policy tag under test is unaffected. + instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 policy_id: ${resources.cluster_policies.pol.id} diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml index ae5c7bd798f..f9f4880725d 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml @@ -1,3 +1,4 @@ Cloud = true +CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml index ae35e046923..1002b7e9e54 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml @@ -1,3 +1,8 @@ +# Creating a real cluster made this the slowest test in the cloud suite (~8 min per env), and +# the per-location classification it checks is fully covered locally. The cheap job-based +# tests in this directory keep the core behaviour in every cloud run. +CloudSlow = true + RecordRequests = false Ignore = [ From 72736b3535e9c021f3ffed71e27223af68878698 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 7 Sep 2026 15:32:56 +0200 Subject: [PATCH 06/25] Make ignore_remote_additions when_set a path It was a bare string joined onto the matched prefix with NewDotString, which is asymmetric with field and silently accepts a dotted value as one field name. Resolving in two steps -- the matched object, then the gate relative to it -- needs no path-join helper and supports a multi-segment gate for free. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 9 ++++++--- bundle/direct/bundle_plan_test.go | 22 +++++++++++++++++++++- bundle/direct/dresources/config.go | 6 +++--- bundle/direct/dresources/config_test.go | 17 +++++++++-------- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 0ab971dfc1e..62569b30045 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -598,8 +598,10 @@ func shouldSkipRemoteAddition(cfg *dresources.ResourceLifecycleConfig, path *str if !path.HasPatternPrefix(rule.Field) { continue } - gate := structpath.NewDotString(path.Prefix(rule.Field.Len()), rule.WhenSet) - value, err := structaccess.Get(newState, gate) + // Resolve the gate in two steps: the concrete object the rule matched, then the gate + // path relative to it. The wildcards in Field are filled in from the change path, so + // each object is gated on its own value. + object, err := structaccess.Get(newState, path.Prefix(rule.Field.Len())) if err != nil { // The gated object is absent from the config entirely (e.g. the remote grew a // whole new_cluster the bundle does not declare), so there is no policy to gate @@ -607,7 +609,8 @@ func shouldSkipRemoteAddition(cfg *dresources.ResourceLifecycleConfig, path *str // are validated against the state type by TestResourcesYMLRemoteAdditionGates. continue } - if allEmpty(value) { + value, err := structaccess.Get(object, rule.WhenSet) + if err != nil || allEmpty(value) { continue } return deployplan.ReasonPolicyManaged, true diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index ba75e1953ed..88413061c36 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -411,7 +411,7 @@ func TestShouldSkipRemoteAddition(t *testing.T) { require.NoError(t, err) cfg := &dresources.ResourceLifecycleConfig{ IgnoreRemoteAdditions: []dresources.RemoteAdditionRule{ - {Field: jobCluster, WhenSet: "policy_id"}, + {Field: jobCluster, WhenSet: structpath.MustParsePath("policy_id")}, }, } @@ -423,6 +423,12 @@ func TestShouldSkipRemoteAddition(t *testing.T) { JobClusterKey: "small", NewCluster: &compute.ClusterSpec{}, }}} + // when_set resolves against the concrete object the rule matched, so two clusters in one + // job are gated independently. + mixed := &jobs.JobSettings{JobClusters: []jobs.JobCluster{ + {JobClusterKey: "gated", NewCluster: &compute.ClusterSpec{PolicyId: "p1"}}, + {JobClusterKey: "plain", NewCluster: &compute.ClusterSpec{}}, + }} const tagPath = "job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter']" @@ -482,6 +488,20 @@ func TestShouldSkipRemoteAddition(t *testing.T) { change: deployplan.ChangeDesc{Remote: "dev-1234"}, expected: false, }, + { + name: "sibling cluster with a policy does not gate one without", + path: "job_clusters[job_cluster_key='plain'].new_cluster.custom_tags['CostCenter']", + state: mixed, + change: deployplan.ChangeDesc{Remote: "dev-1234"}, + expected: false, + }, + { + name: "the cluster with the policy is gated on its own policy_id", + path: "job_clusters[job_cluster_key='gated'].new_cluster.custom_tags['CostCenter']", + state: mixed, + change: deployplan.ChangeDesc{Remote: "dev-1234"}, + expected: true, + }, { // The remote grew a whole cluster the config does not declare: there is no // policy_id to gate on, so this is real drift rather than a policy addition. diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 0fab4e24e8b..0aa011605c0 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -49,11 +49,11 @@ func (b *BackendDefaultRule) UnmarshalYAML(unmarshal func(any) error) error { // object's WhenSet field is set. Inside such an object, a field the config never declared // coming back set from the remote is an addition by the backend, not drift. // -// Field is a prefix pattern selecting the object (omitted = the resource root); WhenSet is -// the name of a field within that object which gates the rule. +// Field is a prefix pattern selecting the object (omitted = the resource root); WhenSet is a +// path within that object, relative to it, whose value gates the rule. type RemoteAdditionRule struct { Field *structpath.PatternNode `yaml:"field"` - WhenSet string `yaml:"when_set"` + WhenSet *structpath.PathNode `yaml:"when_set"` } // ResourceLifecycleConfig defines lifecycle behavior for a resource type. diff --git a/bundle/direct/dresources/config_test.go b/bundle/direct/dresources/config_test.go index 1bcae5f2e27..b50c633bb72 100644 --- a/bundle/direct/dresources/config_test.go +++ b/bundle/direct/dresources/config_test.go @@ -158,17 +158,18 @@ func TestResourcesYMLRemoteAdditionGates(t *testing.T) { require.NoError(t, err) for _, rule := range rc.IgnoreRemoteAdditions { - require.NotEmpty(t, rule.WhenSet, "%s: ignore_remote_additions entry %q has no when_set", resourceType, rule.Field.String()) + field := rule.Field + require.False(t, rule.WhenSet.IsRoot(), + "%s: ignore_remote_additions entry %q needs a when_set", resourceType, field.String()) - gate := rule.Field - if !gate.IsRoot() { - assert.NoError(t, structaccess.ValidatePattern(adapter.StateType(), gate), - "%s: ignore_remote_additions field %q does not resolve in the state type", resourceType, gate.String()) + if !field.IsRoot() { + assert.NoError(t, structaccess.ValidatePattern(adapter.StateType(), field), + "%s: ignore_remote_additions field %q does not resolve in the state type", resourceType, field.String()) } - gatePattern, err := structpath.ParsePattern(joinPattern(gate, rule.WhenSet)) + gate, err := structpath.ParsePattern(joinPattern(field, rule.WhenSet.String())) require.NoError(t, err) - assert.NoError(t, structaccess.ValidatePattern(adapter.StateType(), gatePattern), - "%s: ignore_remote_additions when_set %q does not resolve under %q", resourceType, rule.WhenSet, gate.String()) + assert.NoError(t, structaccess.ValidatePattern(adapter.StateType(), gate), + "%s: ignore_remote_additions when_set %q does not resolve under %q", resourceType, rule.WhenSet, field.String()) } } } From cd7e7d7988d739bbf7de1e35461bf937c7fae486 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Sep 2026 20:15:48 +0200 Subject: [PATCH 07/25] acc: move jobs+policy tests under resources/jobs/cluster_policy fixed_value_conflict, fixed_values_applied, policy_value_semantics and policy_drift attach a cluster policy to a job's cluster, so the resource under test is jobs, not cluster_policies. The old path implied the opposite. Their shared config (Cloud, direct-only engine matrix, no request recording) moves to a single test.toml at the new parent. policy_no_drift_variants stays under resources/cluster_policies: it spans clusters, jobs and pipelines, so no single resource owns it. Co-authored-by: Isaac --- .../cluster_policies/fixed_value_conflict/test.toml | 5 ----- .../cluster_policies/fixed_values_applied/test.toml | 5 ----- .../resources/cluster_policies/policy_drift/test.toml | 5 ----- .../cluster_policies/policy_value_semantics/test.toml | 5 ----- .../fixed_value_conflict/databricks.yml.tmpl | 0 .../fixed_value_conflict/hello_world.py | 0 .../fixed_value_conflict/out.test.toml | 0 .../cluster_policy}/fixed_value_conflict/output.txt | 2 +- .../cluster_policy}/fixed_value_conflict/script | 0 .../fixed_values_applied/databricks.yml.tmpl | 0 .../fixed_values_applied/hello_world.py | 0 .../fixed_values_applied/out.test.toml | 0 .../cluster_policy}/fixed_values_applied/output.txt | 2 +- .../cluster_policy}/fixed_values_applied/script | 0 .../cluster_policy}/policy_drift/databricks.yml.tmpl | 0 .../cluster_policy}/policy_drift/hello_world.py | 0 .../cluster_policy}/policy_drift/out.test.toml | 0 .../cluster_policy}/policy_drift/output.txt | 2 +- .../cluster_policy}/policy_drift/script | 0 .../policy_value_semantics/databricks.yml.tmpl | 0 .../policy_value_semantics/hello_world.py | 0 .../policy_value_semantics/out.test.toml | 0 .../cluster_policy}/policy_value_semantics/output.txt | 2 +- .../cluster_policy}/policy_value_semantics/script | 0 .../bundle/resources/jobs/cluster_policy/test.toml | 11 +++++++++++ bundle/direct/bundle_plan.go | 2 +- bundle/direct/dresources/resources.yml | 4 ++-- libs/testserver/cluster_policies.go | 2 +- 28 files changed, 19 insertions(+), 28 deletions(-) delete mode 100644 acceptance/bundle/resources/cluster_policies/fixed_value_conflict/test.toml delete mode 100644 acceptance/bundle/resources/cluster_policies/fixed_values_applied/test.toml delete mode 100644 acceptance/bundle/resources/cluster_policies/policy_drift/test.toml delete mode 100644 acceptance/bundle/resources/cluster_policies/policy_value_semantics/test.toml rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/fixed_value_conflict/databricks.yml.tmpl (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/fixed_value_conflict/hello_world.py (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/fixed_value_conflict/out.test.toml (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/fixed_value_conflict/output.txt (96%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/fixed_value_conflict/script (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/fixed_values_applied/databricks.yml.tmpl (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/fixed_values_applied/hello_world.py (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/fixed_values_applied/out.test.toml (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/fixed_values_applied/output.txt (97%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/fixed_values_applied/script (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/policy_drift/databricks.yml.tmpl (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/policy_drift/hello_world.py (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/policy_drift/out.test.toml (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/policy_drift/output.txt (98%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/policy_drift/script (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/policy_value_semantics/databricks.yml.tmpl (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/policy_value_semantics/hello_world.py (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/policy_value_semantics/out.test.toml (100%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/policy_value_semantics/output.txt (97%) rename acceptance/bundle/resources/{cluster_policies => jobs/cluster_policy}/policy_value_semantics/script (100%) create mode 100644 acceptance/bundle/resources/jobs/cluster_policy/test.toml diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/test.toml b/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/test.toml deleted file mode 100644 index ae35e046923..00000000000 --- a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/test.toml +++ /dev/null @@ -1,5 +0,0 @@ -RecordRequests = false - -Ignore = [ - "databricks.yml", -] diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/test.toml b/acceptance/bundle/resources/cluster_policies/fixed_values_applied/test.toml deleted file mode 100644 index ae35e046923..00000000000 --- a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/test.toml +++ /dev/null @@ -1,5 +0,0 @@ -RecordRequests = false - -Ignore = [ - "databricks.yml", -] diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/test.toml b/acceptance/bundle/resources/cluster_policies/policy_drift/test.toml deleted file mode 100644 index ae35e046923..00000000000 --- a/acceptance/bundle/resources/cluster_policies/policy_drift/test.toml +++ /dev/null @@ -1,5 +0,0 @@ -RecordRequests = false - -Ignore = [ - "databricks.yml", -] diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/test.toml b/acceptance/bundle/resources/cluster_policies/policy_value_semantics/test.toml deleted file mode 100644 index ae35e046923..00000000000 --- a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/test.toml +++ /dev/null @@ -1,5 +0,0 @@ -RecordRequests = false - -Ignore = [ - "databricks.yml", -] diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/resources/cluster_policies/fixed_value_conflict/databricks.yml.tmpl rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/hello_world.py b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/hello_world.py similarity index 100% rename from acceptance/bundle/resources/cluster_policies/fixed_value_conflict/hello_world.py rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/hello_world.py diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/out.test.toml b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/out.test.toml similarity index 100% rename from acceptance/bundle/resources/cluster_policies/fixed_value_conflict/out.test.toml rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/out.test.toml diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt similarity index 96% rename from acceptance/bundle/resources/cluster_policies/fixed_value_conflict/output.txt rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt index 097e5a7e98e..52729d259af 100644 --- a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt @@ -10,7 +10,7 @@ HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE API message: Cluster validation error: Validation failed for custom_tags, CostCenter must be from-fixed (is "not-what-the-policy-says") -Files: 6 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Exit code: 1 diff --git a/acceptance/bundle/resources/cluster_policies/fixed_value_conflict/script b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/script similarity index 100% rename from acceptance/bundle/resources/cluster_policies/fixed_value_conflict/script rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/script diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/resources/cluster_policies/fixed_values_applied/databricks.yml.tmpl rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/hello_world.py b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/hello_world.py similarity index 100% rename from acceptance/bundle/resources/cluster_policies/fixed_values_applied/hello_world.py rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/hello_world.py diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/out.test.toml b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/out.test.toml similarity index 100% rename from acceptance/bundle/resources/cluster_policies/fixed_values_applied/out.test.toml rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/out.test.toml diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt similarity index 97% rename from acceptance/bundle/resources/cluster_policies/fixed_values_applied/output.txt rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt index 92a3a9b4e23..ba645957aa8 100644 --- a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt @@ -3,7 +3,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.pol Created jobs.j -Files: 6 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Did the fixed policy SUPPLY spark_version and custom_tags, or only validate them? diff --git a/acceptance/bundle/resources/cluster_policies/fixed_values_applied/script b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script similarity index 100% rename from acceptance/bundle/resources/cluster_policies/fixed_values_applied/script rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/resources/cluster_policies/policy_drift/databricks.yml.tmpl rename to acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/hello_world.py b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/hello_world.py similarity index 100% rename from acceptance/bundle/resources/cluster_policies/policy_drift/hello_world.py rename to acceptance/bundle/resources/jobs/cluster_policy/policy_drift/hello_world.py diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/out.test.toml b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/out.test.toml similarity index 100% rename from acceptance/bundle/resources/cluster_policies/policy_drift/out.test.toml rename to acceptance/bundle/resources/jobs/cluster_policy/policy_drift/out.test.toml diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt similarity index 98% rename from acceptance/bundle/resources/cluster_policies/policy_drift/output.txt rename to acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt index 2242def2f4c..a6d746fac13 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_drift/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt @@ -5,7 +5,7 @@ Created cluster_policies.pol Created jobs.no_policy Created jobs.owned_tag Created jobs.with_policy -Files: 6 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 4 created, 0 changed, 0 deleted, 0 unchanged === A tag the cluster policy added is not drift diff --git a/acceptance/bundle/resources/cluster_policies/policy_drift/script b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script similarity index 100% rename from acceptance/bundle/resources/cluster_policies/policy_drift/script rename to acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/resources/cluster_policies/policy_value_semantics/databricks.yml.tmpl rename to acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/databricks.yml.tmpl diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/hello_world.py b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/hello_world.py similarity index 100% rename from acceptance/bundle/resources/cluster_policies/policy_value_semantics/hello_world.py rename to acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/hello_world.py diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/out.test.toml b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/out.test.toml similarity index 100% rename from acceptance/bundle/resources/cluster_policies/policy_value_semantics/out.test.toml rename to acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/out.test.toml diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/output.txt similarity index 97% rename from acceptance/bundle/resources/cluster_policies/policy_value_semantics/output.txt rename to acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/output.txt index 30a4ff46b97..8b5606b0980 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/file Created cluster_policies.defaulted Created cluster_policies.fixed Created jobs.j -Files: 6 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Which policy elements did the backend SUPPLY into each job_cluster? diff --git a/acceptance/bundle/resources/cluster_policies/policy_value_semantics/script b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/script similarity index 100% rename from acceptance/bundle/resources/cluster_policies/policy_value_semantics/script rename to acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/script diff --git a/acceptance/bundle/resources/jobs/cluster_policy/test.toml b/acceptance/bundle/resources/jobs/cluster_policy/test.toml new file mode 100644 index 00000000000..1114c31f86d --- /dev/null +++ b/acceptance/bundle/resources/jobs/cluster_policy/test.toml @@ -0,0 +1,11 @@ +# The drift-suppression under test is a direct-engine feature (bundle/direct), so restrict the +# inherited engine matrix to direct. Cloud = true adds a real-workspace run; every test here +# also runs locally against the testserver. +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 62569b30045..f36da015e7a 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -585,7 +585,7 @@ func shouldSkipNormalized(cfg *dresources.ResourceLifecycleConfig, path *structp // For cluster specs that gate is policy_id: an attached cluster policy supplies values // server-side — "fixed" elements always, "defaultValue" elements when the request sets // apply_policy_default_values — so the remote spec is legitimately a superset of what the -// bundle declares. See acceptance/bundle/resources/cluster_policies/policy_value_semantics +// bundle declares. See acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics // for the measured backend behavior. // // Suppressed values are never echoed back on write: an update sends the config spec as-is diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index ce9ee2a4dcf..d3c1ea32f92 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -90,7 +90,7 @@ resources: # "defaultValue" elements when the request sets apply_policy_default_values. The bundle # is not the source for those, so the remote spec is legitimately a superset of what the # config declares and an added field is not drift. Measured backend behavior is pinned by - # acceptance/bundle/resources/cluster_policies/policy_value_semantics. + # acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics. # https://github.com/databricks/cli/issues/5179 # https://github.com/databricks/cli/issues/6512 - field: tasks[*].new_cluster @@ -612,7 +612,7 @@ resources: # "defaultValue" elements when the request sets apply_policy_default_values. The bundle # is not the source for those, so the remote spec is legitimately a superset of what the # config declares and an added field is not drift. Measured backend behavior is pinned by - # acceptance/bundle/resources/cluster_policies/policy_value_semantics. + # acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics. # https://github.com/databricks/cli/issues/5179 # https://github.com/databricks/cli/issues/6512 ignore_remote_additions: diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go index f2871870594..b37726cccc1 100644 --- a/libs/testserver/cluster_policies.go +++ b/libs/testserver/cluster_policies.go @@ -142,7 +142,7 @@ type policyElement struct { // attribute, or nil if it materializes none. Mirrors the backend: "fixed" applies whether // or not the request sets apply_policy_default_values, "defaultValue" only when it does. // Pinned against a real workspace by -// acceptance/bundle/resources/cluster_policies/policy_value_semantics. +// acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics. func (e policyElement) effectiveValue(applyDefaults bool) any { if e.Type == "fixed" { return e.Value From 33a6ad6c05b4537dfd514fbb257245125414301b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Sep 2026 20:56:37 +0200 Subject: [PATCH 08/25] Address review: honest changelog, inline YAML policies, gron over jq - Changelog no longer claims a field was "supplied by a cluster policy": the rule gates on policy_id being set and suppresses any config-absent remote field, without verifying the field's origin. Reworded to describe that rule. - Cluster policy definitions are authored as inline YAML instead of an embedded JSON string, the form the CLI already supports (see definition_yaml). - The plan/jobs-get assertions use gron.py | grep instead of inline jq, per the repo guidance; the flattened paths are self-documenting. Co-authored-by: Isaac --- .../bundles/cluster-policy-no-drift.md | 2 +- .../databricks.yml.tmpl | 11 ++-- .../policy_no_drift_variants/output.txt | 51 +++++-------------- .../policy_no_drift_variants/script | 5 +- .../fixed_value_conflict/databricks.yml.tmpl | 11 ++-- .../fixed_values_applied/databricks.yml.tmpl | 18 +++---- .../fixed_values_applied/output.txt | 33 ++++-------- .../fixed_values_applied/script | 7 +-- .../policy_drift/databricks.yml.tmpl | 11 ++-- .../cluster_policy/policy_drift/output.txt | 29 ++++------- .../jobs/cluster_policy/policy_drift/script | 5 +- .../databricks.yml.tmpl | 24 ++++----- .../policy_value_semantics/output.txt | 36 +++---------- .../policy_value_semantics/script | 5 +- 14 files changed, 77 insertions(+), 171 deletions(-) diff --git a/.nextchanges/bundles/cluster-policy-no-drift.md b/.nextchanges/bundles/cluster-policy-no-drift.md index 0caa1d030c1..f3f2d04dd64 100644 --- a/.nextchanges/bundles/cluster-policy-no-drift.md +++ b/.nextchanges/bundles/cluster-policy-no-drift.md @@ -1 +1 @@ -* `bundle plan` no longer reports a permanent update on cluster fields supplied by a cluster policy; a field the config never declares is no longer drift when the cluster spec has a `policy_id`. ([#6531](https://github.com/databricks/cli/pull/6531)) +* On the direct engine, `bundle plan` no longer reports a permanent update on a cluster that uses a cluster policy: when the cluster spec sets `policy_id`, a field present in the remote but absent from the bundle config is not treated as drift. ([#6531](https://github.com/databricks/cli/pull/6531)) diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl index 9c7520a8f5f..72b4684279d 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl @@ -8,13 +8,10 @@ resources: cluster_policies: pol: name: test-policy-$UNIQUE_NAME - definition: |- - { - "custom_tags.CostCenter": { - "type": "fixed", - "value": "from-policy" - } - } + definition: + custom_tags.CostCenter: + type: fixed + value: from-policy # Every place a cluster spec can appear must be covered by a rule; a gap here is how # https://github.com/databricks/cli/issues/5179 stayed open after a partial fix. diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt index 585eeb57771..f6e85316b2e 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt @@ -17,44 +17,19 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged === Confirm the tag really is present remotely and was classified, not just absent >>> [CLI] bundle plan -o json -[ - { - "node": "resources.clusters.standalone", - "tags": { - "custom_tags": { - "action": "skip", - "reason": "policy_managed", - "remote": { - "CostCenter": "from-policy" - } - } - } - }, - { - "node": "resources.jobs.for_each_cluster", - "tags": { - "tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags": { - "action": "skip", - "reason": "policy_managed", - "remote": { - "CostCenter": "from-policy" - } - } - } - }, - { - "node": "resources.jobs.task_cluster", - "tags": { - "tasks[task_key='t'].new_cluster.custom_tags": { - "action": "skip", - "reason": "policy_managed", - "remote": { - "CostCenter": "from-policy" - } - } - } - } -] +json.plan.resources.cluster_policies.pol.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}"; +json.plan.resources.clusters.standalone.remote_state.custom_tags.CostCenter = "from-policy"; +json.plan.resources.clusters.standalone.changes.custom_tags.action = "skip"; +json.plan.resources.clusters.standalone.changes.custom_tags.reason = "policy_managed"; +json.plan.resources.clusters.standalone.changes.custom_tags.remote.CostCenter = "from-policy"; +json.plan.resources.jobs.for_each_cluster.remote_state.tasks[0].for_each_task.task.new_cluster.custom_tags.CostCenter = "from-policy"; +json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.action = "skip"; +json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.reason = "policy_managed"; +json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.remote.CostCenter = "from-policy"; +json.plan.resources.jobs.task_cluster.remote_state.tasks[0].new_cluster.custom_tags.CostCenter = "from-policy"; +json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.action = "skip"; +json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.reason = "policy_managed"; +json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.remote.CostCenter = "from-policy"; >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script index c12e8caaf7f..af3a0ec4fca 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script @@ -11,7 +11,4 @@ title "Every cluster-spec location converges: the policy tag is not drift anywhe trace $CLI bundle plan title "Confirm the tag really is present remotely and was classified, not just absent\n" -trace $CLI bundle plan -o json | jq -S ' - [.plan | to_entries[] - | {node: .key, tags: ((.value.changes // {}) | with_entries(select(.key | contains("custom_tags"))))} - | select(.tags | length > 0)]' +trace $CLI bundle plan -o json | gron.py | grep custom_tags diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl index c80fed9f81c..7264a9bb25d 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl @@ -8,13 +8,10 @@ resources: cluster_policies: pol: name: test-policy-$UNIQUE_NAME - definition: |- - { - "custom_tags.CostCenter": { - "type": "fixed", - "value": "from-fixed" - } - } + definition: + custom_tags.CostCenter: + type: fixed + value: from-fixed jobs: # The config sets the fixed attribute to a value the policy forbids. diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl index 8fb232538eb..d03bb2c5775 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl @@ -8,17 +8,13 @@ resources: cluster_policies: pol: name: test-policy-$UNIQUE_NAME - definition: |- - { - "spark_version": { - "type": "fixed", - "value": "$DEFAULT_SPARK_VERSION" - }, - "custom_tags.CostCenter": { - "type": "fixed", - "value": "policy-supplied" - } - } + definition: + spark_version: + type: fixed + value: $DEFAULT_SPARK_VERSION + custom_tags.CostCenter: + type: fixed + value: policy-supplied jobs: # new_cluster deliberately omits spark_version and custom_tags: the policy is diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt index ba645957aa8..71e2547e7b5 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt @@ -8,12 +8,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Did the fixed policy SUPPLY spark_version and custom_tags, or only validate them? The bundle declares neither, and does not set apply_policy_default_values. -{ - "spark_version": "13.3.x-snapshot-scala2.12", - "custom_tags": { - "CostCenter": "policy-supplied" - } -} +json.settings.job_clusters[0].new_cluster.custom_tags.CostCenter = "policy-supplied"; +json.settings.job_clusters[0].new_cluster.spark_version = "13.3.x-snapshot-scala2.12"; === Is the policy-supplied value reported as drift? @@ -23,22 +19,15 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === spark_version is not covered by any backend_defaults rule, so show its verdict >>> [CLI] bundle plan -o json -[ - { - "job_clusters[job_cluster_key='small'].new_cluster.custom_tags": { - "action": "skip", - "reason": "policy_managed", - "remote": { - "CostCenter": "policy-supplied" - } - }, - "job_clusters[job_cluster_key='small'].new_cluster.spark_version": { - "action": "skip", - "reason": "policy_managed", - "remote": "13.3.x-snapshot-scala2.12" - } - } -] +json.plan.resources.cluster_policies.pol.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"policy-supplied\"},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-snapshot-scala2.12\"}}"; +json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "policy-supplied"; +json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.spark_version = "13.3.x-snapshot-scala2.12"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.action = "skip"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.reason = "policy_managed"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.remote.CostCenter = "policy-supplied"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.action = "skip"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.reason = "policy_managed"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.remote = "13.3.x-snapshot-scala2.12"; >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script index a20c901153d..b02afa490c6 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script @@ -14,13 +14,10 @@ job_id=$(read_id.py j) title "Did the fixed policy SUPPLY spark_version and custom_tags, or only validate them?\n" echo "The bundle declares neither, and does not set apply_policy_default_values." -$CLI jobs get "$job_id" | jq '.settings.job_clusters[0].new_cluster | {spark_version, custom_tags}' +$CLI jobs get "$job_id" | gron.py | grep -E 'new_cluster.(spark_version|custom_tags)' title "Is the policy-supplied value reported as drift?\n" trace $CLI bundle plan title "spark_version is not covered by any backend_defaults rule, so show its verdict\n" -trace $CLI bundle plan -o json | jq -S ' - [.plan | to_entries[] | (.value.changes // {}) - | with_entries(select(.key | endswith("spark_version") or endswith("custom_tags")))] - | map(select(length > 0))' +trace $CLI bundle plan -o json | gron.py | grep -E 'custom_tags|spark_version' diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl index 1c988bcdb8f..148576b2dc1 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl @@ -8,13 +8,10 @@ resources: cluster_policies: pol: name: test-policy-$UNIQUE_NAME - definition: |- - { - "custom_tags.CostCenter": { - "type": "fixed", - "value": "from-policy" - } - } + definition: + custom_tags.CostCenter: + type: fixed + value: from-policy jobs: # policy attached: the tag the policy adds must not be drift. diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt index a6d746fac13..9ec031dba53 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt @@ -33,24 +33,17 @@ Plan: 0 to add, 2 to change, 0 to delete, 2 unchanged >>> update_file.py databricks.yml Mine: mine Other: other >>> [CLI] bundle plan -o json -[ - { - "job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter']": { - "action": "skip", - "reason": "policy_managed", - "remote": "from-policy" - }, - "job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine']": { - "action": "update", - "old": "mine", - "remote": "changed-out-of-band" - }, - "job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Other']": { - "action": "update", - "new": "other" - } - } -] +json.plan.resources.jobs.owned_tag.new_state.value.job_clusters[0].new_cluster.custom_tags.Other = "other"; +json.plan.resources.jobs.owned_tag.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "from-policy"; +json.plan.resources.jobs.owned_tag.remote_state.job_clusters[0].new_cluster.custom_tags.Mine = "changed-out-of-band"; +json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].action = "skip"; +json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].reason = "policy_managed"; +json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].remote = "from-policy"; +json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].action = "update"; +json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].old = "mine"; +json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].remote = "changed-out-of-band"; +json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Other'].action = "update"; +json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Other'].new = "other"; >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script index 5d927fd49ab..4eb274bdeb5 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script @@ -29,7 +29,4 @@ trace $CLI bundle plan | contains.py "2 to change" title "Removing a tag from config is a change the user asked for, not an addition\n" trace update_file.py databricks.yml "Mine: mine" "Other: other" -trace $CLI bundle plan -o json | jq -S ' - [.plan | to_entries[] - | select(.key | endswith("owned_tag")) - | .value.changes | with_entries(select(.key | contains("custom_tags")))]' +trace $CLI bundle plan -o json | gron.py | grep -E "owned_tag.*custom_tags" diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/databricks.yml.tmpl index 747138e1661..e4bc9b322c0 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/databricks.yml.tmpl @@ -11,26 +11,20 @@ resources: # that omits the attribute is what this test pins down. fixed: name: test-policy-fixed-$UNIQUE_NAME - definition: |- - { - "custom_tags.FixedTag": { - "type": "fixed", - "value": "from-fixed" - } - } + definition: + custom_tags.FixedTag: + type: fixed + value: from-fixed # A "defaultValue" element. Documented to apply only when the request sets # apply_policy_default_values=true. defaulted: name: test-policy-default-$UNIQUE_NAME - definition: |- - { - "custom_tags.DefaultTag": { - "type": "unlimited", - "defaultValue": "from-default", - "isOptional": true - } - } + definition: + custom_tags.DefaultTag: + type: unlimited + defaultValue: from-default + isOptional: true jobs: j: diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/output.txt index 8b5606b0980..0d0afa5a526 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/output.txt @@ -8,34 +8,14 @@ Files: 5 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Which policy elements did the backend SUPPLY into each job_cluster? -[ - { - "apply_policy_default_values": null, - "custom_tags": { - "FixedTag": "from-fixed" - }, - "job_cluster_key": "a_fixed_noflag" - }, - { - "apply_policy_default_values": null, - "custom_tags": null, - "job_cluster_key": "b_default_noflag" - }, - { - "apply_policy_default_values": null, - "custom_tags": { - "DefaultTag": "from-default" - }, - "job_cluster_key": "c_default_flag" - }, - { - "apply_policy_default_values": null, - "custom_tags": { - "FixedTag": "from-fixed" - }, - "job_cluster_key": "d_fixed_flag" - } -] +json.settings.job_clusters[0].job_cluster_key = "a_fixed_noflag"; +json.settings.job_clusters[0].new_cluster.custom_tags.FixedTag = "from-fixed"; +json.settings.job_clusters[1].job_cluster_key = "b_default_noflag"; +json.settings.job_clusters[2].job_cluster_key = "c_default_flag"; +json.settings.job_clusters[2].new_cluster.custom_tags.DefaultTag = "from-default"; +json.settings.job_clusters[3].job_cluster_key = "d_fixed_flag"; +json.settings.job_clusters[3].new_cluster.custom_tags.FixedTag = "from-fixed"; +json.settings.tasks[0].job_cluster_key = "a_fixed_noflag"; === Are the supplied values reported as drift? diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/script b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/script index 300ee0780c6..c78ff847b4c 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/script @@ -12,10 +12,7 @@ read_id.py defaulted > /dev/null job_id=$(read_id.py j) title "Which policy elements did the backend SUPPLY into each job_cluster?\n" -$CLI jobs get "$job_id" | jq -S '[.settings.job_clusters[] - | {job_cluster_key, - apply_policy_default_values: .new_cluster.apply_policy_default_values, - custom_tags: .new_cluster.custom_tags}]' +$CLI jobs get "$job_id" | gron.py | grep -E 'job_cluster_key|apply_policy_default_values|new_cluster.custom_tags' title "Are the supplied values reported as drift?\n" trace $CLI bundle plan From f5232785fdeb867ead9819907155e999f0ef4c9e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Sep 2026 21:20:47 +0200 Subject: [PATCH 09/25] Rename reason policy_managed -> remote_addition The rule gates on policy_id being set and suppresses a config-absent, remote-present field; it never verifies the value came from the policy. Name the reason after what is observed -- a remote-only addition -- to match the ignore_remote_additions rule and drop the unfounded provenance claim. Co-authored-by: Isaac --- .../cluster_policies/policy_no_drift_variants/output.txt | 6 +++--- .../jobs/cluster_policy/fixed_values_applied/output.txt | 4 ++-- .../jobs/cluster_policy/policy_drift/output.txt | 2 +- bundle/deployplan/plan.go | 9 +++++---- bundle/direct/bundle_plan.go | 2 +- bundle/direct/bundle_plan_test.go | 2 +- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt index f6e85316b2e..52ad8c38dc2 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt @@ -20,15 +20,15 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged json.plan.resources.cluster_policies.pol.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}"; json.plan.resources.clusters.standalone.remote_state.custom_tags.CostCenter = "from-policy"; json.plan.resources.clusters.standalone.changes.custom_tags.action = "skip"; -json.plan.resources.clusters.standalone.changes.custom_tags.reason = "policy_managed"; +json.plan.resources.clusters.standalone.changes.custom_tags.reason = "remote_addition"; json.plan.resources.clusters.standalone.changes.custom_tags.remote.CostCenter = "from-policy"; json.plan.resources.jobs.for_each_cluster.remote_state.tasks[0].for_each_task.task.new_cluster.custom_tags.CostCenter = "from-policy"; json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.action = "skip"; -json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.reason = "policy_managed"; +json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.reason = "remote_addition"; json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.remote.CostCenter = "from-policy"; json.plan.resources.jobs.task_cluster.remote_state.tasks[0].new_cluster.custom_tags.CostCenter = "from-policy"; json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.action = "skip"; -json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.reason = "policy_managed"; +json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.reason = "remote_addition"; json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.remote.CostCenter = "from-policy"; >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt index 71e2547e7b5..367a4ee4e9c 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt @@ -23,10 +23,10 @@ json.plan.resources.cluster_policies.pol.remote_state.definition = "{\"custom_ta json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "policy-supplied"; json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.spark_version = "13.3.x-snapshot-scala2.12"; json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.action = "skip"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.reason = "policy_managed"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.reason = "remote_addition"; json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.remote.CostCenter = "policy-supplied"; json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.action = "skip"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.reason = "policy_managed"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.reason = "remote_addition"; json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.remote = "13.3.x-snapshot-scala2.12"; >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt index 9ec031dba53..3ccf0c91dfb 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt @@ -37,7 +37,7 @@ json.plan.resources.jobs.owned_tag.new_state.value.job_clusters[0].new_cluster.c json.plan.resources.jobs.owned_tag.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "from-policy"; json.plan.resources.jobs.owned_tag.remote_state.job_clusters[0].new_cluster.custom_tags.Mine = "changed-out-of-band"; json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].action = "skip"; -json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].reason = "policy_managed"; +json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].reason = "remote_addition"; json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].remote = "from-policy"; json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].action = "update"; json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].old = "mine"; diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 3ffbb50ae27..6e022fdf564 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -147,10 +147,11 @@ const ( // ReasonMissingInRemote: field is not present in RemoteType (write-only / input-only). // Remote always appears nil, so treat the absence as a no-op when there is no local change. ReasonMissingInRemote = "missing_in_remote" - // ReasonPolicyManaged: the field is inside a cluster spec that has a policy_id, and the - // config never declared it. A cluster policy supplies values server-side, so the backend - // legitimately extends the spec beyond what the bundle asks for; that is not drift. - ReasonPolicyManaged = "policy_managed" + // ReasonRemoteAddition: the field is a remote-only addition (absent from config, present + // in the remote) inside an object whose gate is set (e.g. a cluster with a policy_id). The + // backend may extend such an object beyond what the bundle declares, so the addition is not + // treated as drift. We do not attribute the value to any particular source. + ReasonRemoteAddition = "remote_addition" // Special reason that results in removing this change from the plan ReasonDrop = "!drop" diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index f36da015e7a..2a8b01f0014 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -613,7 +613,7 @@ func shouldSkipRemoteAddition(cfg *dresources.ResourceLifecycleConfig, path *str if err != nil || allEmpty(value) { continue } - return deployplan.ReasonPolicyManaged, true + return deployplan.ReasonRemoteAddition, true } return "", false } diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 88413061c36..3ada183cdd3 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -522,7 +522,7 @@ func TestShouldSkipRemoteAddition(t *testing.T) { reason, ok := shouldSkipRemoteAddition(cfg, path, &change, tt.state) assert.Equal(t, tt.expected, ok) if tt.expected { - assert.Equal(t, deployplan.ReasonPolicyManaged, reason) + assert.Equal(t, deployplan.ReasonRemoteAddition, reason) } }) } From b19fec7014ab968e8664919e04e3f29e022a31b5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 16:14:52 +0200 Subject: [PATCH 10/25] acc: record the whole plan in policy_no_drift_variants, make it local-only The grepped gron lines were hard to read. Record the whole `bundle plan -o json` instead: with everything nested it is legible, and it shows the suppression at every cluster-spec location in one place, including that a policy-attached cluster suppresses backend-default fields (data_security_mode, lifecycle) as remote_addition too -- both still skip, so convergence is unchanged. The whole plan carries remote_state and *_attributes, which are cloud- and region-specific, so it cannot be recorded portably across the aws/azure/gcp sweep; the test becomes local-only. The suppression behavior stays cloud-verified by the projected jobs/cluster_policy tests. Co-authored-by: Isaac --- .../policy_no_drift_variants/out.test.toml | 3 +- .../policy_no_drift_variants/output.txt | 346 +++++++++++++++++- .../policy_no_drift_variants/script | 6 +- .../policy_no_drift_variants/test.toml | 10 +- 4 files changed, 341 insertions(+), 24 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml index f9f4880725d..59b56a2037c 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml @@ -1,4 +1,3 @@ -Cloud = true -CloudSlow = true +Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt index 52ad8c38dc2..97c4e2cbd77 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt @@ -9,27 +9,343 @@ Created pipelines.pipe Files: 6 uploaded, 0 deleted Resources: 5 created, 0 changed, 0 deleted, 0 unchanged -=== Every cluster-spec location converges: the policy tag is not drift anywhere +=== Every cluster-spec location converges: nothing to change on a second plan >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged -=== Confirm the tag really is present remotely and was classified, not just absent +=== The full plan: each cluster's policy-supplied fields are skipped as remote_addition >>> [CLI] bundle plan -o json -json.plan.resources.cluster_policies.pol.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}"; -json.plan.resources.clusters.standalone.remote_state.custom_tags.CostCenter = "from-policy"; -json.plan.resources.clusters.standalone.changes.custom_tags.action = "skip"; -json.plan.resources.clusters.standalone.changes.custom_tags.reason = "remote_addition"; -json.plan.resources.clusters.standalone.changes.custom_tags.remote.CostCenter = "from-policy"; -json.plan.resources.jobs.for_each_cluster.remote_state.tasks[0].for_each_task.task.new_cluster.custom_tags.CostCenter = "from-policy"; -json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.action = "skip"; -json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.reason = "remote_addition"; -json.plan.resources.jobs.for_each_cluster.changes.tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags.remote.CostCenter = "from-policy"; -json.plan.resources.jobs.task_cluster.remote_state.tasks[0].new_cluster.custom_tags.CostCenter = "from-policy"; -json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.action = "skip"; -json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.reason = "remote_addition"; -json.plan.resources.jobs.task_cluster.changes.tasks[task_key='t'].new_cluster.custom_tags.remote.CostCenter = "from-policy"; +{ + "plan_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.cluster_policies.pol": { + "action": "skip", + "remote_state": { + "definition": "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}", + "name": "test-policy-[UNIQUE_NAME]", + "policy_id": "[UUID]" + } + }, + "resources.clusters.standalone": { + "depends_on": [ + { + "node": "resources.cluster_policies.pol", + "label": "${resources.cluster_policies.pol.id}" + } + ], + "action": "skip", + "remote_state": { + "autotermination_minutes": 60, + "aws_attributes": { + "availability": "SPOT_WITH_FALLBACK", + "zone_id": "us-east-1c" + }, + "cluster_id": "[UUID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "custom_tags": { + "CostCenter": "from-policy" + }, + "enable_elastic_disk": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", + "lifecycle": { + "started": true + }, + "num_workers": 1, + "policy_id": "[UUID]", + "spark_version": "13.3.x-snapshot-scala2.12", + "state": "RUNNING" + }, + "changes": { + "aws_attributes": { + "action": "skip", + "reason": "managed", + "remote": { + "availability": "SPOT_WITH_FALLBACK", + "zone_id": "us-east-1c" + } + }, + "custom_tags": { + "action": "skip", + "reason": "remote_addition", + "remote": { + "CostCenter": "from-policy" + } + }, + "enable_elastic_disk": { + "action": "skip", + "reason": "empty", + "remote": false + }, + "lifecycle": { + "action": "skip", + "reason": "remote_addition", + "remote": { + "started": true + } + } + } + }, + "resources.jobs.for_each_cluster": { + "depends_on": [ + { + "node": "resources.cluster_policies.pol", + "label": "${resources.cluster_policies.pol.id}" + } + ], + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "test-for-each-cluster-[UNIQUE_NAME]", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "tasks": [ + { + "email_notifications": {}, + "for_each_task": { + "inputs": "[1,2]", + "task": { + "new_cluster": { + "custom_tags": { + "CostCenter": "from-policy" + }, + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 1, + "policy_id": "[UUID]", + "spark_version": "13.3.x-snapshot-scala2.12" + }, + "spark_python_task": { + "python_file": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/hello_world.py" + }, + "task_key": "inner" + } + }, + "run_if": "ALL_SUCCESS", + "task_key": "outer", + "timeout_seconds": 0 + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='outer'].email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags": { + "action": "skip", + "reason": "remote_addition", + "remote": { + "CostCenter": "from-policy" + } + }, + "tasks[task_key='outer'].run_if": { + "action": "skip", + "reason": "backend_default", + "remote": "ALL_SUCCESS" + }, + "tasks[task_key='outer'].timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.task_cluster": { + "depends_on": [ + { + "node": "resources.cluster_policies.pol", + "label": "${resources.cluster_policies.pol.id}" + } + ], + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "test-task-cluster-[UNIQUE_NAME]", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "tasks": [ + { + "email_notifications": {}, + "new_cluster": { + "aws_attributes": { + "availability": "SPOT_WITH_FALLBACK", + "zone_id": "us-east-1c" + }, + "custom_tags": { + "CostCenter": "from-policy" + }, + "data_security_mode": "SINGLE_USER", + "enable_elastic_disk": false, + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 1, + "policy_id": "[UUID]", + "spark_version": "13.3.x-snapshot-scala2.12" + }, + "run_if": "ALL_SUCCESS", + "spark_python_task": { + "python_file": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/hello_world.py" + }, + "task_key": "t", + "timeout_seconds": 0 + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='t'].email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='t'].new_cluster.aws_attributes": { + "action": "skip", + "reason": "managed", + "remote": { + "availability": "SPOT_WITH_FALLBACK", + "zone_id": "us-east-1c" + } + }, + "tasks[task_key='t'].new_cluster.custom_tags": { + "action": "skip", + "reason": "remote_addition", + "remote": { + "CostCenter": "from-policy" + } + }, + "tasks[task_key='t'].new_cluster.data_security_mode": { + "action": "skip", + "reason": "remote_addition", + "remote": "SINGLE_USER" + }, + "tasks[task_key='t'].new_cluster.enable_elastic_disk": { + "action": "skip", + "reason": "empty", + "remote": false + }, + "tasks[task_key='t'].run_if": { + "action": "skip", + "reason": "backend_default", + "remote": "ALL_SUCCESS" + }, + "tasks[task_key='t'].timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.pipelines.pipe": { + "depends_on": [ + { + "node": "resources.cluster_policies.pol", + "label": "${resources.cluster_policies.pol.id}" + } + ], + "action": "skip", + "remote_state": { + "channel": "CURRENT", + "clusters": [ + { + "label": "default", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 1, + "policy_id": "[UUID]" + } + ], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/metadata.json" + }, + "edition": "ADVANCED", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "id": "[UUID]", + "last_modified": [UNIX_TIME_MILLIS][2], + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/hello_world.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "state": "IDLE", + "storage": "dbfs:/pipelines/[UUID]" + }, + "changes": { + "storage": { + "action": "skip", + "reason": "backend_default", + "remote": "dbfs:/pipelines/[UUID]" + } + } + } + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script index af3a0ec4fca..a592828163c 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script @@ -7,8 +7,8 @@ trap cleanup EXIT trace $CLI bundle deploy -title "Every cluster-spec location converges: the policy tag is not drift anywhere\n" +title "Every cluster-spec location converges: nothing to change on a second plan\n" trace $CLI bundle plan -title "Confirm the tag really is present remotely and was classified, not just absent\n" -trace $CLI bundle plan -o json | gron.py | grep custom_tags +title "The full plan: each cluster's policy-supplied fields are skipped as remote_addition\n" +trace $CLI bundle plan -o json diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml index 1002b7e9e54..6c5ccdd12b9 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml @@ -1,7 +1,9 @@ -# Creating a real cluster made this the slowest test in the cloud suite (~8 min per env), and -# the per-location classification it checks is fully covered locally. The cheap job-based -# tests in this directory keep the core behaviour in every cloud run. -CloudSlow = true +# Local-only (overrides the inherited Cloud = true). This records the whole `bundle plan +# -o json` so the suppression is legible at every cluster-spec location; the plan's +# remote_state and *_attributes are cloud- and region-specific, so it cannot be recorded +# portably across the aws/azure/gcp sweep. The suppression behavior itself is cloud-verified +# by the projected jobs/cluster_policy tests. +Cloud = false RecordRequests = false From c6a36cc2e8d7921e755a43b11cf1077cdefb989e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 17:13:00 +0200 Subject: [PATCH 11/25] acc: record the pruned plan in policy_no_drift_variants; add prune_plan.py Replaces the whole-plan and gron approaches. prune_plan.py drops the parts of `bundle plan -o json` that a testserver cannot reproduce and that differ by cloud -- remote_state, changes with reason in --ignore-reasons (managed, backend_default by default), and changes whose path matches --ignore-keys -- leaving the config- and policy-driven entries. What survives here is the policy custom_tags skipped as remote_addition at every cluster-spec location, plus lifecycle; all cloud-independent, so the test runs on all clouds. Also fixes the confusing comment above `clusters:` in the fixture. Co-authored-by: Isaac --- acceptance/bin/prune_plan.py | 41 +++ .../databricks.yml.tmpl | 7 +- .../policy_no_drift_variants/out.test.toml | 3 +- .../policy_no_drift_variants/output.txt | 305 ++---------------- .../policy_no_drift_variants/script | 7 +- .../policy_no_drift_variants/test.toml | 13 +- 6 files changed, 89 insertions(+), 287 deletions(-) create mode 100644 acceptance/bin/prune_plan.py diff --git a/acceptance/bin/prune_plan.py b/acceptance/bin/prune_plan.py new file mode 100644 index 00000000000..48538f141cd --- /dev/null +++ b/acceptance/bin/prune_plan.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Prune the cloud-dependent parts of `bundle plan -o json` so the plan can be recorded as a +golden that is identical across clouds. + +Reads plan JSON on stdin, writes the pruned JSON on stdout. Per plan node it removes: + + - remote_state: the raw backend read (per-run driver/executor IPs, instance ids, + spark_context_id, timestamps, default_tags, ...), none of which is reproducible. + - changes entries whose reason is in --ignore-reasons (default: managed, backend_default) -- + values the backend chooses, e.g. {aws,azure,gcp}_attributes or node types, which differ by + cloud. + - changes entries whose field path contains any substring in --ignore-keys -- for fields a + backend sets to different values (so the same field lands under different reasons on + different clouds and cannot be matched by reason alone, e.g. enable_elastic_disk). + +What survives is cloud-independent: the action, and the changes driven by config or policy. +""" + +import argparse +import json +import sys + +parser = argparse.ArgumentParser() +parser.add_argument("--ignore-reasons", default="managed,backend_default") +parser.add_argument("--ignore-keys", default="") +args = parser.parse_args() + +ignore_reasons = {r for r in args.ignore_reasons.split(",") if r} +ignore_keys = [k for k in args.ignore_keys.split(",") if k] + +plan = json.load(sys.stdin) + +for node in plan.get("plan", {}).values(): + node.pop("remote_state", None) + changes = node.get("changes") or {} + for path, change in list(changes.items()): + if change.get("reason") in ignore_reasons or any(k in path for k in ignore_keys): + del changes[path] + +json.dump(plan, sys.stdout, indent=2, sort_keys=True) +print() diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl index 72b4684279d..370def57f28 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl @@ -13,8 +13,11 @@ resources: type: fixed value: from-policy - # Every place a cluster spec can appear must be covered by a rule; a gap here is how - # https://github.com/databricks/cli/issues/5179 stayed open after a partial fix. + # A cluster spec can appear in several places, each with its own ignore_remote_additions + # rule. This fixture puts a policy-attached cluster in each place -- a standalone cluster, a + # task's new_cluster, a for_each task's new_cluster, and a pipeline cluster -- so the rule is + # exercised at every location in one plan. (job_clusters is covered by the sibling + # jobs/cluster_policy tests.) The pipeline cluster is the deliberate no-rule case; see below. clusters: standalone: cluster_name: test-cluster-$UNIQUE_NAME diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml index 59b56a2037c..f9f4880725d 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml @@ -1,3 +1,4 @@ -Cloud = false +Cloud = true +CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt index 97c4e2cbd77..d1dc709239b 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt @@ -14,61 +14,19 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged -=== The full plan: each cluster's policy-supplied fields are skipped as remote_addition +=== The plan, minus cloud-dependent remote_state and managed *_attributes >>> [CLI] bundle plan -o json { - "plan_version": 2, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, "plan": { "resources.cluster_policies.pol": { - "action": "skip", - "remote_state": { - "definition": "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}", - "name": "test-policy-[UNIQUE_NAME]", - "policy_id": "[UUID]" - } + "action": "skip" }, "resources.clusters.standalone": { - "depends_on": [ - { - "node": "resources.cluster_policies.pol", - "label": "${resources.cluster_policies.pol.id}" - } - ], "action": "skip", - "remote_state": { - "autotermination_minutes": 60, - "aws_attributes": { - "availability": "SPOT_WITH_FALLBACK", - "zone_id": "us-east-1c" - }, - "cluster_id": "[UUID]", - "cluster_name": "test-cluster-[UNIQUE_NAME]", - "custom_tags": { - "CostCenter": "from-policy" - }, - "enable_elastic_disk": false, - "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", - "lifecycle": { - "started": true - }, - "num_workers": 1, - "policy_id": "[UUID]", - "spark_version": "13.3.x-snapshot-scala2.12", - "state": "RUNNING" - }, "changes": { - "aws_attributes": { - "action": "skip", - "reason": "managed", - "remote": { - "availability": "SPOT_WITH_FALLBACK", - "zone_id": "us-east-1c" - } - }, "custom_tags": { "action": "skip", "reason": "remote_addition", @@ -76,11 +34,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "CostCenter": "from-policy" } }, - "enable_elastic_disk": { - "action": "skip", - "reason": "empty", - "remote": false - }, "lifecycle": { "action": "skip", "reason": "remote_addition", @@ -88,263 +41,63 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "started": true } } - } - }, - "resources.jobs.for_each_cluster": { + }, "depends_on": [ { - "node": "resources.cluster_policies.pol", - "label": "${resources.cluster_policies.pol.id}" + "label": "${resources.cluster_policies.pol.id}", + "node": "resources.cluster_policies.pol" } - ], + ] + }, + "resources.jobs.for_each_cluster": { "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "test-for-each-cluster-[UNIQUE_NAME]", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "tasks": [ - { - "email_notifications": {}, - "for_each_task": { - "inputs": "[1,2]", - "task": { - "new_cluster": { - "custom_tags": { - "CostCenter": "from-policy" - }, - "node_type_id": "[NODE_TYPE_ID]", - "num_workers": 1, - "policy_id": "[UUID]", - "spark_version": "13.3.x-snapshot-scala2.12" - }, - "spark_python_task": { - "python_file": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/hello_world.py" - }, - "task_key": "inner" - } - }, - "run_if": "ALL_SUCCESS", - "task_key": "outer", - "timeout_seconds": 0 - } - ], - "timeout_seconds": 0, - "webhook_notifications": {} - }, "changes": { - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "tasks[task_key='outer'].email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, "tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags": { "action": "skip", "reason": "remote_addition", "remote": { "CostCenter": "from-policy" } - }, - "tasks[task_key='outer'].run_if": { - "action": "skip", - "reason": "backend_default", - "remote": "ALL_SUCCESS" - }, - "tasks[task_key='outer'].timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} } - } - }, - "resources.jobs.task_cluster": { + }, "depends_on": [ { - "node": "resources.cluster_policies.pol", - "label": "${resources.cluster_policies.pol.id}" + "label": "${resources.cluster_policies.pol.id}", + "node": "resources.cluster_policies.pol" } - ], + ] + }, + "resources.jobs.task_cluster": { "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][1], - "creator_user_name": "[USERNAME]", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "test-task-cluster-[UNIQUE_NAME]", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "tasks": [ - { - "email_notifications": {}, - "new_cluster": { - "aws_attributes": { - "availability": "SPOT_WITH_FALLBACK", - "zone_id": "us-east-1c" - }, - "custom_tags": { - "CostCenter": "from-policy" - }, - "data_security_mode": "SINGLE_USER", - "enable_elastic_disk": false, - "node_type_id": "[NODE_TYPE_ID]", - "num_workers": 1, - "policy_id": "[UUID]", - "spark_version": "13.3.x-snapshot-scala2.12" - }, - "run_if": "ALL_SUCCESS", - "spark_python_task": { - "python_file": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/hello_world.py" - }, - "task_key": "t", - "timeout_seconds": 0 - } - ], - "timeout_seconds": 0, - "webhook_notifications": {} - }, "changes": { - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "tasks[task_key='t'].email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "tasks[task_key='t'].new_cluster.aws_attributes": { - "action": "skip", - "reason": "managed", - "remote": { - "availability": "SPOT_WITH_FALLBACK", - "zone_id": "us-east-1c" - } - }, "tasks[task_key='t'].new_cluster.custom_tags": { "action": "skip", "reason": "remote_addition", "remote": { "CostCenter": "from-policy" } - }, - "tasks[task_key='t'].new_cluster.data_security_mode": { - "action": "skip", - "reason": "remote_addition", - "remote": "SINGLE_USER" - }, - "tasks[task_key='t'].new_cluster.enable_elastic_disk": { - "action": "skip", - "reason": "empty", - "remote": false - }, - "tasks[task_key='t'].run_if": { - "action": "skip", - "reason": "backend_default", - "remote": "ALL_SUCCESS" - }, - "tasks[task_key='t'].timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} } - } - }, - "resources.pipelines.pipe": { + }, "depends_on": [ { - "node": "resources.cluster_policies.pol", - "label": "${resources.cluster_policies.pol.id}" + "label": "${resources.cluster_policies.pol.id}", + "node": "resources.cluster_policies.pol" } - ], + ] + }, + "resources.pipelines.pipe": { "action": "skip", - "remote_state": { - "channel": "CURRENT", - "clusters": [ - { - "label": "default", - "node_type_id": "[NODE_TYPE_ID]", - "num_workers": 1, - "policy_id": "[UUID]" - } - ], - "creator_user_name": "[USERNAME]", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/metadata.json" - }, - "edition": "ADVANCED", - "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", - "id": "[UUID]", - "last_modified": [UNIX_TIME_MILLIS][2], - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/hello_world.py" - } - } - ], - "name": "test-pipeline-[UNIQUE_NAME]", - "pipeline_id": "[UUID]", - "run_as_user_name": "[USERNAME]", - "state": "IDLE", - "storage": "dbfs:/pipelines/[UUID]" - }, - "changes": { - "storage": { - "action": "skip", - "reason": "backend_default", - "remote": "dbfs:/pipelines/[UUID]" + "changes": {}, + "depends_on": [ + { + "label": "${resources.cluster_policies.pol.id}", + "node": "resources.cluster_policies.pol" } - } + ] } - } + }, + "plan_version": 2, + "serial": 1 } >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script index a592828163c..249dee30314 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script @@ -6,9 +6,12 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy +read_id.py pol > /dev/null title "Every cluster-spec location converges: nothing to change on a second plan\n" trace $CLI bundle plan -title "The full plan: each cluster's policy-supplied fields are skipped as remote_addition\n" -trace $CLI bundle plan -o json +title "The plan, minus cloud-dependent remote_state and managed *_attributes\n" +trace $CLI bundle plan -o json | prune_plan.py \ + --ignore-reasons managed,backend_default,empty \ + --ignore-keys instance_pool_id,node_type_id,enable_local_disk_encryption,data_security_mode diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml index 6c5ccdd12b9..1baf85cdbbc 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml @@ -1,9 +1,10 @@ -# Local-only (overrides the inherited Cloud = true). This records the whole `bundle plan -# -o json` so the suppression is legible at every cluster-spec location; the plan's -# remote_state and *_attributes are cloud- and region-specific, so it cannot be recorded -# portably across the aws/azure/gcp sweep. The suppression behavior itself is cloud-verified -# by the projected jobs/cluster_policy tests. -Cloud = false +# Records a jq projection of the plan -- the policy-supplied custom_tags at each cluster-spec +# location -- rather than the whole plan: the full remote_state is the per-run cluster/job GET +# (driver IPs, instance ids, spark_context_id) which no testserver can reproduce. The +# projection is identical on every cloud because the tag is policy-set, so all clouds run. +# CloudSlow keeps the real cluster boot out of the per-PR short run. +Cloud = true +CloudSlow = true RecordRequests = false From b78d7cf45b8ebc89842cd5b991de05127aec0612 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 17:18:33 +0200 Subject: [PATCH 12/25] acc: make policy_no_drift_variants local-only It creates a standalone cluster, which boots at deploy time and repeatedly hit aws capacity limits, and whose full remote_state is a per-run cluster GET no testserver reproduces. Run it locally only, where the testserver is deterministic, and cover the four cluster-spec locations there. The suppression behavior on a real workspace stays covered by the jobs/cluster_policy tests. With cloud parity no longer required, the plan projection drops the cloud-portability --ignore-keys and keeps prune_plan.py's defaults. Co-authored-by: Isaac --- .../policy_no_drift_variants/out.test.toml | 3 +- .../policy_no_drift_variants/output.txt | 67 ++++++++++++++++++- .../policy_no_drift_variants/script | 6 +- .../policy_no_drift_variants/test.toml | 13 ++-- 4 files changed, 75 insertions(+), 14 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml index f9f4880725d..59b56a2037c 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml @@ -1,4 +1,3 @@ -Cloud = true -CloudSlow = true +Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt index d1dc709239b..542765f4422 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt @@ -14,7 +14,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged -=== The plan, minus cloud-dependent remote_state and managed *_attributes +=== The plan, minus remote_state and backend-chosen changes (managed, backend_default) >>> [CLI] bundle plan -o json { @@ -34,6 +34,11 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "CostCenter": "from-policy" } }, + "enable_elastic_disk": { + "action": "skip", + "reason": "empty", + "remote": false + }, "lifecycle": { "action": "skip", "reason": "remote_addition", @@ -52,12 +57,37 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "resources.jobs.for_each_cluster": { "action": "skip", "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='outer'].email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, "tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags": { "action": "skip", "reason": "remote_addition", "remote": { "CostCenter": "from-policy" } + }, + "tasks[task_key='outer'].timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} } }, "depends_on": [ @@ -70,12 +100,47 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "resources.jobs.task_cluster": { "action": "skip", "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='t'].email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, "tasks[task_key='t'].new_cluster.custom_tags": { "action": "skip", "reason": "remote_addition", "remote": { "CostCenter": "from-policy" } + }, + "tasks[task_key='t'].new_cluster.data_security_mode": { + "action": "skip", + "reason": "remote_addition", + "remote": "SINGLE_USER" + }, + "tasks[task_key='t'].new_cluster.enable_elastic_disk": { + "action": "skip", + "reason": "empty", + "remote": false + }, + "tasks[task_key='t'].timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} } }, "depends_on": [ diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script index 249dee30314..128d2c34738 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script @@ -11,7 +11,5 @@ read_id.py pol > /dev/null title "Every cluster-spec location converges: nothing to change on a second plan\n" trace $CLI bundle plan -title "The plan, minus cloud-dependent remote_state and managed *_attributes\n" -trace $CLI bundle plan -o json | prune_plan.py \ - --ignore-reasons managed,backend_default,empty \ - --ignore-keys instance_pool_id,node_type_id,enable_local_disk_encryption,data_security_mode +title "The plan, minus remote_state and backend-chosen changes (managed, backend_default)\n" +trace $CLI bundle plan -o json | prune_plan.py diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml index 1baf85cdbbc..a158976fbbd 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml @@ -1,10 +1,9 @@ -# Records a jq projection of the plan -- the policy-supplied custom_tags at each cluster-spec -# location -- rather than the whole plan: the full remote_state is the per-run cluster/job GET -# (driver IPs, instance ids, spark_context_id) which no testserver can reproduce. The -# projection is identical on every cloud because the tag is policy-set, so all clouds run. -# CloudSlow keeps the real cluster boot out of the per-PR short run. -Cloud = true -CloudSlow = true +# Local-only. This test creates a standalone cluster, which only the local testserver can do +# reliably: on a real workspace it boots at deploy and repeatedly hit capacity limits, and the +# full remote_state is a per-run cluster GET no testserver reproduces. The suppression behavior +# is verified against a real workspace by the jobs/cluster_policy tests; here we cover every +# cluster-spec location (standalone cluster, task new_cluster, for_each new_cluster, pipeline). +Cloud = false RecordRequests = false From e447832219490654d0610abca77ab8337f3c6591 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 17:25:44 +0200 Subject: [PATCH 13/25] acc: rename pol -> my_policy and drop dead read_id in policy_no_drift_variants read_id.py pol only registered an id replacement; after pruning remote_state the policy id no longer appears in the output, so it is dead. Rename the resource key pol to my_policy while here. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 10 ++++----- .../policy_no_drift_variants/output.txt | 22 +++++++++---------- .../policy_no_drift_variants/script | 1 - 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl index 370def57f28..3ed70bb4b58 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl @@ -6,7 +6,7 @@ workspace: resources: cluster_policies: - pol: + my_policy: name: test-policy-$UNIQUE_NAME definition: custom_tags.CostCenter: @@ -27,7 +27,7 @@ resources: # cloud suite (6-8 minutes per env), and the policy tag under test is unaffected. instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 - policy_id: ${resources.cluster_policies.pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} jobs: task_cluster: @@ -35,7 +35,7 @@ resources: tasks: - task_key: t new_cluster: - policy_id: ${resources.cluster_policies.pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 @@ -51,7 +51,7 @@ resources: task: task_key: inner new_cluster: - policy_id: ${resources.cluster_policies.pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 @@ -66,7 +66,7 @@ resources: name: test-pipeline-$UNIQUE_NAME clusters: - label: default - policy_id: ${resources.cluster_policies.pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} node_type_id: $NODE_TYPE_ID num_workers: 1 libraries: diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt index 542765f4422..9cb21ec7c59 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Created cluster_policies.pol +Created cluster_policies.my_policy Created clusters.standalone Created jobs.for_each_cluster Created jobs.task_cluster @@ -21,7 +21,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "plan": { - "resources.cluster_policies.pol": { + "resources.cluster_policies.my_policy": { "action": "skip" }, "resources.clusters.standalone": { @@ -49,8 +49,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged }, "depends_on": [ { - "label": "${resources.cluster_policies.pol.id}", - "node": "resources.cluster_policies.pol" + "label": "${resources.cluster_policies.my_policy.id}", + "node": "resources.cluster_policies.my_policy" } ] }, @@ -92,8 +92,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged }, "depends_on": [ { - "label": "${resources.cluster_policies.pol.id}", - "node": "resources.cluster_policies.pol" + "label": "${resources.cluster_policies.my_policy.id}", + "node": "resources.cluster_policies.my_policy" } ] }, @@ -145,8 +145,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged }, "depends_on": [ { - "label": "${resources.cluster_policies.pol.id}", - "node": "resources.cluster_policies.pol" + "label": "${resources.cluster_policies.my_policy.id}", + "node": "resources.cluster_policies.my_policy" } ] }, @@ -155,8 +155,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "changes": {}, "depends_on": [ { - "label": "${resources.cluster_policies.pol.id}", - "node": "resources.cluster_policies.pol" + "label": "${resources.cluster_policies.my_policy.id}", + "node": "resources.cluster_policies.my_policy" } ] } @@ -167,7 +167,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.cluster_policies.pol + delete resources.cluster_policies.my_policy delete resources.clusters.standalone delete resources.jobs.for_each_cluster delete resources.jobs.task_cluster diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script index 128d2c34738..258d6e21140 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script @@ -6,7 +6,6 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -read_id.py pol > /dev/null title "Every cluster-spec location converges: nothing to change on a second plan\n" trace $CLI bundle plan From 78c76def42a19d9f97516e8dc89db1c9f067a6e6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 17:31:39 +0200 Subject: [PATCH 14/25] acc: rename the policy resource to my_pol across the cluster_policy tests Consistent, slightly more descriptive key than pol. Regenerated locally against the testserver; the rename is a resource-name string, independent of the cloud. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 10 ++++----- .../policy_no_drift_variants/output.txt | 22 +++++++++---------- .../fixed_value_conflict/databricks.yml.tmpl | 4 ++-- .../fixed_value_conflict/output.txt | 2 +- .../fixed_values_applied/databricks.yml.tmpl | 4 ++-- .../fixed_values_applied/output.txt | 6 ++--- .../fixed_values_applied/script | 2 +- .../policy_drift/databricks.yml.tmpl | 6 ++--- .../cluster_policy/policy_drift/output.txt | 4 ++-- .../jobs/cluster_policy/policy_drift/script | 2 +- 10 files changed, 31 insertions(+), 31 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl index 3ed70bb4b58..dbb251a4893 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl @@ -6,7 +6,7 @@ workspace: resources: cluster_policies: - my_policy: + my_pol: name: test-policy-$UNIQUE_NAME definition: custom_tags.CostCenter: @@ -27,7 +27,7 @@ resources: # cloud suite (6-8 minutes per env), and the policy tag under test is unaffected. instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 - policy_id: ${resources.cluster_policies.my_policy.id} + policy_id: ${resources.cluster_policies.my_pol.id} jobs: task_cluster: @@ -35,7 +35,7 @@ resources: tasks: - task_key: t new_cluster: - policy_id: ${resources.cluster_policies.my_policy.id} + policy_id: ${resources.cluster_policies.my_pol.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 @@ -51,7 +51,7 @@ resources: task: task_key: inner new_cluster: - policy_id: ${resources.cluster_policies.my_policy.id} + policy_id: ${resources.cluster_policies.my_pol.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 @@ -66,7 +66,7 @@ resources: name: test-pipeline-$UNIQUE_NAME clusters: - label: default - policy_id: ${resources.cluster_policies.my_policy.id} + policy_id: ${resources.cluster_policies.my_pol.id} node_type_id: $NODE_TYPE_ID num_workers: 1 libraries: diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt index 9cb21ec7c59..b66364cff31 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Created cluster_policies.my_policy +Created cluster_policies.my_pol Created clusters.standalone Created jobs.for_each_cluster Created jobs.task_cluster @@ -21,7 +21,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "plan": { - "resources.cluster_policies.my_policy": { + "resources.cluster_policies.my_pol": { "action": "skip" }, "resources.clusters.standalone": { @@ -49,8 +49,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged }, "depends_on": [ { - "label": "${resources.cluster_policies.my_policy.id}", - "node": "resources.cluster_policies.my_policy" + "label": "${resources.cluster_policies.my_pol.id}", + "node": "resources.cluster_policies.my_pol" } ] }, @@ -92,8 +92,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged }, "depends_on": [ { - "label": "${resources.cluster_policies.my_policy.id}", - "node": "resources.cluster_policies.my_policy" + "label": "${resources.cluster_policies.my_pol.id}", + "node": "resources.cluster_policies.my_pol" } ] }, @@ -145,8 +145,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged }, "depends_on": [ { - "label": "${resources.cluster_policies.my_policy.id}", - "node": "resources.cluster_policies.my_policy" + "label": "${resources.cluster_policies.my_pol.id}", + "node": "resources.cluster_policies.my_pol" } ] }, @@ -155,8 +155,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "changes": {}, "depends_on": [ { - "label": "${resources.cluster_policies.my_policy.id}", - "node": "resources.cluster_policies.my_policy" + "label": "${resources.cluster_policies.my_pol.id}", + "node": "resources.cluster_policies.my_pol" } ] } @@ -167,7 +167,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.cluster_policies.my_policy + delete resources.cluster_policies.my_pol delete resources.clusters.standalone delete resources.jobs.for_each_cluster delete resources.jobs.task_cluster diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl index 7264a9bb25d..3d7c9b48db8 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl @@ -6,7 +6,7 @@ workspace: resources: cluster_policies: - pol: + my_pol: name: test-policy-$UNIQUE_NAME definition: custom_tags.CostCenter: @@ -20,7 +20,7 @@ resources: job_clusters: - job_cluster_key: small new_cluster: - policy_id: ${resources.cluster_policies.pol.id} + policy_id: ${resources.cluster_policies.my_pol.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt index 52729d259af..5ecf8373992 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt @@ -16,7 +16,7 @@ Exit code: 1 >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.cluster_policies.pol + delete resources.cluster_policies.my_pol All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl index d03bb2c5775..1af567580ef 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl @@ -6,7 +6,7 @@ workspace: resources: cluster_policies: - pol: + my_pol: name: test-policy-$UNIQUE_NAME definition: spark_version: @@ -24,7 +24,7 @@ resources: job_clusters: - job_cluster_key: small new_cluster: - policy_id: ${resources.cluster_policies.pol.id} + policy_id: ${resources.cluster_policies.my_pol.id} node_type_id: $NODE_TYPE_ID num_workers: 1 tasks: diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt index 367a4ee4e9c..6dc79ade732 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Created cluster_policies.pol +Created cluster_policies.my_pol Created jobs.j Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged @@ -19,7 +19,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === spark_version is not covered by any backend_defaults rule, so show its verdict >>> [CLI] bundle plan -o json -json.plan.resources.cluster_policies.pol.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"policy-supplied\"},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-snapshot-scala2.12\"}}"; +json.plan.resources.cluster_policies.my_pol.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"policy-supplied\"},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-snapshot-scala2.12\"}}"; json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "policy-supplied"; json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.spark_version = "13.3.x-snapshot-scala2.12"; json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.action = "skip"; @@ -31,7 +31,7 @@ json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_clu >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.cluster_policies.pol + delete resources.cluster_policies.my_pol delete resources.jobs.j All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script index b02afa490c6..cc2e81a0290 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script @@ -9,7 +9,7 @@ trace $CLI bundle deploy # Mask the server-assigned policy id (real backend returns a hex id the built-in # UUID replacement misses). -read_id.py pol > /dev/null +read_id.py my_pol > /dev/null job_id=$(read_id.py j) title "Did the fixed policy SUPPLY spark_version and custom_tags, or only validate them?\n" diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl index 148576b2dc1..3f1c2581c6d 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl @@ -6,7 +6,7 @@ workspace: resources: cluster_policies: - pol: + my_pol: name: test-policy-$UNIQUE_NAME definition: custom_tags.CostCenter: @@ -20,7 +20,7 @@ resources: job_clusters: - job_cluster_key: small new_cluster: - policy_id: ${resources.cluster_policies.pol.id} + policy_id: ${resources.cluster_policies.my_pol.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 @@ -37,7 +37,7 @@ resources: job_clusters: - job_cluster_key: small new_cluster: - policy_id: ${resources.cluster_policies.pol.id} + policy_id: ${resources.cluster_policies.my_pol.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt index 3ccf0c91dfb..49309c68b35 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Created cluster_policies.pol +Created cluster_policies.my_pol Created jobs.no_policy Created jobs.owned_tag Created jobs.with_policy @@ -47,7 +47,7 @@ json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'] >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.cluster_policies.pol + delete resources.cluster_policies.my_pol delete resources.jobs.no_policy delete resources.jobs.owned_tag delete resources.jobs.with_policy diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script index 4eb274bdeb5..983bfbe0004 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script @@ -6,7 +6,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -read_id.py pol > /dev/null +read_id.py my_pol > /dev/null no_policy_id="$(read_id.py no_policy)" owned_tag_id="$(read_id.py owned_tag)" From 6d22e3b5aa3b4a7d0d0d438b7cf5c5e0b14aae6d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 18:19:56 +0200 Subject: [PATCH 15/25] acc: use my_policy (not my_pol) for the policy resource key Co-authored-by: Isaac --- .../databricks.yml.tmpl | 10 ++++----- .../policy_no_drift_variants/output.txt | 22 +++++++++---------- .../fixed_value_conflict/databricks.yml.tmpl | 4 ++-- .../fixed_value_conflict/output.txt | 2 +- .../fixed_values_applied/databricks.yml.tmpl | 4 ++-- .../fixed_values_applied/output.txt | 6 ++--- .../fixed_values_applied/script | 2 +- .../policy_drift/databricks.yml.tmpl | 6 ++--- .../cluster_policy/policy_drift/output.txt | 4 ++-- .../jobs/cluster_policy/policy_drift/script | 2 +- 10 files changed, 31 insertions(+), 31 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl index dbb251a4893..3ed70bb4b58 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl @@ -6,7 +6,7 @@ workspace: resources: cluster_policies: - my_pol: + my_policy: name: test-policy-$UNIQUE_NAME definition: custom_tags.CostCenter: @@ -27,7 +27,7 @@ resources: # cloud suite (6-8 minutes per env), and the policy tag under test is unaffected. instance_pool_id: $TEST_INSTANCE_POOL_ID num_workers: 1 - policy_id: ${resources.cluster_policies.my_pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} jobs: task_cluster: @@ -35,7 +35,7 @@ resources: tasks: - task_key: t new_cluster: - policy_id: ${resources.cluster_policies.my_pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 @@ -51,7 +51,7 @@ resources: task: task_key: inner new_cluster: - policy_id: ${resources.cluster_policies.my_pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 @@ -66,7 +66,7 @@ resources: name: test-pipeline-$UNIQUE_NAME clusters: - label: default - policy_id: ${resources.cluster_policies.my_pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} node_type_id: $NODE_TYPE_ID num_workers: 1 libraries: diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt index b66364cff31..9cb21ec7c59 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Created cluster_policies.my_pol +Created cluster_policies.my_policy Created clusters.standalone Created jobs.for_each_cluster Created jobs.task_cluster @@ -21,7 +21,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "plan": { - "resources.cluster_policies.my_pol": { + "resources.cluster_policies.my_policy": { "action": "skip" }, "resources.clusters.standalone": { @@ -49,8 +49,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged }, "depends_on": [ { - "label": "${resources.cluster_policies.my_pol.id}", - "node": "resources.cluster_policies.my_pol" + "label": "${resources.cluster_policies.my_policy.id}", + "node": "resources.cluster_policies.my_policy" } ] }, @@ -92,8 +92,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged }, "depends_on": [ { - "label": "${resources.cluster_policies.my_pol.id}", - "node": "resources.cluster_policies.my_pol" + "label": "${resources.cluster_policies.my_policy.id}", + "node": "resources.cluster_policies.my_policy" } ] }, @@ -145,8 +145,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged }, "depends_on": [ { - "label": "${resources.cluster_policies.my_pol.id}", - "node": "resources.cluster_policies.my_pol" + "label": "${resources.cluster_policies.my_policy.id}", + "node": "resources.cluster_policies.my_policy" } ] }, @@ -155,8 +155,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "changes": {}, "depends_on": [ { - "label": "${resources.cluster_policies.my_pol.id}", - "node": "resources.cluster_policies.my_pol" + "label": "${resources.cluster_policies.my_policy.id}", + "node": "resources.cluster_policies.my_policy" } ] } @@ -167,7 +167,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.cluster_policies.my_pol + delete resources.cluster_policies.my_policy delete resources.clusters.standalone delete resources.jobs.for_each_cluster delete resources.jobs.task_cluster diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl index 3d7c9b48db8..65118118080 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl @@ -6,7 +6,7 @@ workspace: resources: cluster_policies: - my_pol: + my_policy: name: test-policy-$UNIQUE_NAME definition: custom_tags.CostCenter: @@ -20,7 +20,7 @@ resources: job_clusters: - job_cluster_key: small new_cluster: - policy_id: ${resources.cluster_policies.my_pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt index 5ecf8373992..57f311c76bd 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt @@ -16,7 +16,7 @@ Exit code: 1 >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.cluster_policies.my_pol + delete resources.cluster_policies.my_policy All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl index 1af567580ef..8408cecd217 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl @@ -6,7 +6,7 @@ workspace: resources: cluster_policies: - my_pol: + my_policy: name: test-policy-$UNIQUE_NAME definition: spark_version: @@ -24,7 +24,7 @@ resources: job_clusters: - job_cluster_key: small new_cluster: - policy_id: ${resources.cluster_policies.my_pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} node_type_id: $NODE_TYPE_ID num_workers: 1 tasks: diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt index 6dc79ade732..fd1786a5d42 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Created cluster_policies.my_pol +Created cluster_policies.my_policy Created jobs.j Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged @@ -19,7 +19,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === spark_version is not covered by any backend_defaults rule, so show its verdict >>> [CLI] bundle plan -o json -json.plan.resources.cluster_policies.my_pol.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"policy-supplied\"},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-snapshot-scala2.12\"}}"; +json.plan.resources.cluster_policies.my_policy.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"policy-supplied\"},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-snapshot-scala2.12\"}}"; json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "policy-supplied"; json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.spark_version = "13.3.x-snapshot-scala2.12"; json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.action = "skip"; @@ -31,7 +31,7 @@ json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_clu >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.cluster_policies.my_pol + delete resources.cluster_policies.my_policy delete resources.jobs.j All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script index cc2e81a0290..6d871fee2b3 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script @@ -9,7 +9,7 @@ trace $CLI bundle deploy # Mask the server-assigned policy id (real backend returns a hex id the built-in # UUID replacement misses). -read_id.py my_pol > /dev/null +read_id.py my_policy > /dev/null job_id=$(read_id.py j) title "Did the fixed policy SUPPLY spark_version and custom_tags, or only validate them?\n" diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl index 3f1c2581c6d..74867535ae2 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl @@ -6,7 +6,7 @@ workspace: resources: cluster_policies: - my_pol: + my_policy: name: test-policy-$UNIQUE_NAME definition: custom_tags.CostCenter: @@ -20,7 +20,7 @@ resources: job_clusters: - job_cluster_key: small new_cluster: - policy_id: ${resources.cluster_policies.my_pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 @@ -37,7 +37,7 @@ resources: job_clusters: - job_cluster_key: small new_cluster: - policy_id: ${resources.cluster_policies.my_pol.id} + policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt index 49309c68b35..08b0dc227cd 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Created cluster_policies.my_pol +Created cluster_policies.my_policy Created jobs.no_policy Created jobs.owned_tag Created jobs.with_policy @@ -47,7 +47,7 @@ json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'] >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.cluster_policies.my_pol + delete resources.cluster_policies.my_policy delete resources.jobs.no_policy delete resources.jobs.owned_tag delete resources.jobs.with_policy diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script index 983bfbe0004..a15dc422476 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script @@ -6,7 +6,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -read_id.py my_pol > /dev/null +read_id.py my_policy > /dev/null no_policy_id="$(read_id.py no_policy)" owned_tag_id="$(read_id.py owned_tag)" From e0a88b553e0db459f1977bddc4e3139f7028ce74 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 22:41:29 +0200 Subject: [PATCH 16/25] ci: re-trigger checks The pull_request test workflow did not spawn on the previous few pushes (they landed during a GitHub Actions/JFrog incident window). Empty commit to force a fresh synchronize event; squashed at merge. Co-authored-by: Isaac From b1518e0c60df04eb94c51d179436cd3c70b7f5fa Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 10 Sep 2026 14:57:23 +0200 Subject: [PATCH 17/25] acc: opt policy_no_drift_variants out of the DMS matrix after merging main Merging main brought deployment-history recording, which adds a features block and deployment.version_id changes to the plan under DMS=true, diverging from the DMS="" variant. That recording is orthogonal to drift suppression and is covered by the jobs/cluster_policy tests, so run this local-only test once. Co-authored-by: Isaac --- .../cluster_policies/policy_no_drift_variants/out.test.toml | 2 +- .../cluster_policies/policy_no_drift_variants/output.txt | 2 +- .../cluster_policies/policy_no_drift_variants/test.toml | 5 +++++ bundle/direct/bundle_plan_test.go | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml index 59b56a2037c..d0b77dbc454 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [] diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt index 9cb21ec7c59..06871c01add 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt @@ -161,7 +161,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged ] } }, - "plan_version": 2, + "plan_version": [PLAN_VERSION], "serial": 1 } diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml index a158976fbbd..551103bf904 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml @@ -5,6 +5,11 @@ # cluster-spec location (standalone cluster, task new_cluster, for_each new_cluster, pipeline). Cloud = false +# Deployment-history recording (the DMS matrix) is orthogonal to drift suppression and +# adds cloud/version-dependent plan fields; run this test once. DMS is covered by the +# jobs/cluster_policy tests. +EnvMatrix.DMS = [] + RecordRequests = false Ignore = [ diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 6921d709a73..1ebcecede36 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -529,6 +529,7 @@ func TestShouldSkipRemoteAddition(t *testing.T) { }) } } + // Types for TestPrepareChangesWholeBlockOverlap: two levels of nesting under an // optional pointer. type threeWayInner struct { From 2b5c067b8d77b8e56ad202d550fe6e2485a89a0c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 10 Sep 2026 15:11:06 +0200 Subject: [PATCH 18/25] acc: make prune_plan.py executable and pin policy_no_drift_variants to DMS="" Two CI failures on the post-merge run: - prune_plan.py was committed 100644, so on CI it ran as "Permission denied" (exit 126); the plan output was the error and .databricks leaked as an unexpected file. Committed 100755 like the other bin/ helpers. - EnvMatrix.DMS = [] gave the test no DMS tag, so it ran on BOTH the direct and directdms CI cells (the same trap config.go documents for DATABRICKS_BUNDLE_ENGINE=[]); on directdms the plan gained deployment-history fields the DMS-off golden lacks. Pin to DMS = "" to run once on the direct cell. Co-authored-by: Isaac --- acceptance/bin/prune_plan.py | 0 .../policy_no_drift_variants/out.test.toml | 2 +- .../cluster_policies/policy_no_drift_variants/test.toml | 9 +++++---- 3 files changed, 6 insertions(+), 5 deletions(-) mode change 100644 => 100755 acceptance/bin/prune_plan.py diff --git a/acceptance/bin/prune_plan.py b/acceptance/bin/prune_plan.py old mode 100644 new mode 100755 diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml index d0b77dbc454..27ec2a7fcd6 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml index 551103bf904..eb681f0c5d6 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml +++ b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml @@ -5,10 +5,11 @@ # cluster-spec location (standalone cluster, task new_cluster, for_each new_cluster, pipeline). Cloud = false -# Deployment-history recording (the DMS matrix) is orthogonal to drift suppression and -# adds cloud/version-dependent plan fields; run this test once. DMS is covered by the -# jobs/cluster_policy tests. -EnvMatrix.DMS = [] +# Deployment-history recording (DMS) adds version-dependent plan fields and .databricks +# artifacts, and it is orthogonal to drift suppression (covered by the jobs/cluster_policy +# tests). Pin to the off value so the test runs once on the "direct" CI cell; DMS = [] would +# instead run on BOTH the direct and directdms cells (same trap as DATABRICKS_BUNDLE_ENGINE=[]). +EnvMatrix.DMS = [""] RecordRequests = false From 981bde22a538686550dd73305cdccecd054286d8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 11 Sep 2026 09:57:26 +0200 Subject: [PATCH 19/25] Reword changelog: "direct:" prefix Co-authored-by: Isaac --- .nextchanges/bundles/cluster-policy-no-drift.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/cluster-policy-no-drift.md b/.nextchanges/bundles/cluster-policy-no-drift.md index f3f2d04dd64..92b5c88e573 100644 --- a/.nextchanges/bundles/cluster-policy-no-drift.md +++ b/.nextchanges/bundles/cluster-policy-no-drift.md @@ -1 +1 @@ -* On the direct engine, `bundle plan` no longer reports a permanent update on a cluster that uses a cluster policy: when the cluster spec sets `policy_id`, a field present in the remote but absent from the bundle config is not treated as drift. ([#6531](https://github.com/databricks/cli/pull/6531)) +* direct: `bundle plan` no longer reports a permanent update on a cluster that uses a cluster policy: when the cluster spec sets `policy_id`, a field present in the remote but absent from the bundle config is not treated as drift. ([#6531](https://github.com/databricks/cli/pull/6531)) From df2ce3c5eb64ac8d95eaafb85ce4525dba2faf04 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 11 Sep 2026 10:53:15 +0200 Subject: [PATCH 20/25] acc: split cluster-policy tests into single-condition tests Replace the fat, overlapping tests with focused ones, each pinning one dimension: jobs/cluster_policy/ fixed_addition - fixed policy supplies an omitted scalar + map key default_flag - defaultValue supplied only with apply_policy_default_values config_conflicts_policy - config value contradicting a fixed policy -> 400 owned_tag_drift - out-of-band change/removal of a config-owned tag -> update policy_id_scoping - policy_id gates per cluster spec, across every jobs location clusters/cluster_policy - standalone cluster (CloudSlow; boots a real cluster) pipelines/cluster_policy - pipeline cluster: no policy expansion, converges Splitting per resource lets jobs and pipelines run on cloud (no cluster boot), while the standalone clusters resource is isolated as CloudSlow. Outputs are projected to the policy fields so they are cloud- and DMS-invariant, which also retires prune_plan.py and the lumped local-only policy_no_drift_variants. Co-authored-by: Isaac --- acceptance/bin/prune_plan.py | 41 ---- .../databricks.yml.tmpl | 74 ------- .../policy_no_drift_variants/output.txt | 182 ------------------ .../policy_no_drift_variants/script | 14 -- .../policy_no_drift_variants/test.toml | 18 -- .../cluster_policy/databricks.yml.tmpl | 24 +++ .../clusters/cluster_policy/out.test.toml | 4 + .../clusters/cluster_policy/output.txt | 30 +++ .../resources/clusters/cluster_policy/script | 14 ++ .../clusters/cluster_policy/test.toml | 12 ++ .../databricks.yml.tmpl | 0 .../config_conflicts_policy}/hello_world.py | 0 .../out.test.toml | 0 .../output.txt | 0 .../script | 0 .../default_flag/databricks.yml.tmpl | 40 ++++ .../hello_world.py | 0 .../out.test.toml | 0 .../cluster_policy/default_flag/output.txt | 27 +++ .../jobs/cluster_policy/default_flag/script | 15 ++ .../databricks.yml.tmpl | 0 .../hello_world.py | 0 .../out.test.toml | 0 .../output.txt | 0 .../script | 0 .../owned_tag_drift/databricks.yml.tmpl | 35 ++++ .../hello_world.py | 0 .../out.test.toml | 0 .../cluster_policy/owned_tag_drift/output.txt | 41 ++++ .../cluster_policy/owned_tag_drift/script | 20 ++ .../cluster_policy/policy_drift/output.txt | 57 ------ .../jobs/cluster_policy/policy_drift/script | 32 --- .../databricks.yml.tmpl | 57 +++--- .../hello_world.py | 0 .../policy_id_scoping}/out.test.toml | 4 +- .../policy_id_scoping/output.txt | 38 ++++ .../cluster_policy/policy_id_scoping/script | 19 ++ .../databricks.yml.tmpl | 70 ------- .../policy_value_semantics/output.txt | 33 ---- .../policy_value_semantics/script | 18 -- .../cluster_policy/databricks.yml.tmpl | 29 +++ .../pipelines/cluster_policy/hello_world.py | 1 + .../pipelines/cluster_policy/out.test.toml | 3 + .../pipelines/cluster_policy/output.txt | 25 +++ .../resources/pipelines/cluster_policy/script | 11 ++ .../pipelines/cluster_policy/test.toml | 8 + bundle/direct/bundle_plan.go | 2 +- bundle/direct/dresources/resources.yml | 4 +- libs/testserver/cluster_policies.go | 4 +- 49 files changed, 430 insertions(+), 576 deletions(-) delete mode 100755 acceptance/bin/prune_plan.py delete mode 100644 acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl delete mode 100644 acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt delete mode 100644 acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script delete mode 100644 acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml create mode 100644 acceptance/bundle/resources/clusters/cluster_policy/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/clusters/cluster_policy/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/cluster_policy/output.txt create mode 100644 acceptance/bundle/resources/clusters/cluster_policy/script create mode 100644 acceptance/bundle/resources/clusters/cluster_policy/test.toml rename acceptance/bundle/resources/jobs/cluster_policy/{fixed_value_conflict => config_conflicts_policy}/databricks.yml.tmpl (100%) rename acceptance/bundle/resources/{cluster_policies/policy_no_drift_variants => jobs/cluster_policy/config_conflicts_policy}/hello_world.py (100%) rename acceptance/bundle/resources/jobs/cluster_policy/{fixed_value_conflict => config_conflicts_policy}/out.test.toml (100%) rename acceptance/bundle/resources/jobs/cluster_policy/{fixed_value_conflict => config_conflicts_policy}/output.txt (100%) rename acceptance/bundle/resources/jobs/cluster_policy/{fixed_value_conflict => config_conflicts_policy}/script (100%) create mode 100644 acceptance/bundle/resources/jobs/cluster_policy/default_flag/databricks.yml.tmpl rename acceptance/bundle/resources/jobs/cluster_policy/{fixed_value_conflict => default_flag}/hello_world.py (100%) rename acceptance/bundle/resources/jobs/cluster_policy/{fixed_values_applied => default_flag}/out.test.toml (100%) create mode 100644 acceptance/bundle/resources/jobs/cluster_policy/default_flag/output.txt create mode 100644 acceptance/bundle/resources/jobs/cluster_policy/default_flag/script rename acceptance/bundle/resources/jobs/cluster_policy/{fixed_values_applied => fixed_addition}/databricks.yml.tmpl (100%) rename acceptance/bundle/resources/jobs/cluster_policy/{fixed_values_applied => fixed_addition}/hello_world.py (100%) rename acceptance/bundle/resources/jobs/cluster_policy/{policy_drift => fixed_addition}/out.test.toml (100%) rename acceptance/bundle/resources/jobs/cluster_policy/{fixed_values_applied => fixed_addition}/output.txt (100%) rename acceptance/bundle/resources/jobs/cluster_policy/{fixed_values_applied => fixed_addition}/script (100%) create mode 100644 acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/databricks.yml.tmpl rename acceptance/bundle/resources/jobs/cluster_policy/{policy_drift => owned_tag_drift}/hello_world.py (100%) rename acceptance/bundle/resources/jobs/cluster_policy/{policy_value_semantics => owned_tag_drift}/out.test.toml (100%) create mode 100644 acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/output.txt create mode 100644 acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script delete mode 100644 acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt delete mode 100644 acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script rename acceptance/bundle/resources/jobs/cluster_policy/{policy_drift => policy_id_scoping}/databricks.yml.tmpl (53%) rename acceptance/bundle/resources/jobs/cluster_policy/{policy_value_semantics => policy_id_scoping}/hello_world.py (100%) rename acceptance/bundle/resources/{cluster_policies/policy_no_drift_variants => jobs/cluster_policy/policy_id_scoping}/out.test.toml (53%) create mode 100644 acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/output.txt create mode 100644 acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script delete mode 100644 acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/databricks.yml.tmpl delete mode 100644 acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/output.txt delete mode 100644 acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/script create mode 100644 acceptance/bundle/resources/pipelines/cluster_policy/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/pipelines/cluster_policy/hello_world.py create mode 100644 acceptance/bundle/resources/pipelines/cluster_policy/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/cluster_policy/output.txt create mode 100644 acceptance/bundle/resources/pipelines/cluster_policy/script create mode 100644 acceptance/bundle/resources/pipelines/cluster_policy/test.toml diff --git a/acceptance/bin/prune_plan.py b/acceptance/bin/prune_plan.py deleted file mode 100755 index 48538f141cd..00000000000 --- a/acceptance/bin/prune_plan.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -"""Prune the cloud-dependent parts of `bundle plan -o json` so the plan can be recorded as a -golden that is identical across clouds. - -Reads plan JSON on stdin, writes the pruned JSON on stdout. Per plan node it removes: - - - remote_state: the raw backend read (per-run driver/executor IPs, instance ids, - spark_context_id, timestamps, default_tags, ...), none of which is reproducible. - - changes entries whose reason is in --ignore-reasons (default: managed, backend_default) -- - values the backend chooses, e.g. {aws,azure,gcp}_attributes or node types, which differ by - cloud. - - changes entries whose field path contains any substring in --ignore-keys -- for fields a - backend sets to different values (so the same field lands under different reasons on - different clouds and cannot be matched by reason alone, e.g. enable_elastic_disk). - -What survives is cloud-independent: the action, and the changes driven by config or policy. -""" - -import argparse -import json -import sys - -parser = argparse.ArgumentParser() -parser.add_argument("--ignore-reasons", default="managed,backend_default") -parser.add_argument("--ignore-keys", default="") -args = parser.parse_args() - -ignore_reasons = {r for r in args.ignore_reasons.split(",") if r} -ignore_keys = [k for k in args.ignore_keys.split(",") if k] - -plan = json.load(sys.stdin) - -for node in plan.get("plan", {}).values(): - node.pop("remote_state", None) - changes = node.get("changes") or {} - for path, change in list(changes.items()): - if change.get("reason") in ignore_reasons or any(k in path for k in ignore_keys): - del changes[path] - -json.dump(plan, sys.stdout, indent=2, sort_keys=True) -print() diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl deleted file mode 100644 index 3ed70bb4b58..00000000000 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/databricks.yml.tmpl +++ /dev/null @@ -1,74 +0,0 @@ -bundle: - name: test-bundle-$UNIQUE_NAME - -workspace: - root_path: ~/.bundle/$UNIQUE_NAME - -resources: - cluster_policies: - my_policy: - name: test-policy-$UNIQUE_NAME - definition: - custom_tags.CostCenter: - type: fixed - value: from-policy - - # A cluster spec can appear in several places, each with its own ignore_remote_additions - # rule. This fixture puts a policy-attached cluster in each place -- a standalone cluster, a - # task's new_cluster, a for_each task's new_cluster, and a pipeline cluster -- so the rule is - # exercised at every location in one plan. (job_clusters is covered by the sibling - # jobs/cluster_policy tests.) The pipeline cluster is the deliberate no-rule case; see below. - clusters: - standalone: - cluster_name: test-cluster-$UNIQUE_NAME - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - # Use the shared instance pool: a cold cluster boot made this the slowest test in the - # cloud suite (6-8 minutes per env), and the policy tag under test is unaffected. - instance_pool_id: $TEST_INSTANCE_POOL_ID - num_workers: 1 - policy_id: ${resources.cluster_policies.my_policy.id} - - jobs: - task_cluster: - name: test-task-cluster-$UNIQUE_NAME - tasks: - - task_key: t - new_cluster: - policy_id: ${resources.cluster_policies.my_policy.id} - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - spark_python_task: - python_file: ./hello_world.py - - for_each_cluster: - name: test-for-each-cluster-$UNIQUE_NAME - tasks: - - task_key: outer - for_each_task: - inputs: "[1,2]" - task: - task_key: inner - new_cluster: - policy_id: ${resources.cluster_policies.my_policy.id} - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - spark_python_task: - python_file: ./hello_world.py - - # A pipeline cluster carries policy_id too, but the Pipelines API does not expand the - # policy into the stored spec: it reads back exactly as authored, so there is no addition - # to suppress and no ignore_remote_additions rule for pipelines. Kept here to record that. - pipelines: - pipe: - name: test-pipeline-$UNIQUE_NAME - clusters: - - label: default - policy_id: ${resources.cluster_policies.my_policy.id} - node_type_id: $NODE_TYPE_ID - num_workers: 1 - libraries: - - file: - path: ./hello_world.py diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt deleted file mode 100644 index 06871c01add..00000000000 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/output.txt +++ /dev/null @@ -1,182 +0,0 @@ - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Created cluster_policies.my_policy -Created clusters.standalone -Created jobs.for_each_cluster -Created jobs.task_cluster -Created pipelines.pipe -Files: 6 uploaded, 0 deleted -Resources: 5 created, 0 changed, 0 deleted, 0 unchanged - -=== Every cluster-spec location converges: nothing to change on a second plan - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged - -=== The plan, minus remote_state and backend-chosen changes (managed, backend_default) - ->>> [CLI] bundle plan -o json -{ - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "plan": { - "resources.cluster_policies.my_policy": { - "action": "skip" - }, - "resources.clusters.standalone": { - "action": "skip", - "changes": { - "custom_tags": { - "action": "skip", - "reason": "remote_addition", - "remote": { - "CostCenter": "from-policy" - } - }, - "enable_elastic_disk": { - "action": "skip", - "reason": "empty", - "remote": false - }, - "lifecycle": { - "action": "skip", - "reason": "remote_addition", - "remote": { - "started": true - } - } - }, - "depends_on": [ - { - "label": "${resources.cluster_policies.my_policy.id}", - "node": "resources.cluster_policies.my_policy" - } - ] - }, - "resources.jobs.for_each_cluster": { - "action": "skip", - "changes": { - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "tasks[task_key='outer'].email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "tasks[task_key='outer'].for_each_task.task.new_cluster.custom_tags": { - "action": "skip", - "reason": "remote_addition", - "remote": { - "CostCenter": "from-policy" - } - }, - "tasks[task_key='outer'].timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - }, - "depends_on": [ - { - "label": "${resources.cluster_policies.my_policy.id}", - "node": "resources.cluster_policies.my_policy" - } - ] - }, - "resources.jobs.task_cluster": { - "action": "skip", - "changes": { - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "tasks[task_key='t'].email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "tasks[task_key='t'].new_cluster.custom_tags": { - "action": "skip", - "reason": "remote_addition", - "remote": { - "CostCenter": "from-policy" - } - }, - "tasks[task_key='t'].new_cluster.data_security_mode": { - "action": "skip", - "reason": "remote_addition", - "remote": "SINGLE_USER" - }, - "tasks[task_key='t'].new_cluster.enable_elastic_disk": { - "action": "skip", - "reason": "empty", - "remote": false - }, - "tasks[task_key='t'].timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - }, - "depends_on": [ - { - "label": "${resources.cluster_policies.my_policy.id}", - "node": "resources.cluster_policies.my_policy" - } - ] - }, - "resources.pipelines.pipe": { - "action": "skip", - "changes": {}, - "depends_on": [ - { - "label": "${resources.cluster_policies.my_policy.id}", - "node": "resources.cluster_policies.my_policy" - } - ] - } - }, - "plan_version": [PLAN_VERSION], - "serial": 1 -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.cluster_policies.my_policy - delete resources.clusters.standalone - delete resources.jobs.for_each_cluster - delete resources.jobs.task_cluster - delete resources.pipelines.pipe - -This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: - delete resources.pipelines.pipe - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] - -Destroy: 5 deleted diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script deleted file mode 100644 index 258d6e21140..00000000000 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/script +++ /dev/null @@ -1,14 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -cleanup() { - trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -trace $CLI bundle deploy - -title "Every cluster-spec location converges: nothing to change on a second plan\n" -trace $CLI bundle plan - -title "The plan, minus remote_state and backend-chosen changes (managed, backend_default)\n" -trace $CLI bundle plan -o json | prune_plan.py diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml b/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml deleted file mode 100644 index eb681f0c5d6..00000000000 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/test.toml +++ /dev/null @@ -1,18 +0,0 @@ -# Local-only. This test creates a standalone cluster, which only the local testserver can do -# reliably: on a real workspace it boots at deploy and repeatedly hit capacity limits, and the -# full remote_state is a per-run cluster GET no testserver reproduces. The suppression behavior -# is verified against a real workspace by the jobs/cluster_policy tests; here we cover every -# cluster-spec location (standalone cluster, task new_cluster, for_each new_cluster, pipeline). -Cloud = false - -# Deployment-history recording (DMS) adds version-dependent plan fields and .databricks -# artifacts, and it is orthogonal to drift suppression (covered by the jobs/cluster_policy -# tests). Pin to the off value so the test runs once on the "direct" CI cell; DMS = [] would -# instead run on BOTH the direct and directdms cells (same trap as DATABRICKS_BUNDLE_ENGINE=[]). -EnvMatrix.DMS = [""] - -RecordRequests = false - -Ignore = [ - "databricks.yml", -] diff --git a/acceptance/bundle/resources/clusters/cluster_policy/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/cluster_policy/databricks.yml.tmpl new file mode 100644 index 00000000000..9cf0236a1ef --- /dev/null +++ b/acceptance/bundle/resources/clusters/cluster_policy/databricks.yml.tmpl @@ -0,0 +1,24 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + my_policy: + name: test-policy-$UNIQUE_NAME + definition: + custom_tags.CostCenter: + type: fixed + value: from-policy + + clusters: + c: + cluster_name: test-cluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + # Shared pool avoids a cold boot; the policy tag under test is unaffected. + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + policy_id: ${resources.cluster_policies.my_policy.id} diff --git a/acceptance/bundle/resources/clusters/cluster_policy/out.test.toml b/acceptance/bundle/resources/clusters/cluster_policy/out.test.toml new file mode 100644 index 00000000000..f9f4880725d --- /dev/null +++ b/acceptance/bundle/resources/clusters/cluster_policy/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/cluster_policy/output.txt b/acceptance/bundle/resources/clusters/cluster_policy/output.txt new file mode 100644 index 00000000000..28cc98b7413 --- /dev/null +++ b/acceptance/bundle/resources/clusters/cluster_policy/output.txt @@ -0,0 +1,30 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.my_policy +Created clusters.c +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== A cluster policy tag on a standalone cluster is not drift; second plan is clean + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== ...and is classified remote_addition + +>>> [CLI] bundle plan -o json +json.plan.resources.cluster_policies.my_policy.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}"; +json.plan.resources.clusters.c.remote_state.custom_tags.CostCenter = "from-policy"; +json.plan.resources.clusters.c.changes.custom_tags.action = "skip"; +json.plan.resources.clusters.c.changes.custom_tags.reason = "remote_addition"; +json.plan.resources.clusters.c.changes.custom_tags.remote.CostCenter = "from-policy"; + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.my_policy + delete resources.clusters.c + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/clusters/cluster_policy/script b/acceptance/bundle/resources/clusters/cluster_policy/script new file mode 100644 index 00000000000..de502586230 --- /dev/null +++ b/acceptance/bundle/resources/clusters/cluster_policy/script @@ -0,0 +1,14 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "A cluster policy tag on a standalone cluster is not drift; second plan is clean\n" +trace $CLI bundle plan + +title "...and is classified remote_addition\n" +trace $CLI bundle plan -o json | gron.py | grep custom_tags diff --git a/acceptance/bundle/resources/clusters/cluster_policy/test.toml b/acceptance/bundle/resources/clusters/cluster_policy/test.toml new file mode 100644 index 00000000000..a3789595417 --- /dev/null +++ b/acceptance/bundle/resources/clusters/cluster_policy/test.toml @@ -0,0 +1,12 @@ +# The standalone clusters resource boots a real cluster at deploy, which is slow and has hit +# capacity limits on cloud; CloudSlow keeps it out of the per-PR run (it still runs locally and +# in the full cloud sweep). Output is projected to the policy tag so it is cloud-portable. +Cloud = true +CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/databricks.yml.tmpl rename to acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/databricks.yml.tmpl diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/hello_world.py b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/hello_world.py similarity index 100% rename from acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/hello_world.py rename to acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/out.test.toml b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/out.test.toml similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/out.test.toml rename to acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/out.test.toml diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/output.txt similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/output.txt rename to acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/output.txt diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/script b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/script similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/script rename to acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/script diff --git a/acceptance/bundle/resources/jobs/cluster_policy/default_flag/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/databricks.yml.tmpl new file mode 100644 index 00000000000..6b61a845254 --- /dev/null +++ b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/databricks.yml.tmpl @@ -0,0 +1,40 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + my_policy: + name: test-policy-$UNIQUE_NAME + definition: + # A defaultValue element: the backend fills it into an omitted attribute only when + # the request sets apply_policy_default_values (unlike a fixed element). + custom_tags.DefaultTag: + type: unlimited + defaultValue: from-default + isOptional: true + + jobs: + j: + name: test-job-$UNIQUE_NAME + job_clusters: + - job_cluster_key: flag_off + new_cluster: + policy_id: ${resources.cluster_policies.my_policy.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + - job_cluster_key: flag_on + new_cluster: + policy_id: ${resources.cluster_policies.my_policy.id} + apply_policy_default_values: true + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + tasks: + - task_key: t + job_cluster_key: flag_off + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/hello_world.py b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/hello_world.py similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/fixed_value_conflict/hello_world.py rename to acceptance/bundle/resources/jobs/cluster_policy/default_flag/hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/out.test.toml b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/out.test.toml similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/out.test.toml rename to acceptance/bundle/resources/jobs/cluster_policy/default_flag/out.test.toml diff --git a/acceptance/bundle/resources/jobs/cluster_policy/default_flag/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/output.txt new file mode 100644 index 00000000000..0ebba6f3d33 --- /dev/null +++ b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/output.txt @@ -0,0 +1,27 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.my_policy +Created jobs.j +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== defaultValue is supplied only with apply_policy_default_values (flag_on), not flag_off +json.settings.job_clusters[0].job_cluster_key = "flag_off"; +json.settings.job_clusters[1].job_cluster_key = "flag_on"; +json.settings.job_clusters[1].new_cluster.custom_tags.DefaultTag = "from-default"; +json.settings.tasks[0].job_cluster_key = "flag_off"; + +=== Neither is drift on a second plan + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.my_policy + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/jobs/cluster_policy/default_flag/script b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/script new file mode 100644 index 00000000000..7c365313865 --- /dev/null +++ b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/script @@ -0,0 +1,15 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +job_id=$(read_id.py j) + +title "defaultValue is supplied only with apply_policy_default_values (flag_on), not flag_off\n" +$CLI jobs get "$job_id" | gron.py | grep -E 'job_cluster_key|custom_tags' + +title "Neither is drift on a second plan\n" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/databricks.yml.tmpl rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/databricks.yml.tmpl diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/hello_world.py b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/hello_world.py similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/hello_world.py rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/out.test.toml b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/out.test.toml similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/policy_drift/out.test.toml rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/out.test.toml diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/output.txt similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/output.txt rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/output.txt diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/script similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/fixed_values_applied/script rename to acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/script diff --git a/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/databricks.yml.tmpl new file mode 100644 index 00000000000..3e678c334fc --- /dev/null +++ b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/databricks.yml.tmpl @@ -0,0 +1,35 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + my_policy: + name: test-policy-$UNIQUE_NAME + definition: + custom_tags.CostCenter: + type: fixed + value: from-policy + + jobs: + # The config owns custom_tags.Mine. It is deliberately NOT a policy-governed tag: a fixed + # policy enforces on edit too, so an out-of-band change to the policy's own tag would be + # rejected (see config_conflicts_policy) and there would be no drifted state to observe. + j: + name: test-job-$UNIQUE_NAME + job_clusters: + - job_cluster_key: small + new_cluster: + policy_id: ${resources.cluster_policies.my_policy.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + Mine: mine + tasks: + - task_key: t + job_cluster_key: small + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/hello_world.py b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/hello_world.py similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/policy_drift/hello_world.py rename to acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/out.test.toml b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/out.test.toml similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/out.test.toml rename to acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/out.test.toml diff --git a/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/output.txt new file mode 100644 index 00000000000..8228bff8f73 --- /dev/null +++ b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/output.txt @@ -0,0 +1,41 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.my_policy +Created jobs.j +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== An out-of-band change to a config-owned tag is drift, even with a policy attached + +>>> [CLI] bundle plan +update jobs.j + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +=== Removing a config-owned tag is a change the user asked for, not a backend addition + +>>> update_file.py databricks.yml Mine: mine Other: other + +>>> [CLI] bundle plan -o json +json.plan.resources.cluster_policies.my_policy.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}"; +json.plan.resources.jobs.j.new_state.value.job_clusters[0].new_cluster.custom_tags.Other = "other"; +json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "from-policy"; +json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.Mine = "changed-out-of-band"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].action = "skip"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].reason = "remote_addition"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].remote = "from-policy"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].action = "update"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].old = "mine"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].remote = "changed-out-of-band"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Other'].action = "update"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Other'].new = "other"; + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.my_policy + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script new file mode 100644 index 00000000000..6a45d69f047 --- /dev/null +++ b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +job_id=$(read_id.py j) + +title "An out-of-band change to a config-owned tag is drift, even with a policy attached\n" +edit_resource.py jobs "$job_id" <<'EOF' +for jc in r["job_clusters"]: + jc["new_cluster"]["custom_tags"]["Mine"] = "changed-out-of-band" +EOF +trace $CLI bundle plan | contains.py "1 to change" + +title "Removing a config-owned tag is a change the user asked for, not a backend addition\n" +trace update_file.py databricks.yml "Mine: mine" "Other: other" +trace $CLI bundle plan -o json | gron.py | grep custom_tags diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt deleted file mode 100644 index 08b0dc227cd..00000000000 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/output.txt +++ /dev/null @@ -1,57 +0,0 @@ - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Created cluster_policies.my_policy -Created jobs.no_policy -Created jobs.owned_tag -Created jobs.with_policy -Files: 5 uploaded, 0 deleted -Resources: 4 created, 0 changed, 0 deleted, 0 unchanged - -=== A tag the cluster policy added is not drift - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged - -=== Without a policy_id the same remote-only tag IS drift - ->>> [CLI] bundle plan -update jobs.no_policy - -Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged - -=== A policy does not mask a change to a tag the config owns - ->>> [CLI] bundle plan -update jobs.no_policy -update jobs.owned_tag - -Plan: 0 to add, 2 to change, 0 to delete, 2 unchanged - -=== Removing a tag from config is a change the user asked for, not an addition - ->>> update_file.py databricks.yml Mine: mine Other: other - ->>> [CLI] bundle plan -o json -json.plan.resources.jobs.owned_tag.new_state.value.job_clusters[0].new_cluster.custom_tags.Other = "other"; -json.plan.resources.jobs.owned_tag.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "from-policy"; -json.plan.resources.jobs.owned_tag.remote_state.job_clusters[0].new_cluster.custom_tags.Mine = "changed-out-of-band"; -json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].action = "skip"; -json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].reason = "remote_addition"; -json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].remote = "from-policy"; -json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].action = "update"; -json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].old = "mine"; -json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].remote = "changed-out-of-band"; -json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Other'].action = "update"; -json.plan.resources.jobs.owned_tag.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Other'].new = "other"; - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.cluster_policies.my_policy - delete resources.jobs.no_policy - delete resources.jobs.owned_tag - delete resources.jobs.with_policy - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] - -Destroy: 4 deleted diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script b/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script deleted file mode 100644 index a15dc422476..00000000000 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/script +++ /dev/null @@ -1,32 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -cleanup() { - trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -trace $CLI bundle deploy -read_id.py my_policy > /dev/null -no_policy_id="$(read_id.py no_policy)" -owned_tag_id="$(read_id.py owned_tag)" - -title "A tag the cluster policy added is not drift\n" -trace $CLI bundle plan | contains.py "0 to change" - -title "Without a policy_id the same remote-only tag IS drift\n" -edit_resource.py jobs "$no_policy_id" <<'EOF' -for jc in r["job_clusters"]: - jc["new_cluster"]["custom_tags"] = {"CostCenter": "added-out-of-band"} -EOF -trace $CLI bundle plan | contains.py "1 to change" - -title "A policy does not mask a change to a tag the config owns\n" -edit_resource.py jobs "$owned_tag_id" <<'EOF' -for jc in r["job_clusters"]: - jc["new_cluster"]["custom_tags"]["Mine"] = "changed-out-of-band" -EOF -trace $CLI bundle plan | contains.py "2 to change" - -title "Removing a tag from config is a change the user asked for, not an addition\n" -trace update_file.py databricks.yml "Mine: mine" "Other: other" -trace $CLI bundle plan -o json | gron.py | grep -E "owned_tag.*custom_tags" diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/databricks.yml.tmpl similarity index 53% rename from acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl rename to acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/databricks.yml.tmpl index 74867535ae2..cec8acae6d8 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_drift/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/databricks.yml.tmpl @@ -13,53 +13,50 @@ resources: type: fixed value: from-policy + # policy_id gates suppression per cluster spec, not per resource. Gated clusters appear at + # every jobs location (job_clusters, tasks[*].new_cluster, for_each); one cluster omits + # policy_id, and an out-of-band tag on it must still surface as drift. jobs: - # policy attached: the tag the policy adds must not be drift. - with_policy: - name: test-with-policy-$UNIQUE_NAME + j: + name: test-job-$UNIQUE_NAME job_clusters: - - job_cluster_key: small + - job_cluster_key: gated new_cluster: policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 - tasks: - - task_key: t - job_cluster_key: small - spark_python_task: - python_file: ./hello_world.py - - # policy attached, and the config owns a tag of its own. An out-of-band change to that - # tag must still be reported: a policy suppresses additions, never disagreements. - owned_tag: - name: test-owned-tag-$UNIQUE_NAME - job_clusters: - - job_cluster_key: small + - job_cluster_key: ungated new_cluster: - policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 - custom_tags: - Mine: mine tasks: - - task_key: t - job_cluster_key: small + - task_key: on_gated + job_cluster_key: gated spark_python_task: python_file: ./hello_world.py - - # no policy attached: a remote-only tag is still drift. edit_resource.py injects it below. - no_policy: - name: test-no-policy-$UNIQUE_NAME - job_clusters: - - job_cluster_key: small + - task_key: on_ungated + job_cluster_key: ungated + spark_python_task: + python_file: ./hello_world.py + - task_key: task_cluster new_cluster: + policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 - tasks: - - task_key: t - job_cluster_key: small spark_python_task: python_file: ./hello_world.py + - task_key: foreach + for_each_task: + inputs: "[1]" + task: + task_key: inner + new_cluster: + policy_id: ${resources.cluster_policies.my_policy.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/hello_world.py b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/hello_world.py similarity index 100% rename from acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/hello_world.py rename to acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/hello_world.py diff --git a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/out.test.toml similarity index 53% rename from acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml rename to acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/out.test.toml index 27ec2a7fcd6..ae5c7bd798f 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_no_drift_variants/out.test.toml +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/out.test.toml @@ -1,3 +1,3 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/output.txt new file mode 100644 index 00000000000..2cceb5c06a9 --- /dev/null +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/output.txt @@ -0,0 +1,38 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.my_policy +Created jobs.j +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Inject a remote-only tag on the ungated cluster (no policy_id) to show it is not gated + +=== The policy tag is skipped on every gated location; the ungated cluster's tag is drift + +>>> [CLI] bundle plan -o json +json.plan.resources.cluster_policies.my_policy.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}"; +json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "from-policy"; +json.plan.resources.jobs.j.remote_state.job_clusters[1].new_cluster.custom_tags.Injected = "out-of-band"; +json.plan.resources.jobs.j.remote_state.tasks[0].for_each_task.task.new_cluster.custom_tags.CostCenter = "from-policy"; +json.plan.resources.jobs.j.remote_state.tasks[3].new_cluster.custom_tags.CostCenter = "from-policy"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='gated'].new_cluster.custom_tags.action = "skip"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='gated'].new_cluster.custom_tags.reason = "remote_addition"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='gated'].new_cluster.custom_tags.remote.CostCenter = "from-policy"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='ungated'].new_cluster.custom_tags.action = "update"; +json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='ungated'].new_cluster.custom_tags.remote.Injected = "out-of-band"; +json.plan.resources.jobs.j.changes.tasks[task_key='foreach'].for_each_task.task.new_cluster.custom_tags.action = "skip"; +json.plan.resources.jobs.j.changes.tasks[task_key='foreach'].for_each_task.task.new_cluster.custom_tags.reason = "remote_addition"; +json.plan.resources.jobs.j.changes.tasks[task_key='foreach'].for_each_task.task.new_cluster.custom_tags.remote.CostCenter = "from-policy"; +json.plan.resources.jobs.j.changes.tasks[task_key='task_cluster'].new_cluster.custom_tags.action = "skip"; +json.plan.resources.jobs.j.changes.tasks[task_key='task_cluster'].new_cluster.custom_tags.reason = "remote_addition"; +json.plan.resources.jobs.j.changes.tasks[task_key='task_cluster'].new_cluster.custom_tags.remote.CostCenter = "from-policy"; + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.my_policy + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script new file mode 100644 index 00000000000..7e493714806 --- /dev/null +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script @@ -0,0 +1,19 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +job_id=$(read_id.py j) + +title "Inject a remote-only tag on the ungated cluster (no policy_id) to show it is not gated\n" +edit_resource.py jobs "$job_id" <<'EOF' +for jc in r["job_clusters"]: + if jc["job_cluster_key"] == "ungated": + jc["new_cluster"]["custom_tags"] = {"Injected": "out-of-band"} +EOF + +title "The policy tag is skipped on every gated location; the ungated cluster's tag is drift\n" +trace $CLI bundle plan -o json | gron.py | grep custom_tags diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/databricks.yml.tmpl deleted file mode 100644 index e4bc9b322c0..00000000000 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/databricks.yml.tmpl +++ /dev/null @@ -1,70 +0,0 @@ -bundle: - name: test-bundle-$UNIQUE_NAME - -workspace: - root_path: ~/.bundle/$UNIQUE_NAME - -resources: - cluster_policies: - # A "fixed" element. Docs say it "limits the attribute to the specified value", - # which is validation language; whether it also SUPPLIES the value to a request - # that omits the attribute is what this test pins down. - fixed: - name: test-policy-fixed-$UNIQUE_NAME - definition: - custom_tags.FixedTag: - type: fixed - value: from-fixed - - # A "defaultValue" element. Documented to apply only when the request sets - # apply_policy_default_values=true. - defaulted: - name: test-policy-default-$UNIQUE_NAME - definition: - custom_tags.DefaultTag: - type: unlimited - defaultValue: from-default - isOptional: true - - jobs: - j: - name: test-job-$UNIQUE_NAME - job_clusters: - # A: fixed element, apply_policy_default_values NOT set - - job_cluster_key: a_fixed_noflag - new_cluster: - policy_id: ${resources.cluster_policies.fixed.id} - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - - # B: defaultValue element, apply_policy_default_values NOT set - - job_cluster_key: b_default_noflag - new_cluster: - policy_id: ${resources.cluster_policies.defaulted.id} - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - - # C: defaultValue element, apply_policy_default_values = true - - job_cluster_key: c_default_flag - new_cluster: - policy_id: ${resources.cluster_policies.defaulted.id} - apply_policy_default_values: true - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - - # D: fixed element, apply_policy_default_values = true - - job_cluster_key: d_fixed_flag - new_cluster: - policy_id: ${resources.cluster_policies.fixed.id} - apply_policy_default_values: true - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - tasks: - - task_key: t - job_cluster_key: a_fixed_noflag - spark_python_task: - python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/output.txt deleted file mode 100644 index 0d0afa5a526..00000000000 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/output.txt +++ /dev/null @@ -1,33 +0,0 @@ - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Created cluster_policies.defaulted -Created cluster_policies.fixed -Created jobs.j -Files: 5 uploaded, 0 deleted -Resources: 3 created, 0 changed, 0 deleted, 0 unchanged - -=== Which policy elements did the backend SUPPLY into each job_cluster? -json.settings.job_clusters[0].job_cluster_key = "a_fixed_noflag"; -json.settings.job_clusters[0].new_cluster.custom_tags.FixedTag = "from-fixed"; -json.settings.job_clusters[1].job_cluster_key = "b_default_noflag"; -json.settings.job_clusters[2].job_cluster_key = "c_default_flag"; -json.settings.job_clusters[2].new_cluster.custom_tags.DefaultTag = "from-default"; -json.settings.job_clusters[3].job_cluster_key = "d_fixed_flag"; -json.settings.job_clusters[3].new_cluster.custom_tags.FixedTag = "from-fixed"; -json.settings.tasks[0].job_cluster_key = "a_fixed_noflag"; - -=== Are the supplied values reported as drift? - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.cluster_policies.defaulted - delete resources.cluster_policies.fixed - delete resources.jobs.j - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] - -Destroy: 3 deleted diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/script b/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/script deleted file mode 100644 index c78ff847b4c..00000000000 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics/script +++ /dev/null @@ -1,18 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -cleanup() { - trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -trace $CLI bundle deploy - -read_id.py fixed > /dev/null -read_id.py defaulted > /dev/null -job_id=$(read_id.py j) - -title "Which policy elements did the backend SUPPLY into each job_cluster?\n" -$CLI jobs get "$job_id" | gron.py | grep -E 'job_cluster_key|apply_policy_default_values|new_cluster.custom_tags' - -title "Are the supplied values reported as drift?\n" -trace $CLI bundle plan diff --git a/acceptance/bundle/resources/pipelines/cluster_policy/databricks.yml.tmpl b/acceptance/bundle/resources/pipelines/cluster_policy/databricks.yml.tmpl new file mode 100644 index 00000000000..836cd226384 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/cluster_policy/databricks.yml.tmpl @@ -0,0 +1,29 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + my_policy: + name: test-policy-$UNIQUE_NAME + definition: + custom_tags.CostCenter: + type: fixed + value: from-policy + + # A pipeline cluster carries policy_id too, but the Pipelines API does not expand the policy + # into the stored spec: it reads back exactly as authored, so there is no addition to suppress + # and no ignore_remote_additions rule for pipelines. This records that it simply converges. + pipelines: + p: + name: test-pipeline-$UNIQUE_NAME + clusters: + - label: default + policy_id: ${resources.cluster_policies.my_policy.id} + node_type_id: $NODE_TYPE_ID + num_workers: 1 + libraries: + - file: + path: ./hello_world.py diff --git a/acceptance/bundle/resources/pipelines/cluster_policy/hello_world.py b/acceptance/bundle/resources/pipelines/cluster_policy/hello_world.py new file mode 100644 index 00000000000..f301245e242 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/cluster_policy/hello_world.py @@ -0,0 +1 @@ +print("Hello World!") diff --git a/acceptance/bundle/resources/pipelines/cluster_policy/out.test.toml b/acceptance/bundle/resources/pipelines/cluster_policy/out.test.toml new file mode 100644 index 00000000000..ae5c7bd798f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/cluster_policy/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/cluster_policy/output.txt b/acceptance/bundle/resources/pipelines/cluster_policy/output.txt new file mode 100644 index 00000000000..a6355bbeba8 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/cluster_policy/output.txt @@ -0,0 +1,25 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.my_policy +Created pipelines.p +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== A pipeline cluster with policy_id converges: the Pipelines API does not expand the policy + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.my_policy + delete resources.pipelines.p + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.p + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/pipelines/cluster_policy/script b/acceptance/bundle/resources/pipelines/cluster_policy/script new file mode 100644 index 00000000000..dfe1ad7caa2 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/cluster_policy/script @@ -0,0 +1,11 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "A pipeline cluster with policy_id converges: the Pipelines API does not expand the policy\n" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/pipelines/cluster_policy/test.toml b/acceptance/bundle/resources/pipelines/cluster_policy/test.toml new file mode 100644 index 00000000000..6131aeaf3a2 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/cluster_policy/test.toml @@ -0,0 +1,8 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 3a35372692d..abb45c2f674 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -634,7 +634,7 @@ func shouldSkipNormalized(cfg *dresources.ResourceLifecycleConfig, path *structp // For cluster specs that gate is policy_id: an attached cluster policy supplies values // server-side — "fixed" elements always, "defaultValue" elements when the request sets // apply_policy_default_values — so the remote spec is legitimately a superset of what the -// bundle declares. See acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics +// bundle declares. See acceptance/bundle/resources/jobs/cluster_policy (fixed_addition, default_flag) // for the measured backend behavior. // // Suppressed values are never echoed back on write: an update sends the config spec as-is diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 96b2be05b7f..5c2203fbc4d 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -90,7 +90,7 @@ resources: # "defaultValue" elements when the request sets apply_policy_default_values. The bundle # is not the source for those, so the remote spec is legitimately a superset of what the # config declares and an added field is not drift. Measured backend behavior is pinned by - # acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics. + # acceptance/bundle/resources/jobs/cluster_policy (fixed_addition, default_flag). # https://github.com/databricks/cli/issues/5179 # https://github.com/databricks/cli/issues/6512 - field: tasks[*].new_cluster @@ -612,7 +612,7 @@ resources: # "defaultValue" elements when the request sets apply_policy_default_values. The bundle # is not the source for those, so the remote spec is legitimately a superset of what the # config declares and an added field is not drift. Measured backend behavior is pinned by - # acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics. + # acceptance/bundle/resources/jobs/cluster_policy (fixed_addition, default_flag). # https://github.com/databricks/cli/issues/5179 # https://github.com/databricks/cli/issues/6512 ignore_remote_additions: diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go index b37726cccc1..d18aee74fc3 100644 --- a/libs/testserver/cluster_policies.go +++ b/libs/testserver/cluster_policies.go @@ -142,7 +142,7 @@ type policyElement struct { // attribute, or nil if it materializes none. Mirrors the backend: "fixed" applies whether // or not the request sets apply_policy_default_values, "defaultValue" only when it does. // Pinned against a real workspace by -// acceptance/bundle/resources/jobs/cluster_policy/policy_value_semantics. +// acceptance/bundle/resources/jobs/cluster_policy (fixed_addition, default_flag). func (e policyElement) effectiveValue(applyDefaults bool) any { if e.Type == "fixed" { return e.Value @@ -284,7 +284,7 @@ func applyPolicyDefaultValues(body []byte) bool { // // The Pipelines API does not expand cluster policies into the stored spec: a pipeline cluster // with a policy_id reads back exactly as authored, verified against a real workspace by -// acceptance/bundle/resources/cluster_policies/policy_no_drift_variants. So there is +// acceptance/bundle/resources/pipelines/cluster_policy. So there is // deliberately no pipeline equivalent. func (s *FakeWorkspace) applyJobClusterPolicies(settings *jobs.JobSettings) string { for i := range settings.JobClusters { From 8aeb76c49d05b03df8e49db305cec10cf200fbb3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 11 Sep 2026 11:56:19 +0200 Subject: [PATCH 21/25] acc: address review of the cluster_policy tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From an adversarial review (cursor) plus manual review: - Descriptive resource/task keys everywhere (my_job, my_cluster, my_pipeline, gated/ungated, main) instead of c/j/p/t/small. - config_conflicts_policy asserts the rejection with musterr, not errcode, so an unexpected successful deploy fails the test. - Plan-classification output uses a focused `jq` over changes[…] instead of `gron.py | grep`, which also dumped new_state/remote_state. - fixed_addition now supplies the fixed scalar + map key at all three jobs cluster-spec locations (job_clusters, tasks[*].new_cluster, for_each), so the rule is shown firing at every jobs path; policy_id_scoping keeps the gate case. - clusters/cluster_policy stays tag-only, with a comment: the clusters resource force-sends spark_version, so an omitted scalar arrives as "" and a fixed policy rejects it; the scalar case is covered by fixed_addition (omitempty job-cluster fields). - Standardize hello_world.py to the jobs convention. Co-authored-by: Isaac --- .../cluster_policy/databricks.yml.tmpl | 5 +- .../clusters/cluster_policy/output.txt | 22 ++++--- .../resources/clusters/cluster_policy/script | 7 ++- .../databricks.yml.tmpl | 12 ++-- .../config_conflicts_policy/output.txt | 8 +-- .../config_conflicts_policy/script | 4 +- .../default_flag/databricks.yml.tmpl | 4 +- .../cluster_policy/default_flag/output.txt | 20 +++++-- .../jobs/cluster_policy/default_flag/script | 5 +- .../fixed_addition/databricks.yml.tmpl | 32 +++++++--- .../cluster_policy/fixed_addition/output.txt | 60 +++++++++++++------ .../jobs/cluster_policy/fixed_addition/script | 16 ++--- .../owned_tag_drift/databricks.yml.tmpl | 8 +-- .../cluster_policy/owned_tag_drift/output.txt | 34 ++++++----- .../cluster_policy/owned_tag_drift/script | 5 +- .../policy_id_scoping/databricks.yml.tmpl | 27 +-------- .../policy_id_scoping/output.txt | 37 ++++++------ .../cluster_policy/policy_id_scoping/script | 7 ++- .../cluster_policy/databricks.yml.tmpl | 2 +- .../pipelines/cluster_policy/hello_world.py | 2 +- .../pipelines/cluster_policy/output.txt | 6 +- 21 files changed, 178 insertions(+), 145 deletions(-) diff --git a/acceptance/bundle/resources/clusters/cluster_policy/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/cluster_policy/databricks.yml.tmpl index 9cf0236a1ef..a049ea600b4 100644 --- a/acceptance/bundle/resources/clusters/cluster_policy/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/cluster_policy/databricks.yml.tmpl @@ -9,12 +9,15 @@ resources: my_policy: name: test-policy-$UNIQUE_NAME definition: + # Only a map key: the clusters resource force-sends spark_version, so a scalar the + # config omits arrives as "" and a fixed policy rejects it. The scalar-supply case is + # covered by fixed_addition, whose job-cluster new_cluster fields are omitempty. custom_tags.CostCenter: type: fixed value: from-policy clusters: - c: + my_cluster: cluster_name: test-cluster-$UNIQUE_NAME spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID diff --git a/acceptance/bundle/resources/clusters/cluster_policy/output.txt b/acceptance/bundle/resources/clusters/cluster_policy/output.txt index 28cc98b7413..89a80cd005a 100644 --- a/acceptance/bundle/resources/clusters/cluster_policy/output.txt +++ b/acceptance/bundle/resources/clusters/cluster_policy/output.txt @@ -2,28 +2,32 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.my_policy -Created clusters.c +Created clusters.my_cluster Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -=== A cluster policy tag on a standalone cluster is not drift; second plan is clean +=== A cluster policy tag on a standalone cluster is not drift >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged -=== ...and is classified remote_addition +=== ...classified remote_addition >>> [CLI] bundle plan -o json -json.plan.resources.cluster_policies.my_policy.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}"; -json.plan.resources.clusters.c.remote_state.custom_tags.CostCenter = "from-policy"; -json.plan.resources.clusters.c.changes.custom_tags.action = "skip"; -json.plan.resources.clusters.c.changes.custom_tags.reason = "remote_addition"; -json.plan.resources.clusters.c.changes.custom_tags.remote.CostCenter = "from-policy"; +{ + "custom_tags": { + "action": "skip", + "reason": "remote_addition", + "remote": { + "CostCenter": "from-policy" + } + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.cluster_policies.my_policy - delete resources.clusters.c + delete resources.clusters.my_cluster All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/clusters/cluster_policy/script b/acceptance/bundle/resources/clusters/cluster_policy/script index de502586230..d32f03284ae 100644 --- a/acceptance/bundle/resources/clusters/cluster_policy/script +++ b/acceptance/bundle/resources/clusters/cluster_policy/script @@ -7,8 +7,9 @@ trap cleanup EXIT trace $CLI bundle deploy -title "A cluster policy tag on a standalone cluster is not drift; second plan is clean\n" +title "A cluster policy tag on a standalone cluster is not drift\n" trace $CLI bundle plan -title "...and is classified remote_addition\n" -trace $CLI bundle plan -o json | gron.py | grep custom_tags +title "...classified remote_addition\n" +trace $CLI bundle plan -o json | jq -S '.plan["resources.clusters.my_cluster"].changes + | with_entries(select(.key | test("custom_tags")))' diff --git a/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/databricks.yml.tmpl index 65118118080..fc16656ff3e 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/databricks.yml.tmpl @@ -11,14 +11,14 @@ resources: definition: custom_tags.CostCenter: type: fixed - value: from-fixed + value: from-policy jobs: - # The config sets the fixed attribute to a value the policy forbids. - j: + my_job: name: test-job-$UNIQUE_NAME job_clusters: - - job_cluster_key: small + # The config sets the fixed attribute to a value the policy forbids. + - job_cluster_key: main new_cluster: policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION @@ -27,7 +27,7 @@ resources: custom_tags: CostCenter: not-what-the-policy-says tasks: - - task_key: t - job_cluster_key: small + - task_key: main + job_cluster_key: main spark_python_task: python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/output.txt index 57f311c76bd..b28ec290a03 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/output.txt @@ -1,19 +1,17 @@ -=== Deploy a cluster whose config contradicts a fixed policy value +=== A cluster tag contradicting a fixed policy value is rejected at create >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Error: cannot create resources.jobs.j: Cluster validation error: Validation failed for custom_tags, CostCenter must be from-fixed (is "not-what-the-policy-says") (400 INVALID_PARAMETER_VALUE) +Error: cannot create resources.jobs.my_job: Cluster validation error: Validation failed for custom_tags, CostCenter must be from-policy (is "not-what-the-policy-says") (400 INVALID_PARAMETER_VALUE) Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Cluster validation error: Validation failed for custom_tags, CostCenter must be from-fixed (is "not-what-the-policy-says") +API message: Cluster validation error: Validation failed for custom_tags, CostCenter must be from-policy (is "not-what-the-policy-says") Files: 5 uploaded, 0 deleted -Exit code: 1 - >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.cluster_policies.my_policy diff --git a/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/script b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/script index bf454016241..b8cb4a39cbf 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/script @@ -5,5 +5,5 @@ cleanup() { } trap cleanup EXIT -title "Deploy a cluster whose config contradicts a fixed policy value\n" -errcode trace $CLI bundle deploy +title "A cluster tag contradicting a fixed policy value is rejected at create\n" +musterr trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/jobs/cluster_policy/default_flag/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/databricks.yml.tmpl index 6b61a845254..c358664437f 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/default_flag/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/databricks.yml.tmpl @@ -17,7 +17,7 @@ resources: isOptional: true jobs: - j: + my_job: name: test-job-$UNIQUE_NAME job_clusters: - job_cluster_key: flag_off @@ -34,7 +34,7 @@ resources: node_type_id: $NODE_TYPE_ID num_workers: 1 tasks: - - task_key: t + - task_key: main job_cluster_key: flag_off spark_python_task: python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/default_flag/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/output.txt index 0ebba6f3d33..40d65355950 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/default_flag/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/output.txt @@ -2,15 +2,23 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.my_policy -Created jobs.j +Created jobs.my_job Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === defaultValue is supplied only with apply_policy_default_values (flag_on), not flag_off -json.settings.job_clusters[0].job_cluster_key = "flag_off"; -json.settings.job_clusters[1].job_cluster_key = "flag_on"; -json.settings.job_clusters[1].new_cluster.custom_tags.DefaultTag = "from-default"; -json.settings.tasks[0].job_cluster_key = "flag_off"; +[ + { + "custom_tags": null, + "job_cluster_key": "flag_off" + }, + { + "custom_tags": { + "DefaultTag": "from-default" + }, + "job_cluster_key": "flag_on" + } +] === Neither is drift on a second plan @@ -20,7 +28,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.cluster_policies.my_policy - delete resources.jobs.j + delete resources.jobs.my_job All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/jobs/cluster_policy/default_flag/script b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/script index 7c365313865..f359b41276d 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/default_flag/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/script @@ -6,10 +6,11 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -job_id=$(read_id.py j) +job_id=$(read_id.py my_job) title "defaultValue is supplied only with apply_policy_default_values (flag_on), not flag_off\n" -$CLI jobs get "$job_id" | gron.py | grep -E 'job_cluster_key|custom_tags' +$CLI jobs get "$job_id" | jq -S '[.settings.job_clusters[] + | {job_cluster_key, custom_tags: .new_cluster.custom_tags}]' title "Neither is drift on a second plan\n" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/databricks.yml.tmpl index 8408cecd217..63ae19cddc1 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/databricks.yml.tmpl @@ -9,26 +9,44 @@ resources: my_policy: name: test-policy-$UNIQUE_NAME definition: + # A fixed scalar and a fixed map key; every cluster below omits both, so the policy is + # the sole source and each supplied value must be classified remote_addition. spark_version: type: fixed value: $DEFAULT_SPARK_VERSION custom_tags.CostCenter: type: fixed - value: policy-supplied + value: from-policy jobs: - # new_cluster deliberately omits spark_version and custom_tags: the policy is - # expected to supply both. apply_policy_default_values is NOT set. - j: + my_job: name: test-job-$UNIQUE_NAME job_clusters: - - job_cluster_key: small + - job_cluster_key: job_cluster new_cluster: policy_id: ${resources.cluster_policies.my_policy.id} node_type_id: $NODE_TYPE_ID num_workers: 1 tasks: - - task_key: t - job_cluster_key: small + - task_key: on_job_cluster + job_cluster_key: job_cluster spark_python_task: python_file: ./hello_world.py + - task_key: task_cluster + new_cluster: + policy_id: ${resources.cluster_policies.my_policy.id} + node_type_id: $NODE_TYPE_ID + num_workers: 1 + spark_python_task: + python_file: ./hello_world.py + - task_key: for_each + for_each_task: + inputs: "[1]" + task: + task_key: for_each_cluster + new_cluster: + policy_id: ${resources.cluster_policies.my_policy.id} + node_type_id: $NODE_TYPE_ID + num_workers: 1 + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/output.txt index fd1786a5d42..74b1c2333f2 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/output.txt @@ -2,37 +2,61 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.my_policy -Created jobs.j +Created jobs.my_job Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -=== Did the fixed policy SUPPLY spark_version and custom_tags, or only validate them? -The bundle declares neither, and does not set apply_policy_default_values. -json.settings.job_clusters[0].new_cluster.custom_tags.CostCenter = "policy-supplied"; -json.settings.job_clusters[0].new_cluster.spark_version = "13.3.x-snapshot-scala2.12"; - -=== Is the policy-supplied value reported as drift? +=== Policy-supplied spark_version and custom_tags are not drift at any cluster-spec location >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged -=== spark_version is not covered by any backend_defaults rule, so show its verdict +=== Each is classified remote_addition >>> [CLI] bundle plan -o json -json.plan.resources.cluster_policies.my_policy.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"policy-supplied\"},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-snapshot-scala2.12\"}}"; -json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "policy-supplied"; -json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.spark_version = "13.3.x-snapshot-scala2.12"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.action = "skip"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.reason = "remote_addition"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags.remote.CostCenter = "policy-supplied"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.action = "skip"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.reason = "remote_addition"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.spark_version.remote = "13.3.x-snapshot-scala2.12"; +{ + "job_clusters[job_cluster_key='job_cluster'].new_cluster.custom_tags": { + "action": "skip", + "reason": "remote_addition", + "remote": { + "CostCenter": "from-policy" + } + }, + "job_clusters[job_cluster_key='job_cluster'].new_cluster.spark_version": { + "action": "skip", + "reason": "remote_addition", + "remote": "13.3.x-snapshot-scala2.12" + }, + "tasks[task_key='for_each'].for_each_task.task.new_cluster.custom_tags": { + "action": "skip", + "reason": "remote_addition", + "remote": { + "CostCenter": "from-policy" + } + }, + "tasks[task_key='for_each'].for_each_task.task.new_cluster.spark_version": { + "action": "skip", + "reason": "remote_addition", + "remote": "13.3.x-snapshot-scala2.12" + }, + "tasks[task_key='task_cluster'].new_cluster.custom_tags": { + "action": "skip", + "reason": "remote_addition", + "remote": { + "CostCenter": "from-policy" + } + }, + "tasks[task_key='task_cluster'].new_cluster.spark_version": { + "action": "skip", + "reason": "remote_addition", + "remote": "13.3.x-snapshot-scala2.12" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.cluster_policies.my_policy - delete resources.jobs.j + delete resources.jobs.my_job All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/script b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/script index 6d871fee2b3..3c6c5f02c10 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/script @@ -7,17 +7,9 @@ trap cleanup EXIT trace $CLI bundle deploy -# Mask the server-assigned policy id (real backend returns a hex id the built-in -# UUID replacement misses). -read_id.py my_policy > /dev/null -job_id=$(read_id.py j) - -title "Did the fixed policy SUPPLY spark_version and custom_tags, or only validate them?\n" -echo "The bundle declares neither, and does not set apply_policy_default_values." -$CLI jobs get "$job_id" | gron.py | grep -E 'new_cluster.(spark_version|custom_tags)' - -title "Is the policy-supplied value reported as drift?\n" +title "Policy-supplied spark_version and custom_tags are not drift at any cluster-spec location\n" trace $CLI bundle plan -title "spark_version is not covered by any backend_defaults rule, so show its verdict\n" -trace $CLI bundle plan -o json | gron.py | grep -E 'custom_tags|spark_version' +title "Each is classified remote_addition\n" +trace $CLI bundle plan -o json | jq -S '.plan["resources.jobs.my_job"].changes + | with_entries(select(.key | test("custom_tags|spark_version")))' diff --git a/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/databricks.yml.tmpl index 3e678c334fc..0614a23657b 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/databricks.yml.tmpl @@ -17,10 +17,10 @@ resources: # The config owns custom_tags.Mine. It is deliberately NOT a policy-governed tag: a fixed # policy enforces on edit too, so an out-of-band change to the policy's own tag would be # rejected (see config_conflicts_policy) and there would be no drifted state to observe. - j: + my_job: name: test-job-$UNIQUE_NAME job_clusters: - - job_cluster_key: small + - job_cluster_key: main new_cluster: policy_id: ${resources.cluster_policies.my_policy.id} spark_version: $DEFAULT_SPARK_VERSION @@ -29,7 +29,7 @@ resources: custom_tags: Mine: mine tasks: - - task_key: t - job_cluster_key: small + - task_key: main + job_cluster_key: main spark_python_task: python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/output.txt index 8228bff8f73..41f5caf208a 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/output.txt @@ -2,14 +2,14 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.my_policy -Created jobs.j +Created jobs.my_job Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === An out-of-band change to a config-owned tag is drift, even with a policy attached >>> [CLI] bundle plan -update jobs.j +update jobs.my_job Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged @@ -18,23 +18,27 @@ Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged >>> update_file.py databricks.yml Mine: mine Other: other >>> [CLI] bundle plan -o json -json.plan.resources.cluster_policies.my_policy.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}"; -json.plan.resources.jobs.j.new_state.value.job_clusters[0].new_cluster.custom_tags.Other = "other"; -json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "from-policy"; -json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.Mine = "changed-out-of-band"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].action = "skip"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].reason = "remote_addition"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['CostCenter'].remote = "from-policy"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].action = "update"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].old = "mine"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Mine'].remote = "changed-out-of-band"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Other'].action = "update"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='small'].new_cluster.custom_tags['Other'].new = "other"; +{ + "job_clusters[job_cluster_key='main'].new_cluster.custom_tags['CostCenter']": { + "action": "skip", + "reason": "remote_addition", + "remote": "from-policy" + }, + "job_clusters[job_cluster_key='main'].new_cluster.custom_tags['Mine']": { + "action": "update", + "old": "mine", + "remote": "changed-out-of-band" + }, + "job_clusters[job_cluster_key='main'].new_cluster.custom_tags['Other']": { + "action": "update", + "new": "other" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.cluster_policies.my_policy - delete resources.jobs.j + delete resources.jobs.my_job All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script index 6a45d69f047..d86c89445ee 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script @@ -6,7 +6,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -job_id=$(read_id.py j) +job_id=$(read_id.py my_job) title "An out-of-band change to a config-owned tag is drift, even with a policy attached\n" edit_resource.py jobs "$job_id" <<'EOF' @@ -17,4 +17,5 @@ trace $CLI bundle plan | contains.py "1 to change" title "Removing a config-owned tag is a change the user asked for, not a backend addition\n" trace update_file.py databricks.yml "Mine: mine" "Other: other" -trace $CLI bundle plan -o json | gron.py | grep custom_tags +trace $CLI bundle plan -o json | jq -S '.plan["resources.jobs.my_job"].changes + | with_entries(select(.key | test("custom_tags")))' diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/databricks.yml.tmpl index cec8acae6d8..61fc54a7c3b 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/databricks.yml.tmpl @@ -13,11 +13,10 @@ resources: type: fixed value: from-policy - # policy_id gates suppression per cluster spec, not per resource. Gated clusters appear at - # every jobs location (job_clusters, tasks[*].new_cluster, for_each); one cluster omits - # policy_id, and an out-of-band tag on it must still surface as drift. + # policy_id gates suppression per cluster spec, not per resource: the gated cluster's + # policy tag is skipped, while an out-of-band tag on the ungated sibling still surfaces. jobs: - j: + my_job: name: test-job-$UNIQUE_NAME job_clusters: - job_cluster_key: gated @@ -40,23 +39,3 @@ resources: job_cluster_key: ungated spark_python_task: python_file: ./hello_world.py - - task_key: task_cluster - new_cluster: - policy_id: ${resources.cluster_policies.my_policy.id} - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - spark_python_task: - python_file: ./hello_world.py - - task_key: foreach - for_each_task: - inputs: "[1]" - task: - task_key: inner - new_cluster: - policy_id: ${resources.cluster_policies.my_policy.id} - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - spark_python_task: - python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/output.txt b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/output.txt index 2cceb5c06a9..ce1358f501c 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/output.txt +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/output.txt @@ -2,36 +2,35 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.my_policy -Created jobs.j +Created jobs.my_job Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Inject a remote-only tag on the ungated cluster (no policy_id) to show it is not gated -=== The policy tag is skipped on every gated location; the ungated cluster's tag is drift +=== The gated cluster's policy tag is skipped; the ungated cluster's tag is drift >>> [CLI] bundle plan -o json -json.plan.resources.cluster_policies.my_policy.remote_state.definition = "{\"custom_tags.CostCenter\":{\"type\":\"fixed\",\"value\":\"from-policy\"}}"; -json.plan.resources.jobs.j.remote_state.job_clusters[0].new_cluster.custom_tags.CostCenter = "from-policy"; -json.plan.resources.jobs.j.remote_state.job_clusters[1].new_cluster.custom_tags.Injected = "out-of-band"; -json.plan.resources.jobs.j.remote_state.tasks[0].for_each_task.task.new_cluster.custom_tags.CostCenter = "from-policy"; -json.plan.resources.jobs.j.remote_state.tasks[3].new_cluster.custom_tags.CostCenter = "from-policy"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='gated'].new_cluster.custom_tags.action = "skip"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='gated'].new_cluster.custom_tags.reason = "remote_addition"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='gated'].new_cluster.custom_tags.remote.CostCenter = "from-policy"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='ungated'].new_cluster.custom_tags.action = "update"; -json.plan.resources.jobs.j.changes.job_clusters[job_cluster_key='ungated'].new_cluster.custom_tags.remote.Injected = "out-of-band"; -json.plan.resources.jobs.j.changes.tasks[task_key='foreach'].for_each_task.task.new_cluster.custom_tags.action = "skip"; -json.plan.resources.jobs.j.changes.tasks[task_key='foreach'].for_each_task.task.new_cluster.custom_tags.reason = "remote_addition"; -json.plan.resources.jobs.j.changes.tasks[task_key='foreach'].for_each_task.task.new_cluster.custom_tags.remote.CostCenter = "from-policy"; -json.plan.resources.jobs.j.changes.tasks[task_key='task_cluster'].new_cluster.custom_tags.action = "skip"; -json.plan.resources.jobs.j.changes.tasks[task_key='task_cluster'].new_cluster.custom_tags.reason = "remote_addition"; -json.plan.resources.jobs.j.changes.tasks[task_key='task_cluster'].new_cluster.custom_tags.remote.CostCenter = "from-policy"; +{ + "job_clusters[job_cluster_key='gated'].new_cluster.custom_tags": { + "action": "skip", + "reason": "remote_addition", + "remote": { + "CostCenter": "from-policy" + } + }, + "job_clusters[job_cluster_key='ungated'].new_cluster.custom_tags": { + "action": "update", + "remote": { + "Injected": "out-of-band" + } + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.cluster_policies.my_policy - delete resources.jobs.j + delete resources.jobs.my_job All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script index 7e493714806..cdbdcaba7c0 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script @@ -6,7 +6,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -job_id=$(read_id.py j) +job_id=$(read_id.py my_job) title "Inject a remote-only tag on the ungated cluster (no policy_id) to show it is not gated\n" edit_resource.py jobs "$job_id" <<'EOF' @@ -15,5 +15,6 @@ for jc in r["job_clusters"]: jc["new_cluster"]["custom_tags"] = {"Injected": "out-of-band"} EOF -title "The policy tag is skipped on every gated location; the ungated cluster's tag is drift\n" -trace $CLI bundle plan -o json | gron.py | grep custom_tags +title "The gated cluster's policy tag is skipped; the ungated cluster's tag is drift\n" +trace $CLI bundle plan -o json | jq -S '.plan["resources.jobs.my_job"].changes + | with_entries(select(.key | test("custom_tags")))' diff --git a/acceptance/bundle/resources/pipelines/cluster_policy/databricks.yml.tmpl b/acceptance/bundle/resources/pipelines/cluster_policy/databricks.yml.tmpl index 836cd226384..7af2175aa84 100644 --- a/acceptance/bundle/resources/pipelines/cluster_policy/databricks.yml.tmpl +++ b/acceptance/bundle/resources/pipelines/cluster_policy/databricks.yml.tmpl @@ -17,7 +17,7 @@ resources: # into the stored spec: it reads back exactly as authored, so there is no addition to suppress # and no ignore_remote_additions rule for pipelines. This records that it simply converges. pipelines: - p: + my_pipeline: name: test-pipeline-$UNIQUE_NAME clusters: - label: default diff --git a/acceptance/bundle/resources/pipelines/cluster_policy/hello_world.py b/acceptance/bundle/resources/pipelines/cluster_policy/hello_world.py index f301245e242..11b15b1a458 100644 --- a/acceptance/bundle/resources/pipelines/cluster_policy/hello_world.py +++ b/acceptance/bundle/resources/pipelines/cluster_policy/hello_world.py @@ -1 +1 @@ -print("Hello World!") +print("hello") diff --git a/acceptance/bundle/resources/pipelines/cluster_policy/output.txt b/acceptance/bundle/resources/pipelines/cluster_policy/output.txt index a6355bbeba8..2df33606292 100644 --- a/acceptance/bundle/resources/pipelines/cluster_policy/output.txt +++ b/acceptance/bundle/resources/pipelines/cluster_policy/output.txt @@ -2,7 +2,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created cluster_policies.my_policy -Created pipelines.p +Created pipelines.my_pipeline Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged @@ -14,11 +14,11 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.cluster_policies.my_policy - delete resources.pipelines.p + delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: - delete resources.pipelines.p + delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] From 897f89def5c0999b0636871b79d705bbfbeee241 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 11 Sep 2026 11:43:52 +0000 Subject: [PATCH 22/25] test: reproduce config-sync dropping remote cluster additions --- .../databricks.yml.tmpl | 66 +++++++++++++++++++ .../cluster_policy_remote_addition/hello.py | 3 + .../out.test.toml | 4 ++ .../cluster_policy_remote_addition/output.txt | 49 ++++++++++++++ .../cluster_policy_remote_addition/script | 29 ++++++++ .../cluster_policy_remote_addition/test.toml | 4 ++ 6 files changed, 155 insertions(+) create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl new file mode 100644 index 00000000000..b0f61018ee4 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl @@ -0,0 +1,66 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + policy: + name: test-policy-$UNIQUE_NAME + definition: + # This rule controls only PolicyTag, not the Spark setting edited below. + custom_tags.PolicyTag: + type: fixed + value: from-policy + jobs: + test_job: + name: test-job-$UNIQUE_NAME + max_concurrent_runs: 1 + job_clusters: + - job_cluster_key: shared + new_cluster: + policy_id: ${resources.cluster_policies.policy.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + PolicyTag: from-policy + - job_cluster_key: ungated + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + tasks: + - task_key: shared_task + job_cluster_key: shared + spark_python_task: + python_file: ./hello.py + - task_key: ungated_task + job_cluster_key: ungated + spark_python_task: + python_file: ./hello.py + - task_key: own_cluster + new_cluster: + policy_id: ${resources.cluster_policies.policy.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + PolicyTag: from-policy + spark_python_task: + python_file: ./hello.py + - task_key: loop + for_each_task: + inputs: "[1]" + task: + task_key: nested_cluster + new_cluster: + policy_id: ${resources.cluster_policies.policy.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + PolicyTag: from-policy + spark_python_task: + python_file: ./hello.py diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py new file mode 100644 index 00000000000..8cb8f7a2d91 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python3 + +print("hello") diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/out.test.toml b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/out.test.toml new file mode 100644 index 00000000000..0db72c1b195 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt new file mode 100644 index 00000000000..24d5f4a3184 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt @@ -0,0 +1,49 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.policy +Created jobs.test_job +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle config-remote-sync +No changes detected. + + +=== Sync remote user edits unrelated to the attached policy + +>>> [CLI] bundle config-remote-sync --save +Detected changes in 1 resource(s): + +Resource: resources.jobs.test_job + job_clusters[job_cluster_key='shared'].new_cluster.num_workers: replace + job_clusters[job_cluster_key='shared'].new_cluster.spark_conf: add + job_clusters[job_cluster_key='ungated'].new_cluster.num_workers: replace + job_clusters[job_cluster_key='ungated'].new_cluster.spark_conf: add + max_concurrent_runs: replace + tasks[task_key='loop'].for_each_task.task.new_cluster.num_workers: replace + tasks[task_key='loop'].for_each_task.task.new_cluster.spark_conf: add + tasks[task_key='own_cluster'].new_cluster.num_workers: replace + tasks[task_key='own_cluster'].new_cluster.spark_conf: add + + + +>>> [CLI] bundle validate -o json +json.resources.jobs.test_job.job_clusters[0].new_cluster.num_workers = 2; +json.resources.jobs.test_job.job_clusters[0].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; +json.resources.jobs.test_job.job_clusters[1].new_cluster.num_workers = 2; +json.resources.jobs.test_job.job_clusters[1].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; +json.resources.jobs.test_job.max_concurrent_runs = 5; +json.resources.jobs.test_job.tasks[0].for_each_task.task.new_cluster.num_workers = 2; +json.resources.jobs.test_job.tasks[0].for_each_task.task.new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; +json.resources.jobs.test_job.tasks[1].new_cluster.num_workers = 2; +json.resources.jobs.test_job.tasks[1].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.policy + delete resources.jobs.test_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script new file mode 100644 index 00000000000..70f04d2e8f1 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script @@ -0,0 +1,29 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +job_id=$(read_id.py test_job) + +trace $CLI bundle config-remote-sync | contains.py "No changes detected" + +title "Sync remote user edits unrelated to the attached policy\n" +edit_resource.py jobs "$job_id" <<'EOF' +r["max_concurrent_runs"] = 5 +clusters = [jc["new_cluster"] for jc in r["job_clusters"]] +for task in r["tasks"]: + if "new_cluster" in task: + clusters.append(task["new_cluster"]) + if "for_each_task" in task: + clusters.append(task["for_each_task"]["task"]["new_cluster"]) +for cluster in clusters: + cluster["spark_conf"] = {"spark.sql.shuffle.partitions": "7"} + cluster["num_workers"] = 2 +EOF + +trace $CLI bundle config-remote-sync --save +# The snapshot must include the user addition for every cluster, including those with a policy. +trace $CLI bundle validate -o json | gron.py | grep -E 'max_concurrent_runs|num_workers|spark.sql.shuffle.partitions' diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml new file mode 100644 index 00000000000..119189805fd --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml @@ -0,0 +1,4 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = false +Ignore = [".databricks", "databricks.yml"] From 86da9c2601c31cc9e7f7e4a1e976c23b15587e84 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 11 Sep 2026 12:03:27 +0000 Subject: [PATCH 23/25] test: simplify policy sync case and show YAML diff --- .../databricks.yml.tmpl | 41 +----------------- .../cluster_policy_remote_addition/output.txt | 42 +++++++------------ .../cluster_policy_remote_addition/script | 19 ++------- .../cluster_policy_remote_addition/test.toml | 2 +- 4 files changed, 21 insertions(+), 83 deletions(-) diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl index b0f61018ee4..346c8366609 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl @@ -9,38 +9,15 @@ resources: policy: name: test-policy-$UNIQUE_NAME definition: - # This rule controls only PolicyTag, not the Spark setting edited below. + # The policy does not constrain spark_conf. custom_tags.PolicyTag: type: fixed value: from-policy jobs: test_job: name: test-job-$UNIQUE_NAME - max_concurrent_runs: 1 - job_clusters: - - job_cluster_key: shared - new_cluster: - policy_id: ${resources.cluster_policies.policy.id} - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - custom_tags: - PolicyTag: from-policy - - job_cluster_key: ungated - new_cluster: - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 tasks: - - task_key: shared_task - job_cluster_key: shared - spark_python_task: - python_file: ./hello.py - - task_key: ungated_task - job_cluster_key: ungated - spark_python_task: - python_file: ./hello.py - - task_key: own_cluster + - task_key: test new_cluster: policy_id: ${resources.cluster_policies.policy.id} spark_version: $DEFAULT_SPARK_VERSION @@ -50,17 +27,3 @@ resources: PolicyTag: from-policy spark_python_task: python_file: ./hello.py - - task_key: loop - for_each_task: - inputs: "[1]" - task: - task_key: nested_cluster - new_cluster: - policy_id: ${resources.cluster_policies.policy.id} - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - custom_tags: - PolicyTag: from-policy - spark_python_task: - python_file: ./hello.py diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt index 24d5f4a3184..f2756c75248 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt @@ -6,38 +6,24 @@ Created jobs.test_job Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] bundle config-remote-sync -No changes detected. - - -=== Sync remote user edits unrelated to the attached policy - >>> [CLI] bundle config-remote-sync --save Detected changes in 1 resource(s): Resource: resources.jobs.test_job - job_clusters[job_cluster_key='shared'].new_cluster.num_workers: replace - job_clusters[job_cluster_key='shared'].new_cluster.spark_conf: add - job_clusters[job_cluster_key='ungated'].new_cluster.num_workers: replace - job_clusters[job_cluster_key='ungated'].new_cluster.spark_conf: add - max_concurrent_runs: replace - tasks[task_key='loop'].for_each_task.task.new_cluster.num_workers: replace - tasks[task_key='loop'].for_each_task.task.new_cluster.spark_conf: add - tasks[task_key='own_cluster'].new_cluster.num_workers: replace - tasks[task_key='own_cluster'].new_cluster.spark_conf: add - - - ->>> [CLI] bundle validate -o json -json.resources.jobs.test_job.job_clusters[0].new_cluster.num_workers = 2; -json.resources.jobs.test_job.job_clusters[0].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; -json.resources.jobs.test_job.job_clusters[1].new_cluster.num_workers = 2; -json.resources.jobs.test_job.job_clusters[1].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; -json.resources.jobs.test_job.max_concurrent_runs = 5; -json.resources.jobs.test_job.tasks[0].for_each_task.task.new_cluster.num_workers = 2; -json.resources.jobs.test_job.tasks[0].for_each_task.task.new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; -json.resources.jobs.test_job.tasks[1].new_cluster.num_workers = 2; -json.resources.jobs.test_job.tasks[1].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; + tasks[task_key='test'].new_cluster.spark_conf: add + + + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -26,4 +26,6 @@ + custom_tags: + PolicyTag: from-policy ++ spark_conf: ++ spark.sql.shuffle.partitions: "7" + spark_python_task: + python_file: ./hello.py >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script index 70f04d2e8f1..afe89d9a834 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script @@ -8,22 +8,11 @@ trap cleanup EXIT trace $CLI bundle deploy job_id=$(read_id.py test_job) -trace $CLI bundle config-remote-sync | contains.py "No changes detected" - -title "Sync remote user edits unrelated to the attached policy\n" edit_resource.py jobs "$job_id" <<'EOF' -r["max_concurrent_runs"] = 5 -clusters = [jc["new_cluster"] for jc in r["job_clusters"]] -for task in r["tasks"]: - if "new_cluster" in task: - clusters.append(task["new_cluster"]) - if "for_each_task" in task: - clusters.append(task["for_each_task"]["task"]["new_cluster"]) -for cluster in clusters: - cluster["spark_conf"] = {"spark.sql.shuffle.partitions": "7"} - cluster["num_workers"] = 2 +r["tasks"][0]["new_cluster"]["spark_conf"] = {"spark.sql.shuffle.partitions": "7"} EOF +cp databricks.yml databricks.yml.backup trace $CLI bundle config-remote-sync --save -# The snapshot must include the user addition for every cluster, including those with a policy. -trace $CLI bundle validate -o json | gron.py | grep -E 'max_concurrent_runs|num_workers|spark.sql.shuffle.partitions' +trace diff.py databricks.yml.backup databricks.yml +rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml index 119189805fd..b13ee63c9a2 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml @@ -1,4 +1,4 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = false -Ignore = [".databricks", "databricks.yml"] +Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"] From 4d47f8bcdcfedb35034290344a18a3fc65330952 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 11 Sep 2026 14:07:26 +0000 Subject: [PATCH 24/25] test: let cluster policy supply its tag --- .../cluster_policy_remote_addition/databricks.yml.tmpl | 2 -- .../cluster_policy_remote_addition/output.txt | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl index 346c8366609..d4eb124a631 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl @@ -23,7 +23,5 @@ resources: spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 1 - custom_tags: - PolicyTag: from-policy spark_python_task: python_file: ./hello.py diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt index f2756c75248..325be5ed749 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt @@ -17,9 +17,9 @@ Resource: resources.jobs.test_job >>> diff.py databricks.yml.backup databricks.yml --- databricks.yml.backup +++ databricks.yml -@@ -26,4 +26,6 @@ - custom_tags: - PolicyTag: from-policy +@@ -24,4 +24,6 @@ + node_type_id: [NODE_TYPE_ID] + num_workers: 1 + spark_conf: + spark.sql.shuffle.partitions: "7" spark_python_task: From 3e5fc9c41468795ae1ba9856fe9df81f74c592d2 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 11 Sep 2026 14:27:40 +0000 Subject: [PATCH 25/25] fix: preserve config-sync of remote cluster additions --- bundle/configsync/diff.go | 2 +- bundle/direct/bundle_plan.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index b715a1f6fa9..7c79456abae 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -189,7 +189,7 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan if entry.Changes != nil { for path, changeDesc := range entry.Changes { - if changeDesc.Action == deployplan.Skip { + if changeDesc.Action == deployplan.Skip && changeDesc.Reason != deployplan.ReasonRemoteAddition { continue } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index abb45c2f674..2fb60d3677f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -467,15 +467,15 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change } else if action, reason, ok := classifyIDField(generatedCfg, path, ch); ok { ch.Action = action ch.Reason = reason - } else if reason, ok := shouldSkipRemoteAddition(cfg, path, ch, newState); ok { - ch.Action = deployplan.Skip - ch.Reason = reason } else if reason, ok := shouldSkipBackendDefault(cfg, path, ch); ok { ch.Action = deployplan.Skip ch.Reason = reason } else if reason, ok := shouldSkipBackendDefault(generatedCfg, path, ch); ok { ch.Action = deployplan.Skip ch.Reason = reason + } else if reason, ok := shouldSkipRemoteAddition(cfg, path, ch, newState); ok { + ch.Action = deployplan.Skip + ch.Reason = reason } else if reason, ok := shouldSkipNormalized(cfg, path, ch); ok { ch.Action = deployplan.Skip ch.Reason = reason