From 28b9a4b641d6a89f29dd682c7466b45c7265b7db Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 11:37:17 +0000 Subject: [PATCH 01/17] Bump direct deployment state version to 3 --- .nextchanges/bundles/direct-state-v3.md | 1 + .../bundle/deploy/wal/chain-3-jobs/output.txt | 2 +- .../deploy/wal/crash-after-create/output.txt | 2 +- .../deploy/wal/header-only-wal/output.txt | 4 +- .../out.state_after_bind.direct.json | 2 +- .../bundle/dms/declined-deploy/output.txt | 2 +- acceptance/bundle/dms/depends-on/output.txt | 2 +- acceptance/bundle/dms/empty-grants/output.txt | 2 +- .../bundle/dms/existing-state/output.txt | 4 +- .../bundle/dms/failed-delete/output.txt | 4 +- .../bundle/dms/failed-recreate/output.txt | 4 +- .../dms/failed-update-permissions/output.txt | 4 +- .../bundle/dms/multiple-resources/output.txt | 4 +- acceptance/bundle/dms/no-drift/output.txt | 4 +- acceptance/bundle/dms/no-resources/output.txt | 4 +- .../operation-upload-fails-once/output.txt | 4 +- acceptance/bundle/dms/provenance/output.txt | 2 +- .../bundle/dms/record-failure/output.txt | 2 +- acceptance/bundle/dms/stale-plan/output.txt | 2 +- .../bundle/dms/successful-recreate/output.txt | 4 +- .../bundle/migrate/basic/out.new_state.json | 2 +- .../migrate/dashboards/out.new_state.json | 2 +- .../out.state_after_migration.json | 2 +- .../bundle/migrate/grants/out.new_state.json | 2 +- .../migrate/permissions/out.new_state.json | 2 +- .../bundle/migrate/runas/out.new_state.json | 2 +- .../jobs/big_id/out.state.direct.json | 2 +- .../jobs/update/out.state.direct.json | 2 +- .../bundle/state/feature_flags/output.txt | 6 +-- acceptance/bundle/state/feature_flags/script | 2 +- .../permission_level_migration/output.txt | 2 +- .../simple/out.requests.deploy.direct.json | 2 +- bundle/direct/dstate/migrate.go | 26 +++------ bundle/direct/dstate/state.go | 35 ++---------- bundle/direct/dstate/state_test.go | 54 +++++++++---------- 35 files changed, 77 insertions(+), 125 deletions(-) create mode 100644 .nextchanges/bundles/direct-state-v3.md diff --git a/.nextchanges/bundles/direct-state-v3.md b/.nextchanges/bundles/direct-state-v3.md new file mode 100644 index 00000000000..c263c3f8123 --- /dev/null +++ b/.nextchanges/bundles/direct-state-v3.md @@ -0,0 +1 @@ +* Bump the direct deployment state version to 3 for bundles with and without deployment history. diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt index d3bfab4923a..0fea6799b11 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt @@ -11,7 +11,7 @@ Exit code: [KILLED] "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "state_version": 2 + "state_version": 3 } { "k": "resources.jobs.job_01", diff --git a/acceptance/bundle/deploy/wal/crash-after-create/output.txt b/acceptance/bundle/deploy/wal/crash-after-create/output.txt index fc8ace6bf00..56822d5fbc0 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/output.txt +++ b/acceptance/bundle/deploy/wal/crash-after-create/output.txt @@ -12,7 +12,7 @@ Exit code: [KILLED] >>> cat .databricks/bundle/default/resources.json.wal { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1 diff --git a/acceptance/bundle/deploy/wal/header-only-wal/output.txt b/acceptance/bundle/deploy/wal/header-only-wal/output.txt index 3d4eb38635d..86ca2583663 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/output.txt +++ b/acceptance/bundle/deploy/wal/header-only-wal/output.txt @@ -7,7 +7,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/defaul Exit code: [KILLED] >>> cat .databricks/bundle/default/resources.json.wal -{"state_version":2,"cli_version":"[CLI_VERSION]","lineage":"[UUID]","serial":1} +{"state_version":3,"cli_version":"[CLI_VERSION]","lineage":"[UUID]","serial":1} === Second deploy (killed again, leaves another header-only WAL) >>> errcode [CLI] bundle deploy --force-lock @@ -17,7 +17,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/defaul Exit code: [KILLED] >>> cat .databricks/bundle/default/resources.json.wal -{"state_version":2,"cli_version":"[CLI_VERSION]","lineage":"[UUID]","serial":1} +{"state_version":3,"cli_version":"[CLI_VERSION]","lineage":"[UUID]","serial":1} === Third deploy (must recover and succeed, not blocked by the leftover WAL) >>> errcode [CLI] bundle deploy --force-lock diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json index eb3a71d91ff..05614d798bf 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 7a3e4e58bed..1492106a3fa 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -94,7 +94,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 50aeb1d46e5..d1f77897b71 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -86,7 +86,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index a0db7b41dd6..0ff3f79a722 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -158,7 +158,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index a8d2f79a989..38d2c403172 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -21,7 +21,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> jq {state_version, features} .databricks/bundle/default/resources.json { - "state_version": 2, + "state_version": 3, "features": null } @@ -77,7 +77,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> jq {state_version, features} .databricks/bundle/default/resources.json { - "state_version": 2, + "state_version": 3, "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 4f8de666aa9..6c908dd1b50 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -87,7 +87,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw state after successful deploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -178,7 +178,7 @@ API message: Fault injected by test. === The state shows the resource still exists but the operation is marked failed >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 9f354ed381d..efa9264f9ad 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -87,7 +87,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw state after successful deploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -173,7 +173,7 @@ Files: 2 uploaded, 0 deleted === The raw state after failed recreate shows no resource >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 8c5be14e0b5..bdb934a9871 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -40,7 +40,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The raw recorded state after deploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -94,7 +94,7 @@ Files: 1 uploaded, 0 deleted === The raw recorded state after the failure >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 04c529b2380..47bcf5690d9 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -65,7 +65,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -130,7 +130,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged === The raw recorded state after redeploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 4dc8a5c9269..0830a708248 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -45,7 +45,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -178,7 +178,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The raw recorded state the service holds after the update >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index c2b96d2f736..b0167af933e 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -47,7 +47,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -113,7 +113,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged === The raw recorded state after redeploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index e33b870fd84..b9ad9db7eee 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -34,7 +34,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw recorded state after deploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -76,7 +76,7 @@ Files: 2 uploaded, 0 deleted === The raw recorded state after the failure >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index ffe5cec906e..dff91b1e0e7 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -95,7 +95,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms -t dev { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 6b16a68b15b..87824ba440a 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -87,7 +87,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 3ccb46a3e11..c4e2f6844cf 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -30,7 +30,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 2257b78cc8c..65192d759de 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -87,7 +87,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -189,7 +189,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw recorded state after recreate >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json index 7a02965e08f..e924cb4d005 100644 --- a/acceptance/bundle/migrate/basic/out.new_state.json +++ b/acceptance/bundle/migrate/basic/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/dashboards/out.new_state.json b/acceptance/bundle/migrate/dashboards/out.new_state.json index ed623b6b487..3a405e6b088 100644 --- a/acceptance/bundle/migrate/dashboards/out.new_state.json +++ b/acceptance/bundle/migrate/dashboards/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/default-python/out.state_after_migration.json index c1e83c7d577..484db23fd94 100644 --- a/acceptance/bundle/migrate/default-python/out.state_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.state_after_migration.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index 12da0aa5f9b..82b918f76d3 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 9, diff --git a/acceptance/bundle/migrate/permissions/out.new_state.json b/acceptance/bundle/migrate/permissions/out.new_state.json index 522d892b949..53d41f1fe45 100644 --- a/acceptance/bundle/migrate/permissions/out.new_state.json +++ b/acceptance/bundle/migrate/permissions/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 7, diff --git a/acceptance/bundle/migrate/runas/out.new_state.json b/acceptance/bundle/migrate/runas/out.new_state.json index 8c6bb5060ce..134f7aff689 100644 --- a/acceptance/bundle/migrate/runas/out.new_state.json +++ b/acceptance/bundle/migrate/runas/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json index b6075147e57..7bbc03c50c2 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/out.state.direct.json b/acceptance/bundle/resources/jobs/update/out.state.direct.json index 6d978c0208d..4897c138fc8 100644 --- a/acceptance/bundle/resources/jobs/update/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/state/feature_flags/output.txt b/acceptance/bundle/state/feature_flags/output.txt index b6385259dd2..1370eb2bca6 100644 --- a/acceptance/bundle/state/feature_flags/output.txt +++ b/acceptance/bundle/state/feature_flags/output.txt @@ -1,12 +1,12 @@ === a state depending on a feature this CLI does not recognize is rejected >>> errcode [CLI] bundle plan -Error: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information Exit code: 1 -=== a state with an empty features map is accepted, and a deploy keeps it at version 2 +=== a state with an empty features map is accepted, and a deploy keeps it at version 3 >>> [CLI] bundle plan create jobs.my_job @@ -19,4 +19,4 @@ Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> gron.py .databricks/bundle/default/resources.json -json.state_version = 2; +json.state_version = 3; diff --git a/acceptance/bundle/state/feature_flags/script b/acceptance/bundle/state/feature_flags/script index ef9570e318c..c194ee7491f 100644 --- a/acceptance/bundle/state/feature_flags/script +++ b/acceptance/bundle/state/feature_flags/script @@ -4,7 +4,7 @@ title "a state depending on a feature this CLI does not recognize is rejected" cp resources.with_feature.json .databricks/bundle/default/resources.json trace errcode $CLI bundle plan 2>&1 | contains.py "requires features this CLI does not support: future_feature" "upgrade to the latest CLI version" "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" -title "a state with an empty features map is accepted, and a deploy keeps it at version 2" +title "a state with an empty features map is accepted, and a deploy keeps it at version 3" cp resources.empty_features.json .databricks/bundle/default/resources.json trace $CLI bundle plan | contains.py "Plan:" trace $CLI bundle deploy diff --git a/acceptance/bundle/state/permission_level_migration/output.txt b/acceptance/bundle/state/permission_level_migration/output.txt index e4b00bb73fd..919997d9267 100644 --- a/acceptance/bundle/state/permission_level_migration/output.txt +++ b/acceptance/bundle/state/permission_level_migration/output.txt @@ -13,7 +13,7 @@ Resources: 1 created, 1 changed, 0 deleted, 0 unchanged === Print state after deploy >>> print_state.py { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "test-lineage", "serial": 2, diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index 5326f079064..f0c60602386 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -240,7 +240,7 @@ "overwrite": "true" }, "body": { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index e4d21a7054a..3ee818bd38e 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -3,8 +3,6 @@ package dstate import ( "encoding/json" "fmt" - "slices" - "strings" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/structs/structpath" @@ -15,29 +13,15 @@ import ( // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // featureStateVersion states carry a feature list this CLI does not yet write or - // understand (see the featureStateVersion doc comment). A featureStateVersion - // state with no features is equivalent to currentStateVersion, so accept it and - // return without running the migrations below, leaving the on-disk version at - // featureStateVersion rather than flipping it down. One that records any feature - // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. - if db.StateVersion == featureStateVersion { - if len(db.Features) == 0 { - return nil - } - features := make([]string, 0, len(db.Features)) - for name := range db.Features { - features = append(features, name) - } - slices.Sort(features) - return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) + if err := assertNoUnsupportedFeatures(db.Features); err != nil { + return err } if db.StateVersion == currentStateVersion { return nil } - if db.StateVersion > supportedStateVersion { - return fmt.Errorf("state version %d is newer than supported version %d; upgrade the CLI", db.StateVersion, supportedStateVersion) + if db.StateVersion > currentStateVersion { + return fmt.Errorf("state version %d is newer than supported version %d; upgrade the CLI", db.StateVersion, currentStateVersion) } for version := db.StateVersion; version < currentStateVersion; version++ { @@ -59,6 +43,8 @@ func migrateState(db *Database) error { var migrations = map[int]func(*Database) error{ 0: migrateV1ToV2, 1: migrateV1ToV2, + // Version 3 adds feature flags; existing states need no data changes. + 2: func(*Database) error { return nil }, } // migrateV1ToV2 migrates permissions and grants entries from the old format diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index aa2d53620d0..48c32c9e0ee 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -27,35 +27,12 @@ import ( ) const ( - // currentStateVersion is the schema version written for deployments that record - // no feature flags, and the version legacy states are migrated up to on load. - currentStateVersion = 2 + // currentStateVersion is the schema version written by this CLI and the version + // legacy states are migrated up to on load. + currentStateVersion = 3 initialBufferSize = 64 * 1024 maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - - // featureStateVersion is the schema version a future CLI will write once it - // records deployment state "feature flags" (see Header.Features). This CLI does - // not write it and records no features; it exists now only so this CLI reads - // such states correctly (see migrateState): - // - featureStateVersion with no features -> accept and leave the version as-is - // - featureStateVersion with any feature -> refuse, tell the user to upgrade - // - // A featureStateVersion state with no features is equivalent to - // currentStateVersion, but we deliberately do not flip the on-disk version down - // to currentStateVersion: a state written at featureStateVersion stays at - // featureStateVersion. This is forward-compat scaffolding so that a later release - // can start writing featureStateVersion + features without older CLIs (with this - // change) either mishandling a feature they lack or rejecting a featureless state - // outright. featureStateVersion is always 3. - featureStateVersion = 3 - - // supportedStateVersion is the highest schema version this CLI can read. It is - // normally equal to currentStateVersion — the version this CLI reads is the - // version it writes — and exceeds it only during a two-phase version bump like - // the current feature-flag scaffolding, where this CLI reads (but does not - // write) featureStateVersion. A state newer than this is rejected as too new. - supportedStateVersion = featureStateVersion ) // FeatureDeploymentHistory marks a state whose resources are also recorded with the @@ -553,12 +530,6 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe return fmt.Errorf("migrating state %s: %w", path, err) } - // TODO: We can remove and move this assertion to migrateState once we do the state - // version bump to 3 for this CLI. - if err := assertNoUnsupportedFeatures(db.Data.Features); err != nil { - return err - } - db.stateIDs = make(map[string]string) for key, entry := range db.Data.State { db.stateIDs[key] = entry.ID diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 24af630b069..72a74027d78 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -2,6 +2,7 @@ package dstate import ( "encoding/json" + "fmt" "os" "path/filepath" "testing" @@ -182,36 +183,29 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { mustFinalize(t, &recovered) } -// TestEmptyFeatureStateAcceptedWithoutFlippingVersion pins the special case that a -// featureStateVersion state with no features is accepted as-is — the on-disk version -// is left at featureStateVersion, not flipped down to currentStateVersion — and that -// a featureStateVersion state recording any feature is refused. This is scaffolding -// for the deferred version bump, special-cased to featureStateVersion only (see the -// featureStateVersion doc comment). -// -// When the baseline is actually bumped to featureStateVersion, this special case must -// go away. This test is the forcing function: it fails once featureStateVersion is -// removed, making the author decide what the post-bump behavior should be. -func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { - // The special case applies to featureStateVersion (3) only. - require.Equal(t, 2, currentStateVersion, "when currentStateVersion is bumped, remove featureStateVersion and this special case") - require.Equal(t, 3, featureStateVersion) - - empty := &Database{Header: Header{StateVersion: featureStateVersion}} - require.NoError(t, migrateState(empty)) - assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") - - // v3 that records a feature is refused: this CLI does not understand features. - withFeature := &Database{Header: Header{ - StateVersion: featureStateVersion, - Features: map[string]struct{}{"future_feature": {}}, - }} - err := migrateState(withFeature) - require.Error(t, err) - assert.Contains(t, err.Error(), "requires features this CLI does not support") - assert.Contains(t, err.Error(), "future_feature") - assert.Contains(t, err.Error(), "upgrade to the latest CLI version") - assert.Contains(t, err.Error(), featuresDocURL) +func TestMigrateState(t *testing.T) { + for _, version := range []int{0, 1, 2, 3} { + for _, feature := range []string{"", FeatureDeploymentHistory, "future_feature"} { + t.Run(fmt.Sprintf("version=%d/feature=%s", version, feature), func(t *testing.T) { + db := &Database{Header: Header{StateVersion: version}} + if feature != "" { + db.Features = map[string]struct{}{feature: {}} + } + err := migrateState(db) + if feature == "future_feature" { + require.ErrorContains(t, err, "requires features this CLI does not support: future_feature") + assert.Contains(t, err.Error(), featuresDocURL) + assert.Equal(t, version, db.StateVersion) + return + } + require.NoError(t, err) + assert.Equal(t, 3, db.StateVersion) + if feature != "" { + assert.Contains(t, db.Features, feature) + } + }) + } + } } func TestDeleteState(t *testing.T) { From 6dafed64bb000c658897833ac804b266c8400a54 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 11:41:53 +0000 Subject: [PATCH 02/17] Test DMS state features and configuration guards --- .nextchanges/bundles/direct-state-v3.md | 2 +- .../dms/state-features/databricks.yml.tmpl | 9 ++++ .../bundle/dms/state-features/out.test.toml | 3 ++ .../bundle/dms/state-features/output.txt | 51 +++++++++++++++++++ acceptance/bundle/dms/state-features/script | 25 +++++++++ 5 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/dms/state-features/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/state-features/out.test.toml create mode 100644 acceptance/bundle/dms/state-features/output.txt create mode 100644 acceptance/bundle/dms/state-features/script diff --git a/.nextchanges/bundles/direct-state-v3.md b/.nextchanges/bundles/direct-state-v3.md index c263c3f8123..3768bfeef2f 100644 --- a/.nextchanges/bundles/direct-state-v3.md +++ b/.nextchanges/bundles/direct-state-v3.md @@ -1 +1 @@ -* Bump the direct deployment state version to 3 for bundles with and without deployment history. +* Bump the direct deployment state version to 3 for bundles with and without deployment history. ([#6706](https://github.com/databricks/cli/pull/6706)) diff --git a/acceptance/bundle/dms/state-features/databricks.yml.tmpl b/acceptance/bundle/dms/state-features/databricks.yml.tmpl new file mode 100644 index 00000000000..8f0cc75b608 --- /dev/null +++ b/acceptance/bundle/dms/state-features/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-state-features-$UNIQUE_NAME +experimental: + deployment_history: true + +resources: + jobs: + foo: + name: state-features diff --git a/acceptance/bundle/dms/state-features/out.test.toml b/acceptance/bundle/dms/state-features/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/state-features/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/state-features/output.txt b/acceptance/bundle/dms/state-features/output.txt new file mode 100644 index 00000000000..8af400ca848 --- /dev/null +++ b/acceptance/bundle/dms/state-features/output.txt @@ -0,0 +1,51 @@ + +=== Deploy records version 3 and the deployment_history feature, with resources held in DMS +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-state-features-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq {state_version, features, state} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": { + "deployment_history": {} + }, + "state": {} +} + +=== Reopening version 3 reads resources from DMS and produces a no-op plan +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_requests.py --dms //resources --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} + +=== Disabling deployment history refuses plan, deploy, and destroy +>>> musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle plan +Error: unsetting experimental.deployment_history is not supported + +This deployment's resources are recorded with the deployment history feature enabled. Set experimental.deployment_history: true to deploy or destroy this bundle + + +>>> musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle deploy +Error: unsetting experimental.deployment_history is not supported + +This deployment's resources are recorded with the deployment history feature enabled. Set experimental.deployment_history: true to deploy or destroy this bundle + + +>>> musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle destroy --auto-approve +Error: unsetting experimental.deployment_history is not supported + +This deployment's resources are recorded with the deployment history feature enabled. Set experimental.deployment_history: true to deploy or destroy this bundle + + +>>> print_requests.py --dms //api/2.0/bundle --oneline + +=== Reenabling deployment history still reads the unchanged resources from DMS +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_requests.py --dms //resources --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} diff --git a/acceptance/bundle/dms/state-features/script b/acceptance/bundle/dms/state-features/script new file mode 100644 index 00000000000..304cb384fdf --- /dev/null +++ b/acceptance/bundle/dms/state-features/script @@ -0,0 +1,25 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy records version 3 and the deployment_history feature, with resources held in DMS" +trace $CLI bundle deploy +trace jq '{state_version, features, state}' .databricks/bundle/default/resources.json +rm -f "$OUT_REQUESTS" + +title "Reopening version 3 reads resources from DMS and produces a no-op plan" +trace $CLI bundle plan +trace print_requests.py --dms //resources --get --oneline + +title "Disabling deployment history refuses plan, deploy, and destroy" +update_file.py databricks.yml 'deployment_history: true' 'deployment_history: false' +for command in plan deploy; do + trace musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle "$command" 2>&1 | contains.py "unsetting experimental.deployment_history is not supported" +done +trace musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle destroy --auto-approve 2>&1 | contains.py "unsetting experimental.deployment_history is not supported" +trace print_requests.py --dms //api/2.0/bundle --oneline + +title "Reenabling deployment history still reads the unchanged resources from DMS" +update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' +trace $CLI bundle plan +trace print_requests.py --dms //resources --get --oneline + +rm -f "$OUT_REQUESTS" From ae485dd5c1ad8b45cddf2a508e0ed081aa1f8da4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 11:46:57 +0000 Subject: [PATCH 03/17] Explain direct state version compatibility in changelog --- .nextchanges/bundles/direct-state-v3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/direct-state-v3.md b/.nextchanges/bundles/direct-state-v3.md index 3768bfeef2f..3339570da9e 100644 --- a/.nextchanges/bundles/direct-state-v3.md +++ b/.nextchanges/bundles/direct-state-v3.md @@ -1 +1 @@ -* Bump the direct deployment state version to 3 for bundles with and without deployment history. ([#6706](https://github.com/databricks/cli/pull/6706)) +* Bump the direct deployment state version to 3 for bundles with and without deployment history. Starting with this release, deploying or redeploying a bundle with the direct engine writes state that requires CLI v1.8.0 or newer. Older clients can fail with a "state version 3 is newer than supported version 2; upgrade the CLI" error when accessing these deployments. Upgrade local installations and CI/CD clients before deploying with this release or later. Bundles with deployment history enabled additionally require a CLI that supports the `deployment_history` feature and a bundle configured to use it; clients that do not support the feature reject the state even if they support state version 3. ([#6706](https://github.com/databricks/cli/pull/6706)) From 17ea6fabbc26f8ab0bcb4bb0395d33fe296a488f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 11:55:03 +0000 Subject: [PATCH 04/17] Cover state migration with old and current CLI acceptance tests --- acceptance/acceptance_test.go | 17 +- .../bundle/state/feature_flags/out.test.toml | 4 +- .../bundle/state/feature_flags/output.txt | 391 +++++++++++++++++- .../feature_flags/resources.with_feature.json | 10 - acceptance/bundle/state/feature_flags/script | 38 +- .../bundle/state/feature_flags/test.toml | 4 + acceptance/internal/config.go | 4 + acceptance/internal/materialized_config.go | 4 + bundle/direct/dstate/state_test.go | 26 -- 9 files changed, 444 insertions(+), 54 deletions(-) delete mode 100644 acceptance/bundle/state/feature_flags/resources.with_feature.json diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 3c98dff1b88..72838c04660 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -585,6 +585,19 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo t.Skip(skipReason) } + var testRepls testdiff.ReplacementsContext + // Download before t.Parallel so tests sharing a release cannot race to extract it. + for _, name := range slices.Sorted(maps.Keys(config.CLIVersions)) { + path := DownloadCLI(t, buildDir, config.CLIVersions[name]) + if config.Env == nil { + config.Env = make(map[string]string) + } + config.Env[name] = path + testRepls.SetPath(path, "["+name+"]") + } + // Replace executable paths before their parent build directory. + testRepls.Repls = append(testRepls.Repls, repls.Repls...) + runParallel := !inprocessMode if benchmarkMode && strings.Contains(dir, "benchmark") { runParallel = false @@ -625,7 +638,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo if reason := subset.skipReason(dir, nil); reason != "" { t.Skip(reason) } - runTest(t, dir, 0, coverDir, repls.Clone(), config, nil, envFilters, sandboxProxyURL) + runTest(t, dir, 0, coverDir, testRepls.Clone(), config, nil, envFilters, sandboxProxyURL) } else { for ind, envset := range expanded { envname := strings.Join(envset, "/") @@ -643,7 +656,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo if reason := subset.skipReason(dir, envset); reason != "" { t.Skip(reason) } - runTest(t, dir, ind, coverDir, repls.Clone(), config, envset, envFilters, sandboxProxyURL) + runTest(t, dir, ind, coverDir, testRepls.Clone(), config, envset, envFilters, sandboxProxyURL) }) } } diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index 59b56a2037c..fa48c4dac59 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,3 +1,5 @@ Cloud = false +CLIVersions.CLI_1_7 = "1.7.0" +CLIVersions.CLI_1_8 = "1.8.0" EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/feature_flags/output.txt b/acceptance/bundle/state/feature_flags/output.txt index 1370eb2bca6..d98bec6ae86 100644 --- a/acceptance/bundle/state/feature_flags/output.txt +++ b/acceptance/bundle/state/feature_flags/output.txt @@ -1,22 +1,401 @@ -=== a state depending on a feature this CLI does not recognize is rejected +=== state version=0 feature=none +>>> errcode [CLI_1_7] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=0 feature=deployment_history +>>> errcode [CLI_1_7] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=0 feature=future_feature +>>> errcode [CLI_1_7] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=1 feature=none +>>> errcode [CLI_1_7] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=1 feature=deployment_history +>>> errcode [CLI_1_7] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=1 feature=future_feature +>>> errcode [CLI_1_7] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=2 feature=none +>>> errcode [CLI_1_7] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=2 feature=deployment_history +>>> errcode [CLI_1_7] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=2 feature=future_feature +>>> errcode [CLI_1_7] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=3 feature=none +>>> errcode [CLI_1_7] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: state version 3 is newer than supported version 2; upgrade the CLI + + +Exit code: 1 + +=== state version=3 feature=deployment_history +>>> errcode [CLI_1_7] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: state version 3 is newer than supported version 2; upgrade the CLI + + +Exit code: 1 + +=== state version=3 feature=future_feature +>>> errcode [CLI_1_7] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: state version 3 is newer than supported version 2; upgrade the CLI + + +Exit code: 1 + +=== state version=0 feature=none +>>> errcode [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=0 feature=deployment_history +>>> errcode [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=0 feature=future_feature +>>> errcode [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=1 feature=none +>>> errcode [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=1 feature=deployment_history +>>> errcode [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=1 feature=future_feature +>>> errcode [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=2 feature=none +>>> errcode [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=2 feature=deployment_history +>>> errcode [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=2 feature=future_feature +>>> errcode [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=3 feature=none +>>> errcode [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== state version=3 feature=deployment_history +>>> errcode [CLI_1_8] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: deployment_history; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information + + +Exit code: 1 + +=== state version=3 feature=future_feature +>>> errcode [CLI_1_8] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information + + +Exit code: 1 + +=== state version=0 feature=none +>>> errcode [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== Deploy persists the migrated version and features +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.my_job +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": null +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +=== state version=0 feature=deployment_history +>>> errcode [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== Deploy persists the migrated version and features +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.my_job +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": { + "deployment_history": {} + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +=== state version=0 feature=future_feature +>>> errcode [CLI] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information + + +Exit code: 1 + +=== state version=1 feature=none +>>> errcode [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== Deploy persists the migrated version and features +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.my_job +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": null +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +=== state version=1 feature=deployment_history +>>> errcode [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== Deploy persists the migrated version and features +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.my_job +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": { + "deployment_history": {} + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +=== state version=1 feature=future_feature +>>> errcode [CLI] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information + + +Exit code: 1 + +=== state version=2 feature=none +>>> errcode [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== Deploy persists the migrated version and features +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.my_job +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": null +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +=== state version=2 feature=deployment_history +>>> errcode [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== Deploy persists the migrated version and features +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.my_job +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": { + "deployment_history": {} + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +=== state version=2 feature=future_feature >>> errcode [CLI] bundle plan Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information Exit code: 1 -=== a state with an empty features map is accepted, and a deploy keeps it at version 3 ->>> [CLI] bundle plan +=== state version=3 feature=none +>>> errcode [CLI] bundle plan create jobs.my_job Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +=== Deploy persists the migrated version and features >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.my_job -Files: 5 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> gron.py .databricks/bundle/default/resources.json -json.state_version = 3; +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": null +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +=== state version=3 feature=deployment_history +>>> errcode [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== Deploy persists the migrated version and features +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.my_job +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": { + "deployment_history": {} + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +=== state version=3 feature=future_feature +>>> errcode [CLI] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information + + +Exit code: 1 diff --git a/acceptance/bundle/state/feature_flags/resources.with_feature.json b/acceptance/bundle/state/feature_flags/resources.with_feature.json deleted file mode 100644 index acc2ec9823d..00000000000 --- a/acceptance/bundle/state/feature_flags/resources.with_feature.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "state_version": 2, - "features": { - "future_feature": {} - }, - "cli_version": "0.0.0-dev", - "lineage": "test-lineage", - "serial": 1, - "state": {} -} diff --git a/acceptance/bundle/state/feature_flags/script b/acceptance/bundle/state/feature_flags/script index c194ee7491f..1b19fe37b70 100644 --- a/acceptance/bundle/state/feature_flags/script +++ b/acceptance/bundle/state/feature_flags/script @@ -1,11 +1,31 @@ mkdir -p .databricks/bundle/default -title "a state depending on a feature this CLI does not recognize is rejected" -cp resources.with_feature.json .databricks/bundle/default/resources.json -trace errcode $CLI bundle plan 2>&1 | contains.py "requires features this CLI does not support: future_feature" "upgrade to the latest CLI version" "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" - -title "a state with an empty features map is accepted, and a deploy keeps it at version 3" -cp resources.empty_features.json .databricks/bundle/default/resources.json -trace $CLI bundle plan | contains.py "Plan:" -trace $CLI bundle deploy -trace gron.py .databricks/bundle/default/resources.json | grep state_version +for cli in "$CLI_1_7" "$CLI_1_8" "$CLI"; do + for version in 0 1 2 3; do + for feature in none deployment_history future_feature; do + title "state version=$version feature=$feature" + # Version 0 represents a legacy state with no state_version field. + jq --argjson version "$version" --arg feature "$feature" ' + .state_version = $version | + if $version == 0 then del(.state_version) else . end | + .features = (if $feature == "none" then {} else {($feature): {}} end) + ' resources.empty_features.json > .databricks/bundle/default/resources.json + + export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false + if [ "$feature" = deployment_history ]; then + export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true + fi + + cp .databricks/bundle/default/resources.json .databricks/input.json + trace errcode "$cli" bundle plan + cmp .databricks/input.json .databricks/bundle/default/resources.json + + if [ "$cli" = "$CLI" ] && [ "$feature" != future_feature ]; then + title "Deploy persists the migrated version and features" + trace "$cli" bundle deploy + trace jq '{state_version, features}' .databricks/bundle/default/resources.json + trace "$cli" bundle destroy --auto-approve + fi + done + done +done diff --git a/acceptance/bundle/state/feature_flags/test.toml b/acceptance/bundle/state/feature_flags/test.toml index 5640bffa4dc..396eec99947 100644 --- a/acceptance/bundle/state/feature_flags/test.toml +++ b/acceptance/bundle/state/feature_flags/test.toml @@ -1,3 +1,7 @@ Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] + +CLIVersions.CLI_1_7 = "1.7.0" +CLIVersions.CLI_1_8 = "1.8.0" diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index 1cf82bbb341..f8c86e047d7 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -101,6 +101,10 @@ type TestConfig struct { CompiledIgnoreObject *ignore.GitIgnore + // Released CLI versions to download for this test, keyed by the environment variable + // that exposes each executable to the script (e.g. CLIVersions.CLI_1_8 = "1.8.0"). + CLIVersions map[string]string + // Environment variables // If the same variable is defined both in Env and EnvMatrix, the one in EnvMatrix takes precedence // regardless of which config file it is defined in. diff --git a/acceptance/internal/materialized_config.go b/acceptance/internal/materialized_config.go index 41fed59eeeb..27297563cf4 100644 --- a/acceptance/internal/materialized_config.go +++ b/acceptance/internal/materialized_config.go @@ -53,6 +53,10 @@ func GenerateMaterializedConfig(config *TestConfig) string { for _, k := range slices.Sorted(maps.Keys(config.CloudEnvs)) { fmt.Fprintf(&buf, "CloudEnvs.%s = %v\n", k, config.CloudEnvs[k]) } + for _, name := range slices.Sorted(maps.Keys(config.CLIVersions)) { + fmt.Fprintf(&buf, "CLIVersions.%s = %s\n", name, tomlQuote(config.CLIVersions[name])) + } + envMatrix := filteredEnvMatrix(config.EnvMatrix, config.EnvMatrixExclude) for _, k := range slices.Sorted(maps.Keys(envMatrix)) { writeTomlStringArray(&buf, "EnvMatrix."+k, envMatrix[k]) diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 72a74027d78..4a7eb844e9c 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -2,7 +2,6 @@ package dstate import ( "encoding/json" - "fmt" "os" "path/filepath" "testing" @@ -183,31 +182,6 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { mustFinalize(t, &recovered) } -func TestMigrateState(t *testing.T) { - for _, version := range []int{0, 1, 2, 3} { - for _, feature := range []string{"", FeatureDeploymentHistory, "future_feature"} { - t.Run(fmt.Sprintf("version=%d/feature=%s", version, feature), func(t *testing.T) { - db := &Database{Header: Header{StateVersion: version}} - if feature != "" { - db.Features = map[string]struct{}{feature: {}} - } - err := migrateState(db) - if feature == "future_feature" { - require.ErrorContains(t, err, "requires features this CLI does not support: future_feature") - assert.Contains(t, err.Error(), featuresDocURL) - assert.Equal(t, version, db.StateVersion) - return - } - require.NoError(t, err) - assert.Equal(t, 3, db.StateVersion) - if feature != "" { - assert.Contains(t, db.Features, feature) - } - }) - } - } -} - func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") From 13a9c012f062d74c0266a2aae04456c4cb2f4ff2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 11:57:57 +0000 Subject: [PATCH 05/17] Keep featureless version 3 state regression test --- bundle/direct/dstate/state_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 4a7eb844e9c..c8d78d150d6 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -182,6 +182,24 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { mustFinalize(t, &recovered) } +func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { + empty := &Database{Header: Header{StateVersion: 3}} + require.NoError(t, migrateState(empty)) + assert.Equal(t, 3, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") + + // Version 3 states with unknown features must still be refused. + withFeature := &Database{Header: Header{ + StateVersion: 3, + Features: map[string]struct{}{"future_feature": {}}, + }} + err := migrateState(withFeature) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires features this CLI does not support") + assert.Contains(t, err.Error(), "future_feature") + assert.Contains(t, err.Error(), "upgrade to the latest CLI version") + assert.Contains(t, err.Error(), featuresDocURL) +} + func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") From d86a3d9bbbcdc9571c62be506fdc10e70b571b64 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 12:03:58 +0000 Subject: [PATCH 06/17] Make CLI and state compatibility permutations explicit --- .../bundle/state/feature_flags/expected.json | 10 + .../bundle/state/feature_flags/out.test.toml | 4 +- .../bundle/state/feature_flags/output.txt | 402 +----------------- acceptance/bundle/state/feature_flags/script | 67 +-- .../bundle/state/feature_flags/test.toml | 4 +- 5 files changed, 58 insertions(+), 429 deletions(-) create mode 100644 acceptance/bundle/state/feature_flags/expected.json diff --git a/acceptance/bundle/state/feature_flags/expected.json b/acceptance/bundle/state/feature_flags/expected.json new file mode 100644 index 00000000000..44310fa40fd --- /dev/null +++ b/acceptance/bundle/state/feature_flags/expected.json @@ -0,0 +1,10 @@ +{ + "old": { + "2": {"none": "accept", "deployment_history": "accept", "future_feature": "accept"}, + "3": {"none": "accept", "deployment_history": "reject", "future_feature": "reject"} + }, + "new": { + "2": {"none": "accept", "deployment_history": "accept", "future_feature": "reject"}, + "3": {"none": "accept", "deployment_history": "accept", "future_feature": "reject"} + } +} diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index fa48c4dac59..56f998c2d1e 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,5 +1,7 @@ Cloud = false -CLIVersions.CLI_1_7 = "1.7.0" CLIVersions.CLI_1_8 = "1.8.0" +EnvMatrix.CLI_VERSION = ["old", "new"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [""] +EnvMatrix.STATE_FEATURE = ["none", "deployment_history", "future_feature"] +EnvMatrix.STATE_VERSION = ["2", "3"] diff --git a/acceptance/bundle/state/feature_flags/output.txt b/acceptance/bundle/state/feature_flags/output.txt index d98bec6ae86..4e4d5191f43 100644 --- a/acceptance/bundle/state/feature_flags/output.txt +++ b/acceptance/bundle/state/feature_flags/output.txt @@ -1,401 +1 @@ - -=== state version=0 feature=none ->>> errcode [CLI_1_7] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=0 feature=deployment_history ->>> errcode [CLI_1_7] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=0 feature=future_feature ->>> errcode [CLI_1_7] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=1 feature=none ->>> errcode [CLI_1_7] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=1 feature=deployment_history ->>> errcode [CLI_1_7] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=1 feature=future_feature ->>> errcode [CLI_1_7] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=2 feature=none ->>> errcode [CLI_1_7] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=2 feature=deployment_history ->>> errcode [CLI_1_7] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=2 feature=future_feature ->>> errcode [CLI_1_7] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=3 feature=none ->>> errcode [CLI_1_7] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: state version 3 is newer than supported version 2; upgrade the CLI - - -Exit code: 1 - -=== state version=3 feature=deployment_history ->>> errcode [CLI_1_7] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: state version 3 is newer than supported version 2; upgrade the CLI - - -Exit code: 1 - -=== state version=3 feature=future_feature ->>> errcode [CLI_1_7] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: state version 3 is newer than supported version 2; upgrade the CLI - - -Exit code: 1 - -=== state version=0 feature=none ->>> errcode [CLI_1_8] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=0 feature=deployment_history ->>> errcode [CLI_1_8] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=0 feature=future_feature ->>> errcode [CLI_1_8] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=1 feature=none ->>> errcode [CLI_1_8] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=1 feature=deployment_history ->>> errcode [CLI_1_8] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=1 feature=future_feature ->>> errcode [CLI_1_8] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=2 feature=none ->>> errcode [CLI_1_8] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=2 feature=deployment_history ->>> errcode [CLI_1_8] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=2 feature=future_feature ->>> errcode [CLI_1_8] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=3 feature=none ->>> errcode [CLI_1_8] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== state version=3 feature=deployment_history ->>> errcode [CLI_1_8] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: deployment_history; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information - - -Exit code: 1 - -=== state version=3 feature=future_feature ->>> errcode [CLI_1_8] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information - - -Exit code: 1 - -=== state version=0 feature=none ->>> errcode [CLI] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== Deploy persists the migrated version and features ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.my_job -Files: 4 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "features": null -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 1 deleted - -=== state version=0 feature=deployment_history ->>> errcode [CLI] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== Deploy persists the migrated version and features ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.my_job -Files: 4 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "features": { - "deployment_history": {} - } -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 1 deleted - -=== state version=0 feature=future_feature ->>> errcode [CLI] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information - - -Exit code: 1 - -=== state version=1 feature=none ->>> errcode [CLI] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== Deploy persists the migrated version and features ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.my_job -Files: 4 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "features": null -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 1 deleted - -=== state version=1 feature=deployment_history ->>> errcode [CLI] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== Deploy persists the migrated version and features ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.my_job -Files: 4 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "features": { - "deployment_history": {} - } -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 1 deleted - -=== state version=1 feature=future_feature ->>> errcode [CLI] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information - - -Exit code: 1 - -=== state version=2 feature=none ->>> errcode [CLI] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== Deploy persists the migrated version and features ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.my_job -Files: 4 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "features": null -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 1 deleted - -=== state version=2 feature=deployment_history ->>> errcode [CLI] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== Deploy persists the migrated version and features ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.my_job -Files: 4 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "features": { - "deployment_history": {} - } -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 1 deleted - -=== state version=2 feature=future_feature ->>> errcode [CLI] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information - - -Exit code: 1 - -=== state version=3 feature=none ->>> errcode [CLI] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== Deploy persists the migrated version and features ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.my_job -Files: 4 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "features": null -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 1 deleted - -=== state version=3 feature=deployment_history ->>> errcode [CLI] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - -=== Deploy persists the migrated version and features ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.my_job -Files: 4 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "features": { - "deployment_history": {} - } -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 1 deleted - -=== state version=3 feature=future_feature ->>> errcode [CLI] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information - - -Exit code: 1 +State compatibility checks passed. diff --git a/acceptance/bundle/state/feature_flags/script b/acceptance/bundle/state/feature_flags/script index 1b19fe37b70..f8823f8f150 100644 --- a/acceptance/bundle/state/feature_flags/script +++ b/acceptance/bundle/state/feature_flags/script @@ -1,31 +1,46 @@ mkdir -p .databricks/bundle/default -for cli in "$CLI_1_7" "$CLI_1_8" "$CLI"; do - for version in 0 1 2 3; do - for feature in none deployment_history future_feature; do - title "state version=$version feature=$feature" - # Version 0 represents a legacy state with no state_version field. - jq --argjson version "$version" --arg feature "$feature" ' - .state_version = $version | - if $version == 0 then del(.state_version) else . end | - .features = (if $feature == "none" then {} else {($feature): {}} end) - ' resources.empty_features.json > .databricks/bundle/default/resources.json +case "$CLI_VERSION" in + old) tested_cli="$CLI_1_8" ;; + new) tested_cli="$CLI" ;; +esac +expected=$(jq -er --arg cli "$CLI_VERSION" --arg version "$STATE_VERSION" --arg feature "$STATE_FEATURE" '.[$cli][$version][$feature]' expected.json) - export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false - if [ "$feature" = deployment_history ]; then - export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true - fi +jq --argjson version "$STATE_VERSION" --arg feature "$STATE_FEATURE" ' + .state_version = $version | + .features = (if $feature == "none" then {} else {($feature): {}} end) +' resources.empty_features.json > .databricks/bundle/default/resources.json +cp .databricks/bundle/default/resources.json .databricks/input.json - cp .databricks/bundle/default/resources.json .databricks/input.json - trace errcode "$cli" bundle plan - cmp .databricks/input.json .databricks/bundle/default/resources.json +export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false +if [ "$STATE_FEATURE" = deployment_history ]; then + export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true +fi - if [ "$cli" = "$CLI" ] && [ "$feature" != future_feature ]; then - title "Deploy persists the migrated version and features" - trace "$cli" bundle deploy - trace jq '{state_version, features}' .databricks/bundle/default/resources.json - trace "$cli" bundle destroy --auto-approve - fi - done - done -done +if "$tested_cli" bundle plan > .databricks/plan.log 2>&1; then + actual=accept +else + actual=reject +fi +if [ "$actual" != "$expected" ]; then + cat .databricks/plan.log + echo "Expected $expected, got $actual for CLI=$CLI_VERSION state=$STATE_VERSION feature=$STATE_FEATURE" >&2 + exit 1 +fi + +if [ "$expected" = reject ]; then + contains.py "requires features this CLI does not support: $STATE_FEATURE" "upgrade to the latest CLI version" "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" < .databricks/plan.log > /dev/null +else + contains.py "Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged" < .databricks/plan.log > /dev/null +fi +cmp .databricks/input.json .databricks/bundle/default/resources.json + +if [ "$CLI_VERSION" = new ] && [ "$expected" = accept ]; then + "$tested_cli" bundle deploy > .databricks/deploy.log 2>&1 || { cat .databricks/deploy.log; exit 1; } + jq -e --arg feature "$STATE_FEATURE" ' + .state_version == 3 and + (.features // {}) == (if $feature == "none" then {} else {($feature): {}} end) + ' .databricks/bundle/default/resources.json > /dev/null +fi + +echo "State compatibility checks passed." diff --git a/acceptance/bundle/state/feature_flags/test.toml b/acceptance/bundle/state/feature_flags/test.toml index 396eec99947..5b8508a5771 100644 --- a/acceptance/bundle/state/feature_flags/test.toml +++ b/acceptance/bundle/state/feature_flags/test.toml @@ -2,6 +2,8 @@ Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [""] +EnvMatrix.CLI_VERSION = ["old", "new"] +EnvMatrix.STATE_VERSION = ["2", "3"] +EnvMatrix.STATE_FEATURE = ["none", "deployment_history", "future_feature"] -CLIVersions.CLI_1_7 = "1.7.0" CLIVersions.CLI_1_8 = "1.8.0" From 5abe0f17f425dfaac6175ce08813b1e2a3a1cee9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 12:10:05 +0000 Subject: [PATCH 07/17] Name compatibility matrix cases by CLI version --- acceptance/bundle/state/feature_flags/expected.json | 4 ++-- acceptance/bundle/state/feature_flags/out.test.toml | 2 +- acceptance/bundle/state/feature_flags/script | 6 +++--- acceptance/bundle/state/feature_flags/test.toml | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/state/feature_flags/expected.json b/acceptance/bundle/state/feature_flags/expected.json index 44310fa40fd..b1c2210291d 100644 --- a/acceptance/bundle/state/feature_flags/expected.json +++ b/acceptance/bundle/state/feature_flags/expected.json @@ -1,9 +1,9 @@ { - "old": { + "1.8.0": { "2": {"none": "accept", "deployment_history": "accept", "future_feature": "accept"}, "3": {"none": "accept", "deployment_history": "reject", "future_feature": "reject"} }, - "new": { + "1.17.0": { "2": {"none": "accept", "deployment_history": "accept", "future_feature": "reject"}, "3": {"none": "accept", "deployment_history": "accept", "future_feature": "reject"} } diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index 56f998c2d1e..6aee1b7f310 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,6 +1,6 @@ Cloud = false CLIVersions.CLI_1_8 = "1.8.0" -EnvMatrix.CLI_VERSION = ["old", "new"] +EnvMatrix.CLI_VERSION = ["1.8.0", "1.17.0"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [""] EnvMatrix.STATE_FEATURE = ["none", "deployment_history", "future_feature"] diff --git a/acceptance/bundle/state/feature_flags/script b/acceptance/bundle/state/feature_flags/script index f8823f8f150..cc8cc8ec1ba 100644 --- a/acceptance/bundle/state/feature_flags/script +++ b/acceptance/bundle/state/feature_flags/script @@ -1,8 +1,8 @@ mkdir -p .databricks/bundle/default case "$CLI_VERSION" in - old) tested_cli="$CLI_1_8" ;; - new) tested_cli="$CLI" ;; + 1.8.0) tested_cli="$CLI_1_8" ;; + 1.17.0) tested_cli="$CLI" ;; esac expected=$(jq -er --arg cli "$CLI_VERSION" --arg version "$STATE_VERSION" --arg feature "$STATE_FEATURE" '.[$cli][$version][$feature]' expected.json) @@ -35,7 +35,7 @@ else fi cmp .databricks/input.json .databricks/bundle/default/resources.json -if [ "$CLI_VERSION" = new ] && [ "$expected" = accept ]; then +if [ "$CLI_VERSION" = 1.17.0 ] && [ "$expected" = accept ]; then "$tested_cli" bundle deploy > .databricks/deploy.log 2>&1 || { cat .databricks/deploy.log; exit 1; } jq -e --arg feature "$STATE_FEATURE" ' .state_version == 3 and diff --git a/acceptance/bundle/state/feature_flags/test.toml b/acceptance/bundle/state/feature_flags/test.toml index 5b8508a5771..d8e7cc582ae 100644 --- a/acceptance/bundle/state/feature_flags/test.toml +++ b/acceptance/bundle/state/feature_flags/test.toml @@ -2,7 +2,8 @@ Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [""] -EnvMatrix.CLI_VERSION = ["old", "new"] +# 1.17.0 uses the source build containing the state-version bump. +EnvMatrix.CLI_VERSION = ["1.8.0", "1.17.0"] EnvMatrix.STATE_VERSION = ["2", "3"] EnvMatrix.STATE_FEATURE = ["none", "deployment_history", "future_feature"] From eda945a71d3b8f4191a0f5ef79250e6273b3464d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 12:11:43 +0000 Subject: [PATCH 08/17] Use main until the state-version release is published --- acceptance/bundle/state/feature_flags/expected.json | 2 +- acceptance/bundle/state/feature_flags/out.test.toml | 2 +- acceptance/bundle/state/feature_flags/script | 4 ++-- acceptance/bundle/state/feature_flags/test.toml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/state/feature_flags/expected.json b/acceptance/bundle/state/feature_flags/expected.json index b1c2210291d..2d32fde8138 100644 --- a/acceptance/bundle/state/feature_flags/expected.json +++ b/acceptance/bundle/state/feature_flags/expected.json @@ -3,7 +3,7 @@ "2": {"none": "accept", "deployment_history": "accept", "future_feature": "accept"}, "3": {"none": "accept", "deployment_history": "reject", "future_feature": "reject"} }, - "1.17.0": { + "main": { "2": {"none": "accept", "deployment_history": "accept", "future_feature": "reject"}, "3": {"none": "accept", "deployment_history": "accept", "future_feature": "reject"} } diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index 6aee1b7f310..41bdf0e4175 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,6 +1,6 @@ Cloud = false CLIVersions.CLI_1_8 = "1.8.0" -EnvMatrix.CLI_VERSION = ["1.8.0", "1.17.0"] +EnvMatrix.CLI_VERSION = ["1.8.0", "main"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [""] EnvMatrix.STATE_FEATURE = ["none", "deployment_history", "future_feature"] diff --git a/acceptance/bundle/state/feature_flags/script b/acceptance/bundle/state/feature_flags/script index cc8cc8ec1ba..5dac28611d9 100644 --- a/acceptance/bundle/state/feature_flags/script +++ b/acceptance/bundle/state/feature_flags/script @@ -2,7 +2,7 @@ mkdir -p .databricks/bundle/default case "$CLI_VERSION" in 1.8.0) tested_cli="$CLI_1_8" ;; - 1.17.0) tested_cli="$CLI" ;; + main) tested_cli="$CLI" ;; esac expected=$(jq -er --arg cli "$CLI_VERSION" --arg version "$STATE_VERSION" --arg feature "$STATE_FEATURE" '.[$cli][$version][$feature]' expected.json) @@ -35,7 +35,7 @@ else fi cmp .databricks/input.json .databricks/bundle/default/resources.json -if [ "$CLI_VERSION" = 1.17.0 ] && [ "$expected" = accept ]; then +if [ "$CLI_VERSION" = main ] && [ "$expected" = accept ]; then "$tested_cli" bundle deploy > .databricks/deploy.log 2>&1 || { cat .databricks/deploy.log; exit 1; } jq -e --arg feature "$STATE_FEATURE" ' .state_version == 3 and diff --git a/acceptance/bundle/state/feature_flags/test.toml b/acceptance/bundle/state/feature_flags/test.toml index d8e7cc582ae..67830f795e3 100644 --- a/acceptance/bundle/state/feature_flags/test.toml +++ b/acceptance/bundle/state/feature_flags/test.toml @@ -2,8 +2,8 @@ Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [""] -# 1.17.0 uses the source build containing the state-version bump. -EnvMatrix.CLI_VERSION = ["1.8.0", "1.17.0"] +# main uses the source build; pin it to this week's release (v1.17.0) once published. +EnvMatrix.CLI_VERSION = ["1.8.0", "main"] EnvMatrix.STATE_VERSION = ["2", "3"] EnvMatrix.STATE_FEATURE = ["none", "deployment_history", "future_feature"] From bdc131d86833ac7f0c8a75a27ddfcbb98591eda1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 12:25:08 +0000 Subject: [PATCH 09/17] Verify released CLI downloads are reused across tests --- acceptance/acceptance_test.go | 3 +- acceptance/download_cli_test.go | 71 +++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 acceptance/download_cli_test.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 72838c04660..1fd1bbba70f 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -1521,7 +1521,8 @@ func resolveLatestVersion(t *testing.T, buildDir string) string { } // DownloadCLI downloads a released CLI binary archive for the given version, -// extracts the executable, and returns its path. +// extracts the executable, and returns its path. Executables are cached by version +// in the shared build directory and reused across tests and subsequent suite runs. func DownloadCLI(t *testing.T, buildDir, version string) string { // Prepare target directory for this version versionDir := filepath.Join(buildDir, version) diff --git a/acceptance/download_cli_test.go b/acceptance/download_cli_test.go new file mode 100644 index 00000000000..3a3f2bcfe0e --- /dev/null +++ b/acceptance/download_cli_test.go @@ -0,0 +1,71 @@ +package acceptance_test + +import ( + "archive/zip" + "bytes" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type downloadRoundTripper func(*http.Request) (*http.Response, error) + +func (f downloadRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + +func TestDownloadCLIReusesCachedVersions(t *testing.T) { + execName := "databricks" + if runtime.GOOS == "windows" { + execName += ".exe" + } + + var archive bytes.Buffer + writer := zip.NewWriter(&archive) + entry, err := writer.Create(execName) + require.NoError(t, err) + _, err = io.WriteString(entry, "cached CLI") + require.NoError(t, err) + require.NoError(t, writer.Close()) + + downloads := 0 + originalClient := http.DefaultClient + t.Cleanup(func() { http.DefaultClient = originalClient }) + http.DefaultClient = &http.Client{Transport: downloadRoundTripper(func(r *http.Request) (*http.Response, error) { + downloads++ + assert.Equal(t, "github.com", r.URL.Host) + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader(archive.Bytes())), + Header: make(http.Header), + }, nil + })} + + root := t.TempDir() + for _, tc := range []struct { + name string + version string + downloads int + }{ + {"first", "1.8.0", 1}, + {"same_version", "1.8.0", 1}, + {"different_version", "1.9.0", 2}, + {"first_version_again", "1.8.0", 2}, + } { + t.Run(tc.name, func(t *testing.T) { + buildDir := getBuildDir(t, root, runtime.GOOS, runtime.GOARCH) + path := DownloadCLI(t, buildDir, tc.version) + assert.Equal(t, filepath.Join(buildDir, tc.version, execName), path) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "cached CLI", string(data)) + assert.Equal(t, tc.downloads, downloads) + }) + } +} From 40f921102679e81d507e971f79060ee173c32e40 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 12:38:32 +0000 Subject: [PATCH 10/17] Simplify state compatibility tests into accept and reject groups --- .../{ => accept_new}/databricks.yml | 0 .../feature_flags/accept_new/out.test.toml | 9 +++ .../state/feature_flags/accept_new/output.txt | 14 ++++ .../resources.v2.deployment_history.json | 10 +++ .../resources.v2.json} | 0 .../resources.v3.deployment_history.json | 10 +++ .../accept_new/resources.v3.json | 8 +++ .../state/feature_flags/accept_new/script | 13 ++++ .../state/feature_flags/accept_new/test.toml | 1 + .../feature_flags/accept_old/databricks.yml | 7 ++ .../feature_flags/accept_old/out.test.toml | 10 +++ .../state/feature_flags/accept_old/output.txt | 5 ++ .../resources.v2.deployment_history.json | 10 +++ .../resources.v2.future_feature.json | 10 +++ .../accept_old/resources.v2.json | 8 +++ .../accept_old/resources.v3.json | 8 +++ .../state/feature_flags/accept_old/script | 5 ++ .../state/feature_flags/accept_old/test.toml | 4 ++ .../bundle/state/feature_flags/expected.json | 10 --- .../bundle/state/feature_flags/out.test.toml | 7 -- .../bundle/state/feature_flags/output.txt | 1 - .../feature_flags/reject_new/databricks.yml | 7 ++ .../feature_flags/reject_new/out.test.toml | 4 ++ .../state/feature_flags/reject_new/output.txt | 4 ++ .../resources.v2.future_feature.json | 10 +++ .../resources.v3.future_feature.json | 10 +++ .../state/feature_flags/reject_new/script | 5 ++ .../state/feature_flags/reject_new/test.toml | 1 + .../feature_flags/reject_old/databricks.yml | 7 ++ .../feature_flags/reject_old/out.test.toml | 5 ++ .../state/feature_flags/reject_old/output.txt | 4 ++ .../resources.v3.deployment_history.json | 10 +++ .../resources.v3.future_feature.json | 10 +++ .../state/feature_flags/reject_old/script | 5 ++ .../state/feature_flags/reject_old/test.toml | 3 + acceptance/bundle/state/feature_flags/script | 46 ------------ .../bundle/state/feature_flags/test.toml | 6 +- acceptance/download_cli_test.go | 71 ------------------- 38 files changed, 218 insertions(+), 140 deletions(-) rename acceptance/bundle/state/feature_flags/{ => accept_new}/databricks.yml (100%) create mode 100644 acceptance/bundle/state/feature_flags/accept_new/out.test.toml create mode 100644 acceptance/bundle/state/feature_flags/accept_new/output.txt create mode 100644 acceptance/bundle/state/feature_flags/accept_new/resources.v2.deployment_history.json rename acceptance/bundle/state/feature_flags/{resources.empty_features.json => accept_new/resources.v2.json} (100%) create mode 100644 acceptance/bundle/state/feature_flags/accept_new/resources.v3.deployment_history.json create mode 100644 acceptance/bundle/state/feature_flags/accept_new/resources.v3.json create mode 100644 acceptance/bundle/state/feature_flags/accept_new/script create mode 100644 acceptance/bundle/state/feature_flags/accept_new/test.toml create mode 100644 acceptance/bundle/state/feature_flags/accept_old/databricks.yml create mode 100644 acceptance/bundle/state/feature_flags/accept_old/out.test.toml create mode 100644 acceptance/bundle/state/feature_flags/accept_old/output.txt create mode 100644 acceptance/bundle/state/feature_flags/accept_old/resources.v2.deployment_history.json create mode 100644 acceptance/bundle/state/feature_flags/accept_old/resources.v2.future_feature.json create mode 100644 acceptance/bundle/state/feature_flags/accept_old/resources.v2.json create mode 100644 acceptance/bundle/state/feature_flags/accept_old/resources.v3.json create mode 100644 acceptance/bundle/state/feature_flags/accept_old/script create mode 100644 acceptance/bundle/state/feature_flags/accept_old/test.toml delete mode 100644 acceptance/bundle/state/feature_flags/expected.json delete mode 100644 acceptance/bundle/state/feature_flags/out.test.toml delete mode 100644 acceptance/bundle/state/feature_flags/output.txt create mode 100644 acceptance/bundle/state/feature_flags/reject_new/databricks.yml create mode 100644 acceptance/bundle/state/feature_flags/reject_new/out.test.toml create mode 100644 acceptance/bundle/state/feature_flags/reject_new/output.txt create mode 100644 acceptance/bundle/state/feature_flags/reject_new/resources.v2.future_feature.json create mode 100644 acceptance/bundle/state/feature_flags/reject_new/resources.v3.future_feature.json create mode 100644 acceptance/bundle/state/feature_flags/reject_new/script create mode 100644 acceptance/bundle/state/feature_flags/reject_new/test.toml create mode 100644 acceptance/bundle/state/feature_flags/reject_old/databricks.yml create mode 100644 acceptance/bundle/state/feature_flags/reject_old/out.test.toml create mode 100644 acceptance/bundle/state/feature_flags/reject_old/output.txt create mode 100644 acceptance/bundle/state/feature_flags/reject_old/resources.v3.deployment_history.json create mode 100644 acceptance/bundle/state/feature_flags/reject_old/resources.v3.future_feature.json create mode 100644 acceptance/bundle/state/feature_flags/reject_old/script create mode 100644 acceptance/bundle/state/feature_flags/reject_old/test.toml delete mode 100644 acceptance/bundle/state/feature_flags/script delete mode 100644 acceptance/download_cli_test.go diff --git a/acceptance/bundle/state/feature_flags/databricks.yml b/acceptance/bundle/state/feature_flags/accept_new/databricks.yml similarity index 100% rename from acceptance/bundle/state/feature_flags/databricks.yml rename to acceptance/bundle/state/feature_flags/accept_new/databricks.yml diff --git a/acceptance/bundle/state/feature_flags/accept_new/out.test.toml b/acceptance/bundle/state/feature_flags/accept_new/out.test.toml new file mode 100644 index 00000000000..23f30ff55f8 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/out.test.toml @@ -0,0 +1,9 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] +EnvMatrix.STATE_FILE = [ + "resources.v2.json", + "resources.v3.json", + "resources.v2.deployment_history.json", + "resources.v3.deployment_history.json" +] diff --git a/acceptance/bundle/state/feature_flags/accept_new/output.txt b/acceptance/bundle/state/feature_flags/accept_new/output.txt new file mode 100644 index 00000000000..e6ae71ea2fd --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/output.txt @@ -0,0 +1,14 @@ + +>>> [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.my_job +Files: 7 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq .state_version .databricks/bundle/default/resources.json +3 diff --git a/acceptance/bundle/state/feature_flags/accept_new/resources.v2.deployment_history.json b/acceptance/bundle/state/feature_flags/accept_new/resources.v2.deployment_history.json new file mode 100644 index 00000000000..294508f75e7 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/resources.v2.deployment_history.json @@ -0,0 +1,10 @@ +{ + "state_version": 2, + "features": { + "deployment_history": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/resources.empty_features.json b/acceptance/bundle/state/feature_flags/accept_new/resources.v2.json similarity index 100% rename from acceptance/bundle/state/feature_flags/resources.empty_features.json rename to acceptance/bundle/state/feature_flags/accept_new/resources.v2.json diff --git a/acceptance/bundle/state/feature_flags/accept_new/resources.v3.deployment_history.json b/acceptance/bundle/state/feature_flags/accept_new/resources.v3.deployment_history.json new file mode 100644 index 00000000000..1e47559efd8 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/resources.v3.deployment_history.json @@ -0,0 +1,10 @@ +{ + "state_version": 3, + "features": { + "deployment_history": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/accept_new/resources.v3.json b/acceptance/bundle/state/feature_flags/accept_new/resources.v3.json new file mode 100644 index 00000000000..b20c97aa074 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/resources.v3.json @@ -0,0 +1,8 @@ +{ + "state_version": 3, + "features": {}, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/accept_new/script b/acceptance/bundle/state/feature_flags/accept_new/script new file mode 100644 index 00000000000..c9c5c49375a --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/script @@ -0,0 +1,13 @@ +mkdir -p .databricks/bundle/default +cp "$STATE_FILE" .databricks/bundle/default/resources.json + +DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=$(jq '.features | has("deployment_history")' "$STATE_FILE") +export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY + +trace $CLI bundle plan | contains.py "Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged" +cmp "$STATE_FILE" .databricks/bundle/default/resources.json + +trace $CLI bundle deploy +trace jq '.state_version' .databricks/bundle/default/resources.json +# Deployment upgrades the version and preserves the input's feature flags. +diff <(jq -S '.features' "$STATE_FILE") <(jq -S '.features // {}' .databricks/bundle/default/resources.json) diff --git a/acceptance/bundle/state/feature_flags/accept_new/test.toml b/acceptance/bundle/state/feature_flags/accept_new/test.toml new file mode 100644 index 00000000000..55933de2377 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/test.toml @@ -0,0 +1 @@ +EnvMatrix.STATE_FILE = ["resources.v2.json", "resources.v3.json", "resources.v2.deployment_history.json", "resources.v3.deployment_history.json"] diff --git a/acceptance/bundle/state/feature_flags/accept_old/databricks.yml b/acceptance/bundle/state/feature_flags/accept_old/databricks.yml new file mode 100644 index 00000000000..5134dbcc12c --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + my_job: + name: "my job" diff --git a/acceptance/bundle/state/feature_flags/accept_old/out.test.toml b/acceptance/bundle/state/feature_flags/accept_old/out.test.toml new file mode 100644 index 00000000000..17c3f31eb99 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/out.test.toml @@ -0,0 +1,10 @@ +Cloud = false +CLIVersions.CLI_1_8 = "1.8.0" +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] +EnvMatrix.STATE_FILE = [ + "resources.v2.json", + "resources.v2.future_feature.json", + "resources.v2.deployment_history.json", + "resources.v3.json" +] diff --git a/acceptance/bundle/state/feature_flags/accept_old/output.txt b/acceptance/bundle/state/feature_flags/accept_old/output.txt new file mode 100644 index 00000000000..f0711439a0b --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/state/feature_flags/accept_old/resources.v2.deployment_history.json b/acceptance/bundle/state/feature_flags/accept_old/resources.v2.deployment_history.json new file mode 100644 index 00000000000..294508f75e7 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/resources.v2.deployment_history.json @@ -0,0 +1,10 @@ +{ + "state_version": 2, + "features": { + "deployment_history": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/accept_old/resources.v2.future_feature.json b/acceptance/bundle/state/feature_flags/accept_old/resources.v2.future_feature.json new file mode 100644 index 00000000000..acc2ec9823d --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/resources.v2.future_feature.json @@ -0,0 +1,10 @@ +{ + "state_version": 2, + "features": { + "future_feature": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/accept_old/resources.v2.json b/acceptance/bundle/state/feature_flags/accept_old/resources.v2.json new file mode 100644 index 00000000000..876fdaf19ce --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/resources.v2.json @@ -0,0 +1,8 @@ +{ + "state_version": 2, + "features": {}, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/accept_old/resources.v3.json b/acceptance/bundle/state/feature_flags/accept_old/resources.v3.json new file mode 100644 index 00000000000..b20c97aa074 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/resources.v3.json @@ -0,0 +1,8 @@ +{ + "state_version": 3, + "features": {}, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/accept_old/script b/acceptance/bundle/state/feature_flags/accept_old/script new file mode 100644 index 00000000000..47e209dc965 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/script @@ -0,0 +1,5 @@ +mkdir -p .databricks/bundle/default +cp "$STATE_FILE" .databricks/bundle/default/resources.json + +trace $CLI_1_8 bundle plan | contains.py "Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged" +cmp "$STATE_FILE" .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/accept_old/test.toml b/acceptance/bundle/state/feature_flags/accept_old/test.toml new file mode 100644 index 00000000000..1731edf924f --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/test.toml @@ -0,0 +1,4 @@ +# CLI v1.8.0 ignores feature flags in version 2 state. +EnvMatrix.STATE_FILE = ["resources.v2.json", "resources.v2.future_feature.json", "resources.v2.deployment_history.json", "resources.v3.json"] + +CLIVersions.CLI_1_8 = "1.8.0" diff --git a/acceptance/bundle/state/feature_flags/expected.json b/acceptance/bundle/state/feature_flags/expected.json deleted file mode 100644 index 2d32fde8138..00000000000 --- a/acceptance/bundle/state/feature_flags/expected.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "1.8.0": { - "2": {"none": "accept", "deployment_history": "accept", "future_feature": "accept"}, - "3": {"none": "accept", "deployment_history": "reject", "future_feature": "reject"} - }, - "main": { - "2": {"none": "accept", "deployment_history": "accept", "future_feature": "reject"}, - "3": {"none": "accept", "deployment_history": "accept", "future_feature": "reject"} - } -} diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml deleted file mode 100644 index 41bdf0e4175..00000000000 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ /dev/null @@ -1,7 +0,0 @@ -Cloud = false -CLIVersions.CLI_1_8 = "1.8.0" -EnvMatrix.CLI_VERSION = ["1.8.0", "main"] -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] -EnvMatrix.STATE_FEATURE = ["none", "deployment_history", "future_feature"] -EnvMatrix.STATE_VERSION = ["2", "3"] diff --git a/acceptance/bundle/state/feature_flags/output.txt b/acceptance/bundle/state/feature_flags/output.txt deleted file mode 100644 index 4e4d5191f43..00000000000 --- a/acceptance/bundle/state/feature_flags/output.txt +++ /dev/null @@ -1 +0,0 @@ -State compatibility checks passed. diff --git a/acceptance/bundle/state/feature_flags/reject_new/databricks.yml b/acceptance/bundle/state/feature_flags/reject_new/databricks.yml new file mode 100644 index 00000000000..5134dbcc12c --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + my_job: + name: "my job" diff --git a/acceptance/bundle/state/feature_flags/reject_new/out.test.toml b/acceptance/bundle/state/feature_flags/reject_new/out.test.toml new file mode 100644 index 00000000000..5684d9e1005 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/out.test.toml @@ -0,0 +1,4 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] +EnvMatrix.STATE_FILE = ["resources.v2.future_feature.json", "resources.v3.future_feature.json"] diff --git a/acceptance/bundle/state/feature_flags/reject_new/output.txt b/acceptance/bundle/state/feature_flags/reject_new/output.txt new file mode 100644 index 00000000000..e21cc126c86 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/output.txt @@ -0,0 +1,4 @@ + +>>> musterr [CLI] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information + diff --git a/acceptance/bundle/state/feature_flags/reject_new/resources.v2.future_feature.json b/acceptance/bundle/state/feature_flags/reject_new/resources.v2.future_feature.json new file mode 100644 index 00000000000..acc2ec9823d --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/resources.v2.future_feature.json @@ -0,0 +1,10 @@ +{ + "state_version": 2, + "features": { + "future_feature": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/reject_new/resources.v3.future_feature.json b/acceptance/bundle/state/feature_flags/reject_new/resources.v3.future_feature.json new file mode 100644 index 00000000000..b844b098a70 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/resources.v3.future_feature.json @@ -0,0 +1,10 @@ +{ + "state_version": 3, + "features": { + "future_feature": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/reject_new/script b/acceptance/bundle/state/feature_flags/reject_new/script new file mode 100644 index 00000000000..81b010876bf --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/script @@ -0,0 +1,5 @@ +mkdir -p .databricks/bundle/default +cp "$STATE_FILE" .databricks/bundle/default/resources.json + +trace musterr $CLI bundle plan 2>&1 | contains.py "requires features this CLI does not support: future_feature" +cmp "$STATE_FILE" .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/reject_new/test.toml b/acceptance/bundle/state/feature_flags/reject_new/test.toml new file mode 100644 index 00000000000..84e88afe36c --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/test.toml @@ -0,0 +1 @@ +EnvMatrix.STATE_FILE = ["resources.v2.future_feature.json", "resources.v3.future_feature.json"] diff --git a/acceptance/bundle/state/feature_flags/reject_old/databricks.yml b/acceptance/bundle/state/feature_flags/reject_old/databricks.yml new file mode 100644 index 00000000000..5134dbcc12c --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + my_job: + name: "my job" diff --git a/acceptance/bundle/state/feature_flags/reject_old/out.test.toml b/acceptance/bundle/state/feature_flags/reject_old/out.test.toml new file mode 100644 index 00000000000..04719895b0c --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/out.test.toml @@ -0,0 +1,5 @@ +Cloud = false +CLIVersions.CLI_1_8 = "1.8.0" +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] +EnvMatrix.FEATURE = ["future_feature", "deployment_history"] diff --git a/acceptance/bundle/state/feature_flags/reject_old/output.txt b/acceptance/bundle/state/feature_flags/reject_old/output.txt new file mode 100644 index 00000000000..56ec6e6f859 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/output.txt @@ -0,0 +1,4 @@ + +>>> musterr [CLI_1_8] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: [FEATURE]; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information + diff --git a/acceptance/bundle/state/feature_flags/reject_old/resources.v3.deployment_history.json b/acceptance/bundle/state/feature_flags/reject_old/resources.v3.deployment_history.json new file mode 100644 index 00000000000..1e47559efd8 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/resources.v3.deployment_history.json @@ -0,0 +1,10 @@ +{ + "state_version": 3, + "features": { + "deployment_history": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/reject_old/resources.v3.future_feature.json b/acceptance/bundle/state/feature_flags/reject_old/resources.v3.future_feature.json new file mode 100644 index 00000000000..b844b098a70 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/resources.v3.future_feature.json @@ -0,0 +1,10 @@ +{ + "state_version": 3, + "features": { + "future_feature": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/reject_old/script b/acceptance/bundle/state/feature_flags/reject_old/script new file mode 100644 index 00000000000..646990960b8 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/script @@ -0,0 +1,5 @@ +mkdir -p .databricks/bundle/default +cp "resources.v3.$FEATURE.json" .databricks/bundle/default/resources.json + +trace musterr $CLI_1_8 bundle plan 2>&1 | contains.py "requires features this CLI does not support: $FEATURE" +cmp "resources.v3.$FEATURE.json" .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/reject_old/test.toml b/acceptance/bundle/state/feature_flags/reject_old/test.toml new file mode 100644 index 00000000000..0e6af2b7696 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/test.toml @@ -0,0 +1,3 @@ +EnvMatrix.FEATURE = ["future_feature", "deployment_history"] + +CLIVersions.CLI_1_8 = "1.8.0" diff --git a/acceptance/bundle/state/feature_flags/script b/acceptance/bundle/state/feature_flags/script deleted file mode 100644 index 5dac28611d9..00000000000 --- a/acceptance/bundle/state/feature_flags/script +++ /dev/null @@ -1,46 +0,0 @@ -mkdir -p .databricks/bundle/default - -case "$CLI_VERSION" in - 1.8.0) tested_cli="$CLI_1_8" ;; - main) tested_cli="$CLI" ;; -esac -expected=$(jq -er --arg cli "$CLI_VERSION" --arg version "$STATE_VERSION" --arg feature "$STATE_FEATURE" '.[$cli][$version][$feature]' expected.json) - -jq --argjson version "$STATE_VERSION" --arg feature "$STATE_FEATURE" ' - .state_version = $version | - .features = (if $feature == "none" then {} else {($feature): {}} end) -' resources.empty_features.json > .databricks/bundle/default/resources.json -cp .databricks/bundle/default/resources.json .databricks/input.json - -export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false -if [ "$STATE_FEATURE" = deployment_history ]; then - export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true -fi - -if "$tested_cli" bundle plan > .databricks/plan.log 2>&1; then - actual=accept -else - actual=reject -fi -if [ "$actual" != "$expected" ]; then - cat .databricks/plan.log - echo "Expected $expected, got $actual for CLI=$CLI_VERSION state=$STATE_VERSION feature=$STATE_FEATURE" >&2 - exit 1 -fi - -if [ "$expected" = reject ]; then - contains.py "requires features this CLI does not support: $STATE_FEATURE" "upgrade to the latest CLI version" "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" < .databricks/plan.log > /dev/null -else - contains.py "Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged" < .databricks/plan.log > /dev/null -fi -cmp .databricks/input.json .databricks/bundle/default/resources.json - -if [ "$CLI_VERSION" = main ] && [ "$expected" = accept ]; then - "$tested_cli" bundle deploy > .databricks/deploy.log 2>&1 || { cat .databricks/deploy.log; exit 1; } - jq -e --arg feature "$STATE_FEATURE" ' - .state_version == 3 and - (.features // {}) == (if $feature == "none" then {} else {($feature): {}} end) - ' .databricks/bundle/default/resources.json > /dev/null -fi - -echo "State compatibility checks passed." diff --git a/acceptance/bundle/state/feature_flags/test.toml b/acceptance/bundle/state/feature_flags/test.toml index 67830f795e3..e355ab2feb2 100644 --- a/acceptance/bundle/state/feature_flags/test.toml +++ b/acceptance/bundle/state/feature_flags/test.toml @@ -2,9 +2,5 @@ Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [""] -# main uses the source build; pin it to this week's release (v1.17.0) once published. -EnvMatrix.CLI_VERSION = ["1.8.0", "main"] -EnvMatrix.STATE_VERSION = ["2", "3"] -EnvMatrix.STATE_FEATURE = ["none", "deployment_history", "future_feature"] -CLIVersions.CLI_1_8 = "1.8.0" +# New-CLI tests use the source build; pin them to this week's release (v1.17.0) once published. diff --git a/acceptance/download_cli_test.go b/acceptance/download_cli_test.go deleted file mode 100644 index 3a3f2bcfe0e..00000000000 --- a/acceptance/download_cli_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package acceptance_test - -import ( - "archive/zip" - "bytes" - "io" - "net/http" - "os" - "path/filepath" - "runtime" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type downloadRoundTripper func(*http.Request) (*http.Response, error) - -func (f downloadRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { - return f(r) -} - -func TestDownloadCLIReusesCachedVersions(t *testing.T) { - execName := "databricks" - if runtime.GOOS == "windows" { - execName += ".exe" - } - - var archive bytes.Buffer - writer := zip.NewWriter(&archive) - entry, err := writer.Create(execName) - require.NoError(t, err) - _, err = io.WriteString(entry, "cached CLI") - require.NoError(t, err) - require.NoError(t, writer.Close()) - - downloads := 0 - originalClient := http.DefaultClient - t.Cleanup(func() { http.DefaultClient = originalClient }) - http.DefaultClient = &http.Client{Transport: downloadRoundTripper(func(r *http.Request) (*http.Response, error) { - downloads++ - assert.Equal(t, "github.com", r.URL.Host) - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader(archive.Bytes())), - Header: make(http.Header), - }, nil - })} - - root := t.TempDir() - for _, tc := range []struct { - name string - version string - downloads int - }{ - {"first", "1.8.0", 1}, - {"same_version", "1.8.0", 1}, - {"different_version", "1.9.0", 2}, - {"first_version_again", "1.8.0", 2}, - } { - t.Run(tc.name, func(t *testing.T) { - buildDir := getBuildDir(t, root, runtime.GOOS, runtime.GOARCH) - path := DownloadCLI(t, buildDir, tc.version) - assert.Equal(t, filepath.Join(buildDir, tc.version, execName), path) - data, err := os.ReadFile(path) - require.NoError(t, err) - assert.Equal(t, "cached CLI", string(data)) - assert.Equal(t, tc.downloads, downloads) - }) - } -} From e025b6ffde582b6165ef41c903844887f872cb12 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 12:39:55 +0000 Subject: [PATCH 11/17] Show state-preservation assertions in acceptance output --- acceptance/bundle/state/feature_flags/accept_new/output.txt | 2 ++ acceptance/bundle/state/feature_flags/accept_new/script | 2 +- acceptance/bundle/state/feature_flags/accept_old/output.txt | 2 ++ acceptance/bundle/state/feature_flags/accept_old/script | 2 +- acceptance/bundle/state/feature_flags/reject_new/output.txt | 2 ++ acceptance/bundle/state/feature_flags/reject_new/script | 2 +- acceptance/bundle/state/feature_flags/reject_old/output.txt | 2 ++ acceptance/bundle/state/feature_flags/reject_old/script | 2 +- 8 files changed, 12 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/state/feature_flags/accept_new/output.txt b/acceptance/bundle/state/feature_flags/accept_new/output.txt index e6ae71ea2fd..9ed3a7f5ae7 100644 --- a/acceptance/bundle/state/feature_flags/accept_new/output.txt +++ b/acceptance/bundle/state/feature_flags/accept_new/output.txt @@ -4,6 +4,8 @@ create jobs.my_job Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +>>> cmp [STATE_FILE] .databricks/bundle/default/resources.json + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.my_job diff --git a/acceptance/bundle/state/feature_flags/accept_new/script b/acceptance/bundle/state/feature_flags/accept_new/script index c9c5c49375a..4928c4c2c6f 100644 --- a/acceptance/bundle/state/feature_flags/accept_new/script +++ b/acceptance/bundle/state/feature_flags/accept_new/script @@ -5,7 +5,7 @@ DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=$(jq '.features | has("deployment_history") export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY trace $CLI bundle plan | contains.py "Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged" -cmp "$STATE_FILE" .databricks/bundle/default/resources.json +trace cmp "$STATE_FILE" .databricks/bundle/default/resources.json trace $CLI bundle deploy trace jq '.state_version' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/accept_old/output.txt b/acceptance/bundle/state/feature_flags/accept_old/output.txt index f0711439a0b..5ac51c952d0 100644 --- a/acceptance/bundle/state/feature_flags/accept_old/output.txt +++ b/acceptance/bundle/state/feature_flags/accept_old/output.txt @@ -3,3 +3,5 @@ create jobs.my_job Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> cmp [STATE_FILE] .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/accept_old/script b/acceptance/bundle/state/feature_flags/accept_old/script index 47e209dc965..5b753c4212d 100644 --- a/acceptance/bundle/state/feature_flags/accept_old/script +++ b/acceptance/bundle/state/feature_flags/accept_old/script @@ -2,4 +2,4 @@ mkdir -p .databricks/bundle/default cp "$STATE_FILE" .databricks/bundle/default/resources.json trace $CLI_1_8 bundle plan | contains.py "Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged" -cmp "$STATE_FILE" .databricks/bundle/default/resources.json +trace cmp "$STATE_FILE" .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/reject_new/output.txt b/acceptance/bundle/state/feature_flags/reject_new/output.txt index e21cc126c86..35eb79cf7e7 100644 --- a/acceptance/bundle/state/feature_flags/reject_new/output.txt +++ b/acceptance/bundle/state/feature_flags/reject_new/output.txt @@ -2,3 +2,5 @@ >>> musterr [CLI] bundle plan Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information + +>>> cmp [STATE_FILE] .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/reject_new/script b/acceptance/bundle/state/feature_flags/reject_new/script index 81b010876bf..979cebdcf5a 100644 --- a/acceptance/bundle/state/feature_flags/reject_new/script +++ b/acceptance/bundle/state/feature_flags/reject_new/script @@ -2,4 +2,4 @@ mkdir -p .databricks/bundle/default cp "$STATE_FILE" .databricks/bundle/default/resources.json trace musterr $CLI bundle plan 2>&1 | contains.py "requires features this CLI does not support: future_feature" -cmp "$STATE_FILE" .databricks/bundle/default/resources.json +trace cmp "$STATE_FILE" .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/reject_old/output.txt b/acceptance/bundle/state/feature_flags/reject_old/output.txt index 56ec6e6f859..f4e77bc90c5 100644 --- a/acceptance/bundle/state/feature_flags/reject_old/output.txt +++ b/acceptance/bundle/state/feature_flags/reject_old/output.txt @@ -2,3 +2,5 @@ >>> musterr [CLI_1_8] bundle plan Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: [FEATURE]; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information + +>>> cmp resources.v3.[FEATURE].json .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/reject_old/script b/acceptance/bundle/state/feature_flags/reject_old/script index 646990960b8..f3b49e0c453 100644 --- a/acceptance/bundle/state/feature_flags/reject_old/script +++ b/acceptance/bundle/state/feature_flags/reject_old/script @@ -2,4 +2,4 @@ mkdir -p .databricks/bundle/default cp "resources.v3.$FEATURE.json" .databricks/bundle/default/resources.json trace musterr $CLI_1_8 bundle plan 2>&1 | contains.py "requires features this CLI does not support: $FEATURE" -cmp "resources.v3.$FEATURE.json" .databricks/bundle/default/resources.json +trace cmp "resources.v3.$FEATURE.json" .databricks/bundle/default/resources.json From 41cfb4c636c60162784a14ec4da7446802a1c896 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 12:48:56 +0000 Subject: [PATCH 12/17] Expose cached CLI v1.8.0 by default in acceptance tests --- acceptance/acceptance_test.go | 24 ++++++------------- .../feature_flags/accept_old/out.test.toml | 1 - .../state/feature_flags/accept_old/test.toml | 2 -- .../feature_flags/reject_old/out.test.toml | 1 - .../state/feature_flags/reject_old/test.toml | 2 -- acceptance/internal/config.go | 4 ---- acceptance/internal/materialized_config.go | 4 ---- 7 files changed, 7 insertions(+), 31 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 1fd1bbba70f..547cccf3eab 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -399,6 +399,10 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo cli293Path := DownloadCLI(t, buildDir, "0.293.0") t.Setenv("CLI_293", cli293Path) repls.SetPath(cli293Path, "[CLI_293]") + + cli18Path := DownloadCLI(t, buildDir, "1.8.0") + t.Setenv("CLI_1_8", cli18Path) + repls.SetPath(cli18Path, "[CLI_1_8]") } paths := []string{ @@ -585,19 +589,6 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo t.Skip(skipReason) } - var testRepls testdiff.ReplacementsContext - // Download before t.Parallel so tests sharing a release cannot race to extract it. - for _, name := range slices.Sorted(maps.Keys(config.CLIVersions)) { - path := DownloadCLI(t, buildDir, config.CLIVersions[name]) - if config.Env == nil { - config.Env = make(map[string]string) - } - config.Env[name] = path - testRepls.SetPath(path, "["+name+"]") - } - // Replace executable paths before their parent build directory. - testRepls.Repls = append(testRepls.Repls, repls.Repls...) - runParallel := !inprocessMode if benchmarkMode && strings.Contains(dir, "benchmark") { runParallel = false @@ -638,7 +629,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo if reason := subset.skipReason(dir, nil); reason != "" { t.Skip(reason) } - runTest(t, dir, 0, coverDir, testRepls.Clone(), config, nil, envFilters, sandboxProxyURL) + runTest(t, dir, 0, coverDir, repls.Clone(), config, nil, envFilters, sandboxProxyURL) } else { for ind, envset := range expanded { envname := strings.Join(envset, "/") @@ -656,7 +647,7 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo if reason := subset.skipReason(dir, envset); reason != "" { t.Skip(reason) } - runTest(t, dir, ind, coverDir, testRepls.Clone(), config, envset, envFilters, sandboxProxyURL) + runTest(t, dir, ind, coverDir, repls.Clone(), config, envset, envFilters, sandboxProxyURL) }) } } @@ -1521,8 +1512,7 @@ func resolveLatestVersion(t *testing.T, buildDir string) string { } // DownloadCLI downloads a released CLI binary archive for the given version, -// extracts the executable, and returns its path. Executables are cached by version -// in the shared build directory and reused across tests and subsequent suite runs. +// extracts the executable, and returns its path. func DownloadCLI(t *testing.T, buildDir, version string) string { // Prepare target directory for this version versionDir := filepath.Join(buildDir, version) diff --git a/acceptance/bundle/state/feature_flags/accept_old/out.test.toml b/acceptance/bundle/state/feature_flags/accept_old/out.test.toml index 17c3f31eb99..104d04f213a 100644 --- a/acceptance/bundle/state/feature_flags/accept_old/out.test.toml +++ b/acceptance/bundle/state/feature_flags/accept_old/out.test.toml @@ -1,5 +1,4 @@ Cloud = false -CLIVersions.CLI_1_8 = "1.8.0" EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [""] EnvMatrix.STATE_FILE = [ diff --git a/acceptance/bundle/state/feature_flags/accept_old/test.toml b/acceptance/bundle/state/feature_flags/accept_old/test.toml index 1731edf924f..f7d4af44c7b 100644 --- a/acceptance/bundle/state/feature_flags/accept_old/test.toml +++ b/acceptance/bundle/state/feature_flags/accept_old/test.toml @@ -1,4 +1,2 @@ # CLI v1.8.0 ignores feature flags in version 2 state. EnvMatrix.STATE_FILE = ["resources.v2.json", "resources.v2.future_feature.json", "resources.v2.deployment_history.json", "resources.v3.json"] - -CLIVersions.CLI_1_8 = "1.8.0" diff --git a/acceptance/bundle/state/feature_flags/reject_old/out.test.toml b/acceptance/bundle/state/feature_flags/reject_old/out.test.toml index 04719895b0c..e47e156b114 100644 --- a/acceptance/bundle/state/feature_flags/reject_old/out.test.toml +++ b/acceptance/bundle/state/feature_flags/reject_old/out.test.toml @@ -1,5 +1,4 @@ Cloud = false -CLIVersions.CLI_1_8 = "1.8.0" EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [""] EnvMatrix.FEATURE = ["future_feature", "deployment_history"] diff --git a/acceptance/bundle/state/feature_flags/reject_old/test.toml b/acceptance/bundle/state/feature_flags/reject_old/test.toml index 0e6af2b7696..5fc4cac284e 100644 --- a/acceptance/bundle/state/feature_flags/reject_old/test.toml +++ b/acceptance/bundle/state/feature_flags/reject_old/test.toml @@ -1,3 +1 @@ EnvMatrix.FEATURE = ["future_feature", "deployment_history"] - -CLIVersions.CLI_1_8 = "1.8.0" diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index f8c86e047d7..1cf82bbb341 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -101,10 +101,6 @@ type TestConfig struct { CompiledIgnoreObject *ignore.GitIgnore - // Released CLI versions to download for this test, keyed by the environment variable - // that exposes each executable to the script (e.g. CLIVersions.CLI_1_8 = "1.8.0"). - CLIVersions map[string]string - // Environment variables // If the same variable is defined both in Env and EnvMatrix, the one in EnvMatrix takes precedence // regardless of which config file it is defined in. diff --git a/acceptance/internal/materialized_config.go b/acceptance/internal/materialized_config.go index 27297563cf4..41fed59eeeb 100644 --- a/acceptance/internal/materialized_config.go +++ b/acceptance/internal/materialized_config.go @@ -53,10 +53,6 @@ func GenerateMaterializedConfig(config *TestConfig) string { for _, k := range slices.Sorted(maps.Keys(config.CloudEnvs)) { fmt.Fprintf(&buf, "CloudEnvs.%s = %v\n", k, config.CloudEnvs[k]) } - for _, name := range slices.Sorted(maps.Keys(config.CLIVersions)) { - fmt.Fprintf(&buf, "CLIVersions.%s = %s\n", name, tomlQuote(config.CLIVersions[name])) - } - envMatrix := filteredEnvMatrix(config.EnvMatrix, config.EnvMatrixExclude) for _, k := range slices.Sorted(maps.Keys(envMatrix)) { writeTomlStringArray(&buf, "EnvMatrix."+k, envMatrix[k]) From fe6dc22643f8afbd3676955886902b1f208be49b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 12:51:01 +0000 Subject: [PATCH 13/17] Simplify state-version compatibility changelog --- .nextchanges/bundles/direct-state-v3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/direct-state-v3.md b/.nextchanges/bundles/direct-state-v3.md index 3339570da9e..04dbaad1d94 100644 --- a/.nextchanges/bundles/direct-state-v3.md +++ b/.nextchanges/bundles/direct-state-v3.md @@ -1 +1 @@ -* Bump the direct deployment state version to 3 for bundles with and without deployment history. Starting with this release, deploying or redeploying a bundle with the direct engine writes state that requires CLI v1.8.0 or newer. Older clients can fail with a "state version 3 is newer than supported version 2; upgrade the CLI" error when accessing these deployments. Upgrade local installations and CI/CD clients before deploying with this release or later. Bundles with deployment history enabled additionally require a CLI that supports the `deployment_history` feature and a bundle configured to use it; clients that do not support the feature reject the state even if they support state version 3. ([#6706](https://github.com/databricks/cli/pull/6706)) +* Upgrade local and CI/CD clients to CLI v1.8.0 or newer before deploying with this release. Direct-engine deployments now write state version 3, which older clients do not support. ([#6706](https://github.com/databricks/cli/pull/6706)) From 2b9feb6b9ded27ebc8ba22b3e80e5bbd6df34a29 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 12:54:01 +0000 Subject: [PATCH 14/17] Prioritize state version errors and highlight compatibility change --- .../{bundles => notable-changes}/direct-state-v3.md | 0 acceptance/bundle/state/future_version/out.test.toml | 1 + acceptance/bundle/state/future_version/output.txt | 5 +++-- .../future_version/resources.future_with_feature.json | 10 ++++++++++ acceptance/bundle/state/future_version/script | 4 ++-- acceptance/bundle/state/future_version/test.toml | 2 ++ bundle/direct/dstate/migrate.go | 6 +++--- 7 files changed, 21 insertions(+), 7 deletions(-) rename .nextchanges/{bundles => notable-changes}/direct-state-v3.md (100%) create mode 100644 acceptance/bundle/state/future_version/resources.future_with_feature.json diff --git a/.nextchanges/bundles/direct-state-v3.md b/.nextchanges/notable-changes/direct-state-v3.md similarity index 100% rename from .nextchanges/bundles/direct-state-v3.md rename to .nextchanges/notable-changes/direct-state-v3.md diff --git a/acceptance/bundle/state/future_version/out.test.toml b/acceptance/bundle/state/future_version/out.test.toml index 59b56a2037c..5d6d91241d2 100644 --- a/acceptance/bundle/state/future_version/out.test.toml +++ b/acceptance/bundle/state/future_version/out.test.toml @@ -1,3 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] +EnvMatrix.STATE_FILE = ["resources.future.json", "resources.future_with_feature.json"] diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 0a16971f472..9bd6da41ceb 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,4 @@ -state version 999 is newer than supported version 3; upgrade the CLI -Exit code: 1 +>>> musterr [CLI] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: state version 999 is newer than supported version 3; upgrade the CLI + diff --git a/acceptance/bundle/state/future_version/resources.future_with_feature.json b/acceptance/bundle/state/future_version/resources.future_with_feature.json new file mode 100644 index 00000000000..e1118814c48 --- /dev/null +++ b/acceptance/bundle/state/future_version/resources.future_with_feature.json @@ -0,0 +1,10 @@ +{ + "state_version": 999, + "cli_version": "99.0.0", + "lineage": "test-lineage", + "serial": 1, + "state": {}, + "features": { + "future_feature": {} + } +} diff --git a/acceptance/bundle/state/future_version/script b/acceptance/bundle/state/future_version/script index 85fdf969f72..343526571cf 100644 --- a/acceptance/bundle/state/future_version/script +++ b/acceptance/bundle/state/future_version/script @@ -1,4 +1,4 @@ mkdir -p .databricks/bundle/default -cp resources.future.json .databricks/bundle/default/resources.json +cp "$STATE_FILE" .databricks/bundle/default/resources.json -trace $CLI bundle plan 2>&1 | grep -o 'state version.*' +trace musterr $CLI bundle plan 2>&1 | contains.py "state version 999 is newer than supported version 3; upgrade the CLI" "!requires features this CLI does not support" diff --git a/acceptance/bundle/state/future_version/test.toml b/acceptance/bundle/state/future_version/test.toml index 5640bffa4dc..0d49a0272e7 100644 --- a/acceptance/bundle/state/future_version/test.toml +++ b/acceptance/bundle/state/future_version/test.toml @@ -1,3 +1,5 @@ Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +EnvMatrix.STATE_FILE = ["resources.future.json", "resources.future_with_feature.json"] diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 3ee818bd38e..e0ce585bedf 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -13,6 +13,9 @@ import ( // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { + if db.StateVersion > currentStateVersion { + return fmt.Errorf("state version %d is newer than supported version %d; upgrade the CLI", db.StateVersion, currentStateVersion) + } if err := assertNoUnsupportedFeatures(db.Features); err != nil { return err } @@ -20,9 +23,6 @@ func migrateState(db *Database) error { if db.StateVersion == currentStateVersion { return nil } - if db.StateVersion > currentStateVersion { - return fmt.Errorf("state version %d is newer than supported version %d; upgrade the CLI", db.StateVersion, currentStateVersion) - } for version := db.StateVersion; version < currentStateVersion; version++ { fn, ok := migrations[version] From f65216f85cc788b482790a1752ad30fffcdef6ba Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 12:57:09 +0000 Subject: [PATCH 15/17] Simplify notable changelog and assert future state remains unchanged --- .nextchanges/notable-changes/direct-state-v3.md | 2 +- acceptance/bundle/state/future_version/output.txt | 2 ++ acceptance/bundle/state/future_version/script | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.nextchanges/notable-changes/direct-state-v3.md b/.nextchanges/notable-changes/direct-state-v3.md index 04dbaad1d94..d95eeb0c011 100644 --- a/.nextchanges/notable-changes/direct-state-v3.md +++ b/.nextchanges/notable-changes/direct-state-v3.md @@ -1 +1 @@ -* Upgrade local and CI/CD clients to CLI v1.8.0 or newer before deploying with this release. Direct-engine deployments now write state version 3, which older clients do not support. ([#6706](https://github.com/databricks/cli/pull/6706)) +* Direct-engine deployments now write state version 3. Bundles deployed with this release or later require CLI v1.8.0 or newer. ([#6706](https://github.com/databricks/cli/pull/6706)) diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 9bd6da41ceb..5929ce8d6c8 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -2,3 +2,5 @@ >>> musterr [CLI] bundle plan Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: state version 999 is newer than supported version 3; upgrade the CLI + +>>> cmp [STATE_FILE] .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/future_version/script b/acceptance/bundle/state/future_version/script index 343526571cf..b3bba844e28 100644 --- a/acceptance/bundle/state/future_version/script +++ b/acceptance/bundle/state/future_version/script @@ -2,3 +2,4 @@ mkdir -p .databricks/bundle/default cp "$STATE_FILE" .databricks/bundle/default/resources.json trace musterr $CLI bundle plan 2>&1 | contains.py "state version 999 is newer than supported version 3; upgrade the CLI" "!requires features this CLI does not support" +trace cmp "$STATE_FILE" .databricks/bundle/default/resources.json From 17353ebe44bcc047d7290afa5ea88f29d430f1dd Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 12:58:08 +0000 Subject: [PATCH 16/17] Clarify notable state-version compatibility change --- .nextchanges/notable-changes/direct-state-v3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/notable-changes/direct-state-v3.md b/.nextchanges/notable-changes/direct-state-v3.md index d95eeb0c011..df11c106e9d 100644 --- a/.nextchanges/notable-changes/direct-state-v3.md +++ b/.nextchanges/notable-changes/direct-state-v3.md @@ -1 +1 @@ -* Direct-engine deployments now write state version 3. Bundles deployed with this release or later require CLI v1.8.0 or newer. ([#6706](https://github.com/databricks/cli/pull/6706)) +* Bump the direct deployment state version to 3. Clients older than v1.8.0 will reject bundles deployed with this release. ([#6706](https://github.com/databricks/cli/pull/6706)) From 6306735c3a9ee46ca64e9981d0f240b805ab9076 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 16 Sep 2026 13:07:09 +0000 Subject: [PATCH 17/17] Link changelog to same-repository PR --- .nextchanges/notable-changes/direct-state-v3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/notable-changes/direct-state-v3.md b/.nextchanges/notable-changes/direct-state-v3.md index df11c106e9d..f140c1a49b9 100644 --- a/.nextchanges/notable-changes/direct-state-v3.md +++ b/.nextchanges/notable-changes/direct-state-v3.md @@ -1 +1 @@ -* Bump the direct deployment state version to 3. Clients older than v1.8.0 will reject bundles deployed with this release. ([#6706](https://github.com/databricks/cli/pull/6706)) +* Bump the direct deployment state version to 3. Clients older than v1.8.0 will reject bundles deployed with this release. ([#6713](https://github.com/databricks/cli/pull/6713))