diff --git a/.nextchanges/notable-changes/direct-state-v3.md b/.nextchanges/notable-changes/direct-state-v3.md new file mode 100644 index 00000000000..f140c1a49b9 --- /dev/null +++ b/.nextchanges/notable-changes/direct-state-v3.md @@ -0,0 +1 @@ +* Bump the direct deployment state version to 3. Clients older than v1.8.0 will reject bundles deployed with this release. ([#6713](https://github.com/databricks/cli/pull/6713)) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 51d1a4572e0..4920203c0ac 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -400,6 +400,10 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo t.Setenv("CLI_293", cli293Path) repls.SetPath(cli293Path, "[CLI_293]") + cli18Path := DownloadCLI(t, buildDir, "1.8.0") + t.Setenv("CLI_1_8", cli18Path) + repls.SetPath(cli18Path, "[CLI_1_8]") + // CLI version that predates the hashed_fields feature. Used by acceptance/bundle/hashing cli1161Path := DownloadCLI(t, buildDir, "1.16.1") t.Setenv("CLI_1_16_1", cli1161Path) diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt index d3bfab4923a..0fea6799b11 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt @@ -11,7 +11,7 @@ Exit code: [KILLED] "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, - "state_version": 2 + "state_version": 3 } { "k": "resources.jobs.job_01", diff --git a/acceptance/bundle/deploy/wal/crash-after-create/output.txt b/acceptance/bundle/deploy/wal/crash-after-create/output.txt index fc8ace6bf00..56822d5fbc0 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/output.txt +++ b/acceptance/bundle/deploy/wal/crash-after-create/output.txt @@ -12,7 +12,7 @@ Exit code: [KILLED] >>> cat .databricks/bundle/default/resources.json.wal { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1 diff --git a/acceptance/bundle/deploy/wal/header-only-wal/output.txt b/acceptance/bundle/deploy/wal/header-only-wal/output.txt index 3d4eb38635d..86ca2583663 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/output.txt +++ b/acceptance/bundle/deploy/wal/header-only-wal/output.txt @@ -7,7 +7,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/defaul Exit code: [KILLED] >>> cat .databricks/bundle/default/resources.json.wal -{"state_version":2,"cli_version":"[CLI_VERSION]","lineage":"[UUID]","serial":1} +{"state_version":3,"cli_version":"[CLI_VERSION]","lineage":"[UUID]","serial":1} === Second deploy (killed again, leaves another header-only WAL) >>> errcode [CLI] bundle deploy --force-lock @@ -17,7 +17,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/defaul Exit code: [KILLED] >>> cat .databricks/bundle/default/resources.json.wal -{"state_version":2,"cli_version":"[CLI_VERSION]","lineage":"[UUID]","serial":1} +{"state_version":3,"cli_version":"[CLI_VERSION]","lineage":"[UUID]","serial":1} === Third deploy (must recover and succeed, not blocked by the leftover WAL) >>> errcode [CLI] bundle deploy --force-lock diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json index eb3a71d91ff..05614d798bf 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 7a3e4e58bed..1492106a3fa 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -94,7 +94,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 50aeb1d46e5..d1f77897b71 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -86,7 +86,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index a0db7b41dd6..0ff3f79a722 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -158,7 +158,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index a8d2f79a989..38d2c403172 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -21,7 +21,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> jq {state_version, features} .databricks/bundle/default/resources.json { - "state_version": 2, + "state_version": 3, "features": null } @@ -77,7 +77,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> jq {state_version, features} .databricks/bundle/default/resources.json { - "state_version": 2, + "state_version": 3, "features": { "deployment_history": {} } diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 4f8de666aa9..6c908dd1b50 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -87,7 +87,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw state after successful deploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -178,7 +178,7 @@ API message: Fault injected by test. === The state shows the resource still exists but the operation is marked failed >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 9f354ed381d..efa9264f9ad 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -87,7 +87,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw state after successful deploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -173,7 +173,7 @@ Files: 2 uploaded, 0 deleted === The raw state after failed recreate shows no resource >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 8c5be14e0b5..bdb934a9871 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -40,7 +40,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The raw recorded state after deploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -94,7 +94,7 @@ Files: 1 uploaded, 0 deleted === The raw recorded state after the failure >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 04c529b2380..47bcf5690d9 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -65,7 +65,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -130,7 +130,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged === The raw recorded state after redeploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 4dc8a5c9269..0830a708248 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -45,7 +45,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -178,7 +178,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The raw recorded state the service holds after the update >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index c2b96d2f736..b0167af933e 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -47,7 +47,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -113,7 +113,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged === The raw recorded state after redeploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index e33b870fd84..b9ad9db7eee 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -34,7 +34,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw recorded state after deploy >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -76,7 +76,7 @@ Files: 2 uploaded, 0 deleted === The raw recorded state after the failure >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index ffe5cec906e..dff91b1e0e7 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -95,7 +95,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms -t dev { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 6b16a68b15b..87824ba440a 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -87,7 +87,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 3ccb46a3e11..c4e2f6844cf 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -30,7 +30,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/dms/state-features/databricks.yml.tmpl b/acceptance/bundle/dms/state-features/databricks.yml.tmpl new file mode 100644 index 00000000000..8f0cc75b608 --- /dev/null +++ b/acceptance/bundle/dms/state-features/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-state-features-$UNIQUE_NAME +experimental: + deployment_history: true + +resources: + jobs: + foo: + name: state-features diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/dms/state-features/out.test.toml similarity index 52% rename from acceptance/bundle/state/feature_flags/out.test.toml rename to acceptance/bundle/dms/state-features/out.test.toml index 59b56a2037c..23c07f70dca 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/dms/state-features/out.test.toml @@ -1,3 +1,3 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/state-features/output.txt b/acceptance/bundle/dms/state-features/output.txt new file mode 100644 index 00000000000..8af400ca848 --- /dev/null +++ b/acceptance/bundle/dms/state-features/output.txt @@ -0,0 +1,51 @@ + +=== Deploy records version 3 and the deployment_history feature, with resources held in DMS +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-state-features-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq {state_version, features, state} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": { + "deployment_history": {} + }, + "state": {} +} + +=== Reopening version 3 reads resources from DMS and produces a no-op plan +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_requests.py --dms //resources --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} + +=== Disabling deployment history refuses plan, deploy, and destroy +>>> musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle plan +Error: unsetting experimental.deployment_history is not supported + +This deployment's resources are recorded with the deployment history feature enabled. Set experimental.deployment_history: true to deploy or destroy this bundle + + +>>> musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle deploy +Error: unsetting experimental.deployment_history is not supported + +This deployment's resources are recorded with the deployment history feature enabled. Set experimental.deployment_history: true to deploy or destroy this bundle + + +>>> musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle destroy --auto-approve +Error: unsetting experimental.deployment_history is not supported + +This deployment's resources are recorded with the deployment history feature enabled. Set experimental.deployment_history: true to deploy or destroy this bundle + + +>>> print_requests.py --dms //api/2.0/bundle --oneline + +=== Reenabling deployment history still reads the unchanged resources from DMS +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_requests.py --dms //resources --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} diff --git a/acceptance/bundle/dms/state-features/script b/acceptance/bundle/dms/state-features/script new file mode 100644 index 00000000000..304cb384fdf --- /dev/null +++ b/acceptance/bundle/dms/state-features/script @@ -0,0 +1,25 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy records version 3 and the deployment_history feature, with resources held in DMS" +trace $CLI bundle deploy +trace jq '{state_version, features, state}' .databricks/bundle/default/resources.json +rm -f "$OUT_REQUESTS" + +title "Reopening version 3 reads resources from DMS and produces a no-op plan" +trace $CLI bundle plan +trace print_requests.py --dms //resources --get --oneline + +title "Disabling deployment history refuses plan, deploy, and destroy" +update_file.py databricks.yml 'deployment_history: true' 'deployment_history: false' +for command in plan deploy; do + trace musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle "$command" 2>&1 | contains.py "unsetting experimental.deployment_history is not supported" +done +trace musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle destroy --auto-approve 2>&1 | contains.py "unsetting experimental.deployment_history is not supported" +trace print_requests.py --dms //api/2.0/bundle --oneline + +title "Reenabling deployment history still reads the unchanged resources from DMS" +update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' +trace $CLI bundle plan +trace print_requests.py --dms //resources --get --oneline + +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 2257b78cc8c..65192d759de 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -87,7 +87,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { @@ -189,7 +189,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The raw recorded state after recreate >>> print_state.py --no-dms { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "features": { diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json index 7a02965e08f..e924cb4d005 100644 --- a/acceptance/bundle/migrate/basic/out.new_state.json +++ b/acceptance/bundle/migrate/basic/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/dashboards/out.new_state.json b/acceptance/bundle/migrate/dashboards/out.new_state.json index 6b83da507a6..c1bdc4c7b78 100644 --- a/acceptance/bundle/migrate/dashboards/out.new_state.json +++ b/acceptance/bundle/migrate/dashboards/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/default-python/out.state_after_migration.json index c1e83c7d577..484db23fd94 100644 --- a/acceptance/bundle/migrate/default-python/out.state_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.state_after_migration.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index 12da0aa5f9b..82b918f76d3 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 9, diff --git a/acceptance/bundle/migrate/permissions/out.new_state.json b/acceptance/bundle/migrate/permissions/out.new_state.json index 522d892b949..53d41f1fe45 100644 --- a/acceptance/bundle/migrate/permissions/out.new_state.json +++ b/acceptance/bundle/migrate/permissions/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 7, diff --git a/acceptance/bundle/migrate/runas/out.new_state.json b/acceptance/bundle/migrate/runas/out.new_state.json index 8c6bb5060ce..134f7aff689 100644 --- a/acceptance/bundle/migrate/runas/out.new_state.json +++ b/acceptance/bundle/migrate/runas/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json index b6075147e57..7bbc03c50c2 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/out.state.direct.json b/acceptance/bundle/resources/jobs/update/out.state.direct.json index 6d978c0208d..4897c138fc8 100644 --- a/acceptance/bundle/resources/jobs/update/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/state/feature_flags/databricks.yml b/acceptance/bundle/state/feature_flags/accept_new/databricks.yml similarity index 100% rename from acceptance/bundle/state/feature_flags/databricks.yml rename to acceptance/bundle/state/feature_flags/accept_new/databricks.yml diff --git a/acceptance/bundle/state/feature_flags/accept_new/out.test.toml b/acceptance/bundle/state/feature_flags/accept_new/out.test.toml new file mode 100644 index 00000000000..23f30ff55f8 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/out.test.toml @@ -0,0 +1,9 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] +EnvMatrix.STATE_FILE = [ + "resources.v2.json", + "resources.v3.json", + "resources.v2.deployment_history.json", + "resources.v3.deployment_history.json" +] diff --git a/acceptance/bundle/state/feature_flags/accept_new/output.txt b/acceptance/bundle/state/feature_flags/accept_new/output.txt new file mode 100644 index 00000000000..9ed3a7f5ae7 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/output.txt @@ -0,0 +1,16 @@ + +>>> [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> cmp [STATE_FILE] .databricks/bundle/default/resources.json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.my_job +Files: 7 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq .state_version .databricks/bundle/default/resources.json +3 diff --git a/acceptance/bundle/state/feature_flags/accept_new/resources.v2.deployment_history.json b/acceptance/bundle/state/feature_flags/accept_new/resources.v2.deployment_history.json new file mode 100644 index 00000000000..294508f75e7 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/resources.v2.deployment_history.json @@ -0,0 +1,10 @@ +{ + "state_version": 2, + "features": { + "deployment_history": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/resources.empty_features.json b/acceptance/bundle/state/feature_flags/accept_new/resources.v2.json similarity index 100% rename from acceptance/bundle/state/feature_flags/resources.empty_features.json rename to acceptance/bundle/state/feature_flags/accept_new/resources.v2.json diff --git a/acceptance/bundle/state/feature_flags/accept_new/resources.v3.deployment_history.json b/acceptance/bundle/state/feature_flags/accept_new/resources.v3.deployment_history.json new file mode 100644 index 00000000000..1e47559efd8 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/resources.v3.deployment_history.json @@ -0,0 +1,10 @@ +{ + "state_version": 3, + "features": { + "deployment_history": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/accept_new/resources.v3.json b/acceptance/bundle/state/feature_flags/accept_new/resources.v3.json new file mode 100644 index 00000000000..b20c97aa074 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/resources.v3.json @@ -0,0 +1,8 @@ +{ + "state_version": 3, + "features": {}, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/accept_new/script b/acceptance/bundle/state/feature_flags/accept_new/script new file mode 100644 index 00000000000..4928c4c2c6f --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/script @@ -0,0 +1,13 @@ +mkdir -p .databricks/bundle/default +cp "$STATE_FILE" .databricks/bundle/default/resources.json + +DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=$(jq '.features | has("deployment_history")' "$STATE_FILE") +export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY + +trace $CLI bundle plan | contains.py "Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged" +trace cmp "$STATE_FILE" .databricks/bundle/default/resources.json + +trace $CLI bundle deploy +trace jq '.state_version' .databricks/bundle/default/resources.json +# Deployment upgrades the version and preserves the input's feature flags. +diff <(jq -S '.features' "$STATE_FILE") <(jq -S '.features // {}' .databricks/bundle/default/resources.json) diff --git a/acceptance/bundle/state/feature_flags/accept_new/test.toml b/acceptance/bundle/state/feature_flags/accept_new/test.toml new file mode 100644 index 00000000000..55933de2377 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_new/test.toml @@ -0,0 +1 @@ +EnvMatrix.STATE_FILE = ["resources.v2.json", "resources.v3.json", "resources.v2.deployment_history.json", "resources.v3.deployment_history.json"] diff --git a/acceptance/bundle/state/feature_flags/accept_old/databricks.yml b/acceptance/bundle/state/feature_flags/accept_old/databricks.yml new file mode 100644 index 00000000000..5134dbcc12c --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + my_job: + name: "my job" diff --git a/acceptance/bundle/state/feature_flags/accept_old/out.test.toml b/acceptance/bundle/state/feature_flags/accept_old/out.test.toml new file mode 100644 index 00000000000..104d04f213a --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/out.test.toml @@ -0,0 +1,9 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] +EnvMatrix.STATE_FILE = [ + "resources.v2.json", + "resources.v2.future_feature.json", + "resources.v2.deployment_history.json", + "resources.v3.json" +] diff --git a/acceptance/bundle/state/feature_flags/accept_old/output.txt b/acceptance/bundle/state/feature_flags/accept_old/output.txt new file mode 100644 index 00000000000..5ac51c952d0 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/output.txt @@ -0,0 +1,7 @@ + +>>> [CLI_1_8] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> cmp [STATE_FILE] .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/accept_old/resources.v2.deployment_history.json b/acceptance/bundle/state/feature_flags/accept_old/resources.v2.deployment_history.json new file mode 100644 index 00000000000..294508f75e7 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/resources.v2.deployment_history.json @@ -0,0 +1,10 @@ +{ + "state_version": 2, + "features": { + "deployment_history": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/resources.with_feature.json b/acceptance/bundle/state/feature_flags/accept_old/resources.v2.future_feature.json similarity index 100% rename from acceptance/bundle/state/feature_flags/resources.with_feature.json rename to acceptance/bundle/state/feature_flags/accept_old/resources.v2.future_feature.json diff --git a/acceptance/bundle/state/feature_flags/accept_old/resources.v2.json b/acceptance/bundle/state/feature_flags/accept_old/resources.v2.json new file mode 100644 index 00000000000..876fdaf19ce --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/resources.v2.json @@ -0,0 +1,8 @@ +{ + "state_version": 2, + "features": {}, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/accept_old/resources.v3.json b/acceptance/bundle/state/feature_flags/accept_old/resources.v3.json new file mode 100644 index 00000000000..b20c97aa074 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/resources.v3.json @@ -0,0 +1,8 @@ +{ + "state_version": 3, + "features": {}, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/accept_old/script b/acceptance/bundle/state/feature_flags/accept_old/script new file mode 100644 index 00000000000..5b753c4212d --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/script @@ -0,0 +1,5 @@ +mkdir -p .databricks/bundle/default +cp "$STATE_FILE" .databricks/bundle/default/resources.json + +trace $CLI_1_8 bundle plan | contains.py "Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged" +trace cmp "$STATE_FILE" .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/accept_old/test.toml b/acceptance/bundle/state/feature_flags/accept_old/test.toml new file mode 100644 index 00000000000..f7d4af44c7b --- /dev/null +++ b/acceptance/bundle/state/feature_flags/accept_old/test.toml @@ -0,0 +1,2 @@ +# CLI v1.8.0 ignores feature flags in version 2 state. +EnvMatrix.STATE_FILE = ["resources.v2.json", "resources.v2.future_feature.json", "resources.v2.deployment_history.json", "resources.v3.json"] diff --git a/acceptance/bundle/state/feature_flags/output.txt b/acceptance/bundle/state/feature_flags/output.txt deleted file mode 100644 index b6385259dd2..00000000000 --- a/acceptance/bundle/state/feature_flags/output.txt +++ /dev/null @@ -1,22 +0,0 @@ - -=== a state depending on a feature this CLI does not recognize is rejected ->>> errcode [CLI] bundle plan -Error: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information - - -Exit code: 1 - -=== a state with an empty features map is accepted, and a deploy keeps it at version 2 ->>> [CLI] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.my_job -Files: 5 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> gron.py .databricks/bundle/default/resources.json -json.state_version = 2; diff --git a/acceptance/bundle/state/feature_flags/reject_new/databricks.yml b/acceptance/bundle/state/feature_flags/reject_new/databricks.yml new file mode 100644 index 00000000000..5134dbcc12c --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + my_job: + name: "my job" diff --git a/acceptance/bundle/state/feature_flags/reject_new/out.test.toml b/acceptance/bundle/state/feature_flags/reject_new/out.test.toml new file mode 100644 index 00000000000..5684d9e1005 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/out.test.toml @@ -0,0 +1,4 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] +EnvMatrix.STATE_FILE = ["resources.v2.future_feature.json", "resources.v3.future_feature.json"] diff --git a/acceptance/bundle/state/feature_flags/reject_new/output.txt b/acceptance/bundle/state/feature_flags/reject_new/output.txt new file mode 100644 index 00000000000..35eb79cf7e7 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/output.txt @@ -0,0 +1,6 @@ + +>>> musterr [CLI] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information + + +>>> cmp [STATE_FILE] .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/reject_new/resources.v2.future_feature.json b/acceptance/bundle/state/feature_flags/reject_new/resources.v2.future_feature.json new file mode 100644 index 00000000000..acc2ec9823d --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/resources.v2.future_feature.json @@ -0,0 +1,10 @@ +{ + "state_version": 2, + "features": { + "future_feature": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/reject_new/resources.v3.future_feature.json b/acceptance/bundle/state/feature_flags/reject_new/resources.v3.future_feature.json new file mode 100644 index 00000000000..b844b098a70 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/resources.v3.future_feature.json @@ -0,0 +1,10 @@ +{ + "state_version": 3, + "features": { + "future_feature": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/reject_new/script b/acceptance/bundle/state/feature_flags/reject_new/script new file mode 100644 index 00000000000..979cebdcf5a --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/script @@ -0,0 +1,5 @@ +mkdir -p .databricks/bundle/default +cp "$STATE_FILE" .databricks/bundle/default/resources.json + +trace musterr $CLI bundle plan 2>&1 | contains.py "requires features this CLI does not support: future_feature" +trace cmp "$STATE_FILE" .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/reject_new/test.toml b/acceptance/bundle/state/feature_flags/reject_new/test.toml new file mode 100644 index 00000000000..84e88afe36c --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_new/test.toml @@ -0,0 +1 @@ +EnvMatrix.STATE_FILE = ["resources.v2.future_feature.json", "resources.v3.future_feature.json"] diff --git a/acceptance/bundle/state/feature_flags/reject_old/databricks.yml b/acceptance/bundle/state/feature_flags/reject_old/databricks.yml new file mode 100644 index 00000000000..5134dbcc12c --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + my_job: + name: "my job" diff --git a/acceptance/bundle/state/feature_flags/reject_old/out.test.toml b/acceptance/bundle/state/feature_flags/reject_old/out.test.toml new file mode 100644 index 00000000000..e47e156b114 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/out.test.toml @@ -0,0 +1,4 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] +EnvMatrix.FEATURE = ["future_feature", "deployment_history"] diff --git a/acceptance/bundle/state/feature_flags/reject_old/output.txt b/acceptance/bundle/state/feature_flags/reject_old/output.txt new file mode 100644 index 00000000000..f4e77bc90c5 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/output.txt @@ -0,0 +1,6 @@ + +>>> musterr [CLI_1_8] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: [FEATURE]; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information + + +>>> cmp resources.v3.[FEATURE].json .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/reject_old/resources.v3.deployment_history.json b/acceptance/bundle/state/feature_flags/reject_old/resources.v3.deployment_history.json new file mode 100644 index 00000000000..1e47559efd8 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/resources.v3.deployment_history.json @@ -0,0 +1,10 @@ +{ + "state_version": 3, + "features": { + "deployment_history": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/reject_old/resources.v3.future_feature.json b/acceptance/bundle/state/feature_flags/reject_old/resources.v3.future_feature.json new file mode 100644 index 00000000000..b844b098a70 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/resources.v3.future_feature.json @@ -0,0 +1,10 @@ +{ + "state_version": 3, + "features": { + "future_feature": {} + }, + "cli_version": "0.0.0-dev", + "lineage": "test-lineage", + "serial": 1, + "state": {} +} diff --git a/acceptance/bundle/state/feature_flags/reject_old/script b/acceptance/bundle/state/feature_flags/reject_old/script new file mode 100644 index 00000000000..f3b49e0c453 --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/script @@ -0,0 +1,5 @@ +mkdir -p .databricks/bundle/default +cp "resources.v3.$FEATURE.json" .databricks/bundle/default/resources.json + +trace musterr $CLI_1_8 bundle plan 2>&1 | contains.py "requires features this CLI does not support: $FEATURE" +trace cmp "resources.v3.$FEATURE.json" .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/reject_old/test.toml b/acceptance/bundle/state/feature_flags/reject_old/test.toml new file mode 100644 index 00000000000..5fc4cac284e --- /dev/null +++ b/acceptance/bundle/state/feature_flags/reject_old/test.toml @@ -0,0 +1 @@ +EnvMatrix.FEATURE = ["future_feature", "deployment_history"] diff --git a/acceptance/bundle/state/feature_flags/script b/acceptance/bundle/state/feature_flags/script deleted file mode 100644 index ef9570e318c..00000000000 --- a/acceptance/bundle/state/feature_flags/script +++ /dev/null @@ -1,11 +0,0 @@ -mkdir -p .databricks/bundle/default - -title "a state depending on a feature this CLI does not recognize is rejected" -cp resources.with_feature.json .databricks/bundle/default/resources.json -trace errcode $CLI bundle plan 2>&1 | contains.py "requires features this CLI does not support: future_feature" "upgrade to the latest CLI version" "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" - -title "a state with an empty features map is accepted, and a deploy keeps it at version 2" -cp resources.empty_features.json .databricks/bundle/default/resources.json -trace $CLI bundle plan | contains.py "Plan:" -trace $CLI bundle deploy -trace gron.py .databricks/bundle/default/resources.json | grep state_version diff --git a/acceptance/bundle/state/feature_flags/test.toml b/acceptance/bundle/state/feature_flags/test.toml index 5640bffa4dc..e355ab2feb2 100644 --- a/acceptance/bundle/state/feature_flags/test.toml +++ b/acceptance/bundle/state/feature_flags/test.toml @@ -1,3 +1,6 @@ Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] + +# New-CLI tests use the source build; pin them to this week's release (v1.17.0) once published. diff --git a/acceptance/bundle/state/future_version/out.test.toml b/acceptance/bundle/state/future_version/out.test.toml index 59b56a2037c..5d6d91241d2 100644 --- a/acceptance/bundle/state/future_version/out.test.toml +++ b/acceptance/bundle/state/future_version/out.test.toml @@ -1,3 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] +EnvMatrix.STATE_FILE = ["resources.future.json", "resources.future_with_feature.json"] diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 0a16971f472..5929ce8d6c8 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,6 @@ -state version 999 is newer than supported version 3; upgrade the CLI -Exit code: 1 +>>> musterr [CLI] bundle plan +Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: state version 999 is newer than supported version 3; upgrade the CLI + + +>>> cmp [STATE_FILE] .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/future_version/resources.future_with_feature.json b/acceptance/bundle/state/future_version/resources.future_with_feature.json new file mode 100644 index 00000000000..e1118814c48 --- /dev/null +++ b/acceptance/bundle/state/future_version/resources.future_with_feature.json @@ -0,0 +1,10 @@ +{ + "state_version": 999, + "cli_version": "99.0.0", + "lineage": "test-lineage", + "serial": 1, + "state": {}, + "features": { + "future_feature": {} + } +} diff --git a/acceptance/bundle/state/future_version/script b/acceptance/bundle/state/future_version/script index 85fdf969f72..b3bba844e28 100644 --- a/acceptance/bundle/state/future_version/script +++ b/acceptance/bundle/state/future_version/script @@ -1,4 +1,5 @@ mkdir -p .databricks/bundle/default -cp resources.future.json .databricks/bundle/default/resources.json +cp "$STATE_FILE" .databricks/bundle/default/resources.json -trace $CLI bundle plan 2>&1 | grep -o 'state version.*' +trace musterr $CLI bundle plan 2>&1 | contains.py "state version 999 is newer than supported version 3; upgrade the CLI" "!requires features this CLI does not support" +trace cmp "$STATE_FILE" .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/future_version/test.toml b/acceptance/bundle/state/future_version/test.toml index 5640bffa4dc..0d49a0272e7 100644 --- a/acceptance/bundle/state/future_version/test.toml +++ b/acceptance/bundle/state/future_version/test.toml @@ -1,3 +1,5 @@ Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +EnvMatrix.STATE_FILE = ["resources.future.json", "resources.future_with_feature.json"] diff --git a/acceptance/bundle/state/permission_level_migration/output.txt b/acceptance/bundle/state/permission_level_migration/output.txt index e4b00bb73fd..919997d9267 100644 --- a/acceptance/bundle/state/permission_level_migration/output.txt +++ b/acceptance/bundle/state/permission_level_migration/output.txt @@ -13,7 +13,7 @@ Resources: 1 created, 1 changed, 0 deleted, 0 unchanged === Print state after deploy >>> print_state.py { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "test-lineage", "serial": 2, diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index 5326f079064..f0c60602386 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -240,7 +240,7 @@ "overwrite": "true" }, "body": { - "state_version": 2, + "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index e4d21a7054a..e0ce585bedf 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -3,8 +3,6 @@ package dstate import ( "encoding/json" "fmt" - "slices" - "strings" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/structs/structpath" @@ -15,30 +13,16 @@ import ( // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // featureStateVersion states carry a feature list this CLI does not yet write or - // understand (see the featureStateVersion doc comment). A featureStateVersion - // state with no features is equivalent to currentStateVersion, so accept it and - // return without running the migrations below, leaving the on-disk version at - // featureStateVersion rather than flipping it down. One that records any feature - // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. - if db.StateVersion == featureStateVersion { - if len(db.Features) == 0 { - return nil - } - features := make([]string, 0, len(db.Features)) - for name := range db.Features { - features = append(features, name) - } - slices.Sort(features) - return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) + if db.StateVersion > currentStateVersion { + return fmt.Errorf("state version %d is newer than supported version %d; upgrade the CLI", db.StateVersion, currentStateVersion) + } + if err := assertNoUnsupportedFeatures(db.Features); err != nil { + return err } if db.StateVersion == currentStateVersion { return nil } - if db.StateVersion > supportedStateVersion { - return fmt.Errorf("state version %d is newer than supported version %d; upgrade the CLI", db.StateVersion, supportedStateVersion) - } for version := db.StateVersion; version < currentStateVersion; version++ { fn, ok := migrations[version] @@ -59,6 +43,8 @@ func migrateState(db *Database) error { var migrations = map[int]func(*Database) error{ 0: migrateV1ToV2, 1: migrateV1ToV2, + // Version 3 adds feature flags; existing states need no data changes. + 2: func(*Database) error { return nil }, } // migrateV1ToV2 migrates permissions and grants entries from the old format diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index aa2d53620d0..48c32c9e0ee 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -27,35 +27,12 @@ import ( ) const ( - // currentStateVersion is the schema version written for deployments that record - // no feature flags, and the version legacy states are migrated up to on load. - currentStateVersion = 2 + // currentStateVersion is the schema version written by this CLI and the version + // legacy states are migrated up to on load. + currentStateVersion = 3 initialBufferSize = 64 * 1024 maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - - // featureStateVersion is the schema version a future CLI will write once it - // records deployment state "feature flags" (see Header.Features). This CLI does - // not write it and records no features; it exists now only so this CLI reads - // such states correctly (see migrateState): - // - featureStateVersion with no features -> accept and leave the version as-is - // - featureStateVersion with any feature -> refuse, tell the user to upgrade - // - // A featureStateVersion state with no features is equivalent to - // currentStateVersion, but we deliberately do not flip the on-disk version down - // to currentStateVersion: a state written at featureStateVersion stays at - // featureStateVersion. This is forward-compat scaffolding so that a later release - // can start writing featureStateVersion + features without older CLIs (with this - // change) either mishandling a feature they lack or rejecting a featureless state - // outright. featureStateVersion is always 3. - featureStateVersion = 3 - - // supportedStateVersion is the highest schema version this CLI can read. It is - // normally equal to currentStateVersion — the version this CLI reads is the - // version it writes — and exceeds it only during a two-phase version bump like - // the current feature-flag scaffolding, where this CLI reads (but does not - // write) featureStateVersion. A state newer than this is rejected as too new. - supportedStateVersion = featureStateVersion ) // FeatureDeploymentHistory marks a state whose resources are also recorded with the @@ -553,12 +530,6 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe return fmt.Errorf("migrating state %s: %w", path, err) } - // TODO: We can remove and move this assertion to migrateState once we do the state - // version bump to 3 for this CLI. - if err := assertNoUnsupportedFeatures(db.Data.Features); err != nil { - return err - } - db.stateIDs = make(map[string]string) for key, entry := range db.Data.State { db.stateIDs[key] = entry.ID diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 24af630b069..c8d78d150d6 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -182,28 +182,14 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { mustFinalize(t, &recovered) } -// TestEmptyFeatureStateAcceptedWithoutFlippingVersion pins the special case that a -// featureStateVersion state with no features is accepted as-is — the on-disk version -// is left at featureStateVersion, not flipped down to currentStateVersion — and that -// a featureStateVersion state recording any feature is refused. This is scaffolding -// for the deferred version bump, special-cased to featureStateVersion only (see the -// featureStateVersion doc comment). -// -// When the baseline is actually bumped to featureStateVersion, this special case must -// go away. This test is the forcing function: it fails once featureStateVersion is -// removed, making the author decide what the post-bump behavior should be. func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { - // The special case applies to featureStateVersion (3) only. - require.Equal(t, 2, currentStateVersion, "when currentStateVersion is bumped, remove featureStateVersion and this special case") - require.Equal(t, 3, featureStateVersion) - - empty := &Database{Header: Header{StateVersion: featureStateVersion}} + empty := &Database{Header: Header{StateVersion: 3}} require.NoError(t, migrateState(empty)) - assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") + assert.Equal(t, 3, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") - // v3 that records a feature is refused: this CLI does not understand features. + // Version 3 states with unknown features must still be refused. withFeature := &Database{Header: Header{ - StateVersion: featureStateVersion, + StateVersion: 3, Features: map[string]struct{}{"future_feature": {}}, }} err := migrateState(withFeature)