From ab5c4bd3438b70bc364385ccf55fb0337a7ed5f5 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 15 Sep 2026 12:08:19 +0200 Subject: [PATCH 1/3] Fix uploading libraries when running bundle deploy --plan --- .../deploy/readplan/whl-not-uploaded/script | 28 ------------ .../databricks.yml | 7 ++- .../dist/my_test_code-0.0.1-py3-none-any.whl | Bin .../out.plan.json | 10 +++-- .../out.test.toml | 0 .../output.txt | 24 ++++++---- .../bundle/deploy/readplan/whl-upload/script | 25 +++++++++++ .../deploy/readplan/whl-upload/test.toml | 4 ++ bundle/phases/build.go | 42 +++++++++++++----- cmd/bundle/utils/process.go | 24 +++++++++- 10 files changed, 110 insertions(+), 54 deletions(-) delete mode 100644 acceptance/bundle/deploy/readplan/whl-not-uploaded/script rename acceptance/bundle/deploy/readplan/{whl-not-uploaded => whl-upload}/databricks.yml (70%) rename acceptance/bundle/deploy/readplan/{whl-not-uploaded => whl-upload}/dist/my_test_code-0.0.1-py3-none-any.whl (100%) rename acceptance/bundle/deploy/readplan/{whl-not-uploaded => whl-upload}/out.plan.json (82%) rename acceptance/bundle/deploy/readplan/{whl-not-uploaded => whl-upload}/out.test.toml (100%) rename acceptance/bundle/deploy/readplan/{whl-not-uploaded => whl-upload}/output.txt (62%) create mode 100644 acceptance/bundle/deploy/readplan/whl-upload/script create mode 100644 acceptance/bundle/deploy/readplan/whl-upload/test.toml diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script deleted file mode 100644 index bfcfeafca9e..00000000000 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script +++ /dev/null @@ -1,28 +0,0 @@ -# A local wheel referenced by a job library is uploaded by `bundle deploy` but NOT by -# `bundle deploy --plan`: applying a saved plan skips the build phase, which is what -# computes the artifact upload list (phases.Build -> libraries.ReplaceWithRemotePath). -# The job still points at the wheel's workspace path, so its run fails at runtime with a -# missing-file error. This test pins the difference between the two deploy paths. - -title "bundle deploy uploads the wheel" -trace $CLI bundle deploy -# An import-file request uploaded the wheel to its .../artifacts/.internal/ workspace path. -# raw_body is the wheel's binary payload; drop it to keep the golden readable. -trace print_requests.py //artifacts/.internal --del-field raw_body - -trace $CLI bundle destroy --auto-approve -# Discard destroy's requests so the saved-plan deploy below starts from a clean recording. -rm "$OUT_REQUESTS" - -title "bundle deploy --plan does NOT upload the wheel (bug)" -trace $CLI bundle plan -o json > tmp.plan.json -# The serial differs between the DMS variants here: destroy removes the recorded deployment, so -# under recording there is no version left to take it from, while the local state keeps counting. -nostamp < tmp.plan.json | jq 'del(.serial)' > out.plan.json -trace $CLI bundle deploy --plan tmp.plan.json -# BUG: no import-file request for the wheel here (output below is empty), yet the job was -# created pointing the task at the same .../artifacts/.internal/ path that was never uploaded. -trace print_requests.py //artifacts/.internal --del-field raw_body - -trace $CLI bundle destroy --auto-approve -rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/databricks.yml b/acceptance/bundle/deploy/readplan/whl-upload/databricks.yml similarity index 70% rename from acceptance/bundle/deploy/readplan/whl-not-uploaded/databricks.yml rename to acceptance/bundle/deploy/readplan/whl-upload/databricks.yml index f67f45222fb..28c39c69f0c 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/databricks.yml +++ b/acceptance/bundle/deploy/readplan/whl-upload/databricks.yml @@ -1,10 +1,10 @@ bundle: - name: whl-not-uploaded + name: whl-upload resources: jobs: my_job: - name: whl-not-uploaded-job + name: whl-upload-job tasks: - task_key: main new_cluster: @@ -15,6 +15,5 @@ resources: package_name: my_test_code entry_point: run libraries: - # Local wheel, prebuilt (no build: step) to keep the test hermetic. See script - # for the bug this exercises. Behaves the same as a wheel with a build: step. + # Local wheel, prebuilt (no build: step) to keep the test hermetic. - whl: ./dist/*.whl diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/dist/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/deploy/readplan/whl-upload/dist/my_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from acceptance/bundle/deploy/readplan/whl-not-uploaded/dist/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/deploy/readplan/whl-upload/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json b/acceptance/bundle/deploy/readplan/whl-upload/out.plan.json similarity index 82% rename from acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json rename to acceptance/bundle/deploy/readplan/whl-upload/out.plan.json index cc6ec22302d..73a837fd9f3 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json +++ b/acceptance/bundle/deploy/readplan/whl-upload/out.plan.json @@ -2,6 +2,9 @@ "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "features": { + "deployment_history": {} + }, "plan": { "resources.jobs.my_job": { "action": "create", @@ -9,12 +12,13 @@ "value": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, - "name": "whl-not-uploaded-job", + "name": "whl-upload-job", "queue": { "enabled": true }, @@ -22,7 +26,7 @@ { "libraries": [ { - "whl": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + "whl": "/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" } ], "new_cluster": { diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml b/acceptance/bundle/deploy/readplan/whl-upload/out.test.toml similarity index 100% rename from acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml rename to acceptance/bundle/deploy/readplan/whl-upload/out.test.toml diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt b/acceptance/bundle/deploy/readplan/whl-upload/output.txt similarity index 62% rename from acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt rename to acceptance/bundle/deploy/readplan/whl-upload/output.txt index 981e942ee4f..6d08054977d 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt +++ b/acceptance/bundle/deploy/readplan/whl-upload/output.txt @@ -2,7 +2,7 @@ === bundle deploy uploads the wheel >>> [CLI] bundle deploy Uploading dist/my_test_code-0.0.1-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-upload/default/files... Created jobs.my_job Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged @@ -10,7 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //artifacts/.internal --del-field raw_body { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl", "q": { "overwrite": "true" } @@ -20,25 +20,33 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged The following resources will be deleted: delete resources.jobs.my_job -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/whl-upload/default Destroy: 1 deleted -=== bundle deploy --plan does NOT upload the wheel (bug) +=== bundle deploy --plan also uploads the wheel >>> [CLI] bundle plan -o json ->>> [CLI] bundle deploy --plan tmp.plan.json -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/files... +>>> [CLI] bundle deploy --plan out.plan.json +Uploading dist/my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-upload/default/files... Created jobs.my_job -Files: 6 uploaded, 0 deleted +Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //artifacts/.internal --del-field raw_body +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl", + "q": { + "overwrite": "true" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.my_job -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/whl-upload/default Destroy: 1 deleted diff --git a/acceptance/bundle/deploy/readplan/whl-upload/script b/acceptance/bundle/deploy/readplan/whl-upload/script new file mode 100644 index 00000000000..1b7657b5415 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-upload/script @@ -0,0 +1,25 @@ +# A local wheel referenced by a job library must be uploaded to its .../artifacts/.internal/ +# workspace path by both `bundle deploy` and `bundle deploy --plan`. Applying a saved plan +# still runs the build phase, which computes the artifact upload list +# (phases.Build -> libraries.ReplaceWithRemotePath), so the wheel the job task points at +# actually exists remotely. This test pins that the two deploy paths upload it identically. + +title "bundle deploy uploads the wheel" +trace $CLI bundle deploy +# An import-file request uploaded the wheel to its .../artifacts/.internal/ workspace path. +# raw_body is the wheel's binary payload; drop it to keep the golden readable. +trace print_requests.py //artifacts/.internal --del-field raw_body + +trace $CLI bundle destroy --auto-approve +# Discard destroy's requests so the saved-plan deploy below starts from a clean recording. +rm "$OUT_REQUESTS" + +title "bundle deploy --plan also uploads the wheel" +trace $CLI bundle plan -o json > out.plan.json +trace $CLI bundle deploy --plan out.plan.json +# The same import-file request uploads the wheel to the .../artifacts/.internal/ path the +# job task points at, matching the plain `bundle deploy` above. +trace print_requests.py //artifacts/.internal --del-field raw_body + +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/deploy/readplan/whl-upload/test.toml b/acceptance/bundle/deploy/readplan/whl-upload/test.toml new file mode 100644 index 00000000000..f987e32edfd --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-upload/test.toml @@ -0,0 +1,4 @@ +# --plan is direct-engine only. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true +Ignore = [".databricks"] diff --git a/bundle/phases/build.go b/bundle/phases/build.go index f8069b73bcf..e8a4b13d275 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -18,15 +18,11 @@ import ( // Computed by Build and consumed by Deploy to upload the right files. type LibLocationMap map[string][]libraries.LocationToUpdate -// Build runs the build phase, which builds artifacts. -func Build(ctx context.Context, b *bundle.Bundle) LibLocationMap { - log.Info(ctx, "Phase: build") - +// resolveLibraries runs variable resolution, glob expansion, path rewriting, +// and wheel-task transformation to produce the local→remote upload map. +// It is the shared tail of both Build and FindLibraries. +func resolveLibraries(ctx context.Context, b *bundle.Bundle) LibLocationMap { bundle.ApplySeqContext(ctx, b, - scripts.Execute(config.ScriptPreBuild), - artifacts.Build(), - scripts.Execute(config.ScriptPostBuild), - mutator.ResolveVariableReferencesWithoutResources( "artifacts", ), @@ -40,8 +36,6 @@ func Build(ctx context.Context, b *bundle.Bundle) LibLocationMap { // mutator is part of the deploy step rather than validate. libraries.ExpandGlobReferences(), libraries.CheckForSameNameLibraries(), - // SwitchToPatchedWheels must be run after ExpandGlobReferences and after build phase because it Artifact.Source and Artifact.Patched populated - libraries.SwitchToPatchedWheels(), ) if logdiag.HasError(ctx) { @@ -55,3 +49,31 @@ func Build(ctx context.Context, b *bundle.Bundle) LibLocationMap { bundle.ApplyContext(ctx, b, trampoline.TransformWheelTask()) return libs } + +// Build runs the build phase, which builds artifacts. +func Build(ctx context.Context, b *bundle.Bundle) LibLocationMap { + log.Info(ctx, "Phase: build") + + bundle.ApplySeqContext(ctx, b, + scripts.Execute(config.ScriptPreBuild), + artifacts.Build(), + scripts.Execute(config.ScriptPostBuild), + // SwitchToPatchedWheels must be run after ExpandGlobReferences and after build phase because it Artifact.Source and Artifact.Patched populated + libraries.SwitchToPatchedWheels(), + ) + + if logdiag.HasError(ctx) { + return nil + } + + return resolveLibraries(ctx, b) +} + +// FindLibraries discovers which local library files need uploading by reading +// the bundle config (glob expansion and path rewriting) without running any +// build commands. Used when applying a saved plan: artifacts were already built +// at plan time and the plan's new_state carries the correct remote paths. +func FindLibraries(ctx context.Context, b *bundle.Bundle) LibLocationMap { + log.Info(ctx, "Phase: find libraries") + return resolveLibraries(ctx, b) +} diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 2dcd549eba2..20a9dfb239a 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -78,11 +78,15 @@ type ProcessOptions struct { FastValidate bool Validate bool Build bool + FindLibraries bool PreDeployChecks bool Deploy bool // Path to pre-computed plan JSON file (direct engine only). - // When set, skips Build and PreDeployChecks phases, loads plan from file instead of calculating. + // When set, skips build commands and the PreDeployChecks phase, and loads the + // plan from the file instead of calculating it. Library file discovery runs + // (via FindLibraries) so that the correct local files are found and uploaded + // to the remote paths the plan already references. ReadPlanPath string // PostStateFunc is called at the end of ProcessBundleRet, within the state lifecycle scope @@ -358,7 +362,14 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle logdiag.LogError(ctx, errors.New("--plan is only supported with direct engine (set bundle.engine to \"direct\" or DATABRICKS_BUNDLE_ENGINE=direct)")) return b, stateDesc, root.ErrAlreadyPrinted } + // Build commands are not re-run when applying a saved plan: re-running them + // could produce new artifact files with a different mtime which, with + // dynamic_version, would yield a different patched filename than the remote + // path the plan already has baked in. Library discovery still runs (via + // FindLibraries) so the correct local files are found and uploaded to the + // remote paths the plan references. opts.Build = false + opts.FindLibraries = true opts.PreDeployChecks = false var err error @@ -419,6 +430,17 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle Value: time.Since(t2).Milliseconds(), }) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } else if opts.FindLibraries { + t2 := time.Now() + libs = phases.FindLibraries(ctx, b) + b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ + Key: "phases.FindLibraries", + Value: time.Since(t2).Milliseconds(), + }) + if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } From d4c1c607920110796df4d7ef2c78ab6ea9a58fcb Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 15 Sep 2026 13:55:05 +0200 Subject: [PATCH 2/3] fixes --- .../deploy/readplan/whl-upload/out.plan.json | 7 ++----- .../deploy/readplan/whl-upload/out.test.toml | 2 +- .../deploy/readplan/whl-upload/output.txt | 6 ++---- .../bundle/deploy/readplan/whl-upload/script | 12 +++++++----- .../bundle/deploy/readplan/whl-upload/test.toml | 4 +++- bundle/phases/build.go | 17 +++++++++++------ cmd/bundle/utils/process.go | 7 +++---- 7 files changed, 29 insertions(+), 26 deletions(-) diff --git a/acceptance/bundle/deploy/readplan/whl-upload/out.plan.json b/acceptance/bundle/deploy/readplan/whl-upload/out.plan.json index 73a837fd9f3..15daca145a0 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/out.plan.json +++ b/acceptance/bundle/deploy/readplan/whl-upload/out.plan.json @@ -2,9 +2,7 @@ "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "features": { - "deployment_history": {} - }, + "serial": 2, "plan": { "resources.jobs.my_job": { "action": "create", @@ -12,8 +10,7 @@ "value": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/state/metadata.json", - "version_id": "1" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/deploy/readplan/whl-upload/out.test.toml b/acceptance/bundle/deploy/readplan/whl-upload/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/out.test.toml +++ b/acceptance/bundle/deploy/readplan/whl-upload/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/readplan/whl-upload/output.txt b/acceptance/bundle/deploy/readplan/whl-upload/output.txt index 6d08054977d..8e6b5430f39 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/output.txt +++ b/acceptance/bundle/deploy/readplan/whl-upload/output.txt @@ -25,13 +25,11 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === bundle deploy --plan also uploads the wheel ->>> [CLI] bundle plan -o json - ->>> [CLI] bundle deploy --plan out.plan.json +>>> [CLI] bundle deploy --plan tmp.plan.json Uploading dist/my_test_code-0.0.1-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-upload/default/files... Created jobs.my_job -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //artifacts/.internal --del-field raw_body diff --git a/acceptance/bundle/deploy/readplan/whl-upload/script b/acceptance/bundle/deploy/readplan/whl-upload/script index 1b7657b5415..c06707c5dd1 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/script +++ b/acceptance/bundle/deploy/readplan/whl-upload/script @@ -1,8 +1,7 @@ # A local wheel referenced by a job library must be uploaded to its .../artifacts/.internal/ # workspace path by both `bundle deploy` and `bundle deploy --plan`. Applying a saved plan -# still runs the build phase, which computes the artifact upload list -# (phases.Build -> libraries.ReplaceWithRemotePath), so the wheel the job task points at -# actually exists remotely. This test pins that the two deploy paths upload it identically. +# runs phases.FindLibraries to discover which local files to upload without re-running any +# build commands. This test pins that both deploy paths upload the wheel identically. title "bundle deploy uploads the wheel" trace $CLI bundle deploy @@ -15,8 +14,11 @@ trace $CLI bundle destroy --auto-approve rm "$OUT_REQUESTS" title "bundle deploy --plan also uploads the wheel" -trace $CLI bundle plan -o json > out.plan.json -trace $CLI bundle deploy --plan out.plan.json +# tmp.plan.json is the live plan (with features/serial for validation); +# out.plan.json is the nostamp-stripped golden for cross-variant comparison. +$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json | nostamp > out.plan.json +trace $CLI bundle deploy --plan tmp.plan.json # The same import-file request uploads the wheel to the .../artifacts/.internal/ path the # job task points at, matching the plain `bundle deploy` above. trace print_requests.py //artifacts/.internal --del-field raw_body diff --git a/acceptance/bundle/deploy/readplan/whl-upload/test.toml b/acceptance/bundle/deploy/readplan/whl-upload/test.toml index f987e32edfd..8e5c2176367 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/test.toml +++ b/acceptance/bundle/deploy/readplan/whl-upload/test.toml @@ -1,4 +1,6 @@ # --plan is direct-engine only. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +# Saved plans don't carry the deployment stamp, so this test skips the DMS variant. +EnvMatrix.DMS = [""] RecordRequests = true -Ignore = [".databricks"] +Ignore = [".databricks", "tmp.plan.json"] diff --git a/bundle/phases/build.go b/bundle/phases/build.go index e8a4b13d275..1db2fbc815e 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -20,9 +20,11 @@ type LibLocationMap map[string][]libraries.LocationToUpdate // resolveLibraries runs variable resolution, glob expansion, path rewriting, // and wheel-task transformation to produce the local→remote upload map. -// It is the shared tail of both Build and FindLibraries. -func resolveLibraries(ctx context.Context, b *bundle.Bundle) LibLocationMap { - bundle.ApplySeqContext(ctx, b, +// extra mutators are applied after CheckForSameNameLibraries and before +// ReplaceWithRemotePath; Build passes libraries.SwitchToPatchedWheels() there. +func resolveLibraries(ctx context.Context, b *bundle.Bundle, extra ...bundle.Mutator) LibLocationMap { + mutators := make([]bundle.Mutator, 0, 4+len(extra)) + mutators = append(mutators, mutator.ResolveVariableReferencesWithoutResources( "artifacts", ), @@ -37,6 +39,8 @@ func resolveLibraries(ctx context.Context, b *bundle.Bundle) LibLocationMap { libraries.ExpandGlobReferences(), libraries.CheckForSameNameLibraries(), ) + mutators = append(mutators, extra...) + bundle.ApplySeqContext(ctx, b, mutators...) if logdiag.HasError(ctx) { return nil @@ -58,15 +62,16 @@ func Build(ctx context.Context, b *bundle.Bundle) LibLocationMap { scripts.Execute(config.ScriptPreBuild), artifacts.Build(), scripts.Execute(config.ScriptPostBuild), - // SwitchToPatchedWheels must be run after ExpandGlobReferences and after build phase because it Artifact.Source and Artifact.Patched populated - libraries.SwitchToPatchedWheels(), ) if logdiag.HasError(ctx) { return nil } - return resolveLibraries(ctx, b) + // SwitchToPatchedWheels must be passed to resolveLibraries so it runs after + // ExpandGlobReferences (which expands *.whl patterns in job library paths) + // and after the build phase (which populates Artifact.Source and Artifact.Patched). + return resolveLibraries(ctx, b, libraries.SwitchToPatchedWheels()) } // FindLibraries discovers which local library files need uploading by reading diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 20a9dfb239a..b98f859f149 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -78,7 +78,6 @@ type ProcessOptions struct { FastValidate bool Validate bool Build bool - FindLibraries bool PreDeployChecks bool Deploy bool @@ -356,7 +355,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } var plan *deployplan.Plan - + var findLibraries bool if opts.ReadPlanPath != "" { if !stateDesc.Engine.IsDirect() { logdiag.LogError(ctx, errors.New("--plan is only supported with direct engine (set bundle.engine to \"direct\" or DATABRICKS_BUNDLE_ENGINE=direct)")) @@ -369,8 +368,8 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // FindLibraries) so the correct local files are found and uploaded to the // remote paths the plan references. opts.Build = false - opts.FindLibraries = true opts.PreDeployChecks = false + findLibraries = true var err error plan, err = deployplan.LoadPlanFromFile(opts.ReadPlanPath) @@ -433,7 +432,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } - } else if opts.FindLibraries { + } else if findLibraries { t2 := time.Now() libs = phases.FindLibraries(ctx, b) b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ From d52c8839e52019a72b94288b768ecec547650fef Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 15 Sep 2026 16:42:44 +0200 Subject: [PATCH 3/3] expand globs --- bundle/artifacts/expand_globs.go | 23 +++++++++++++++++++++++ bundle/phases/build.go | 7 +++++++ 2 files changed, 30 insertions(+) diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index af358d3dda9..484ed05deae 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -3,7 +3,9 @@ package artifacts import ( "context" "fmt" + "maps" "path/filepath" + "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -122,3 +124,24 @@ func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti return diags } + +type expandAllGlobs struct{} + +func (e *expandAllGlobs) Name() string { + return "artifacts.ExpandGlobReferences" +} + +func (e *expandAllGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, name := range slices.Sorted(maps.Keys(b.Config.Artifacts)) { + bundle.ApplyContext(ctx, b, expandGlobs{name: name}) + } + return nil +} + +// ExpandGlobReferences returns a mutator that expands glob patterns in artifact +// file sources for every artifact. Prepare handles artifacts without a build +// command; Build handles those with one. This mutator covers both, making it +// safe to call after Prepare but before a build has run (e.g. FindLibraries). +func ExpandGlobReferences() bundle.Mutator { + return &expandAllGlobs{} +} diff --git a/bundle/phases/build.go b/bundle/phases/build.go index 1db2fbc815e..a6e8a31c1fe 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -32,6 +32,13 @@ func resolveLibraries(ctx context.Context, b *bundle.Bundle, extra ...bundle.Mut "artifacts", ), + // Expand artifact file globs (e.g. dist/*.whl in artifacts[*].files[*].source). + // Prepare() skips this for artifacts with a build command because the files don't + // exist yet; Build() does it after running the command. For FindLibraries (plan + // apply) the artifacts were already built at plan time, so we expand here instead. + // Safe to call from Build too: already-expanded paths are left unchanged. + artifacts.ExpandGlobReferences(), + // libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we // know what are the actual library paths. // libraries.ExpandGlobReferences() has to be run after the libraries are built and thus this