From 593da4a021263340c32ab4ca566c208565e5a2a1 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 14 Sep 2026 10:00:01 +0000 Subject: [PATCH 1/4] Capture UC dependencies inside names that already use references CaptureUCDependencies parsed compound UC names (vector search index Name, quality monitor OutputSchemaName, model service Parent) with strings.SplitN, which split on the dots inside a ${resources...} reference and produced nonsensical catalog/schema fragments. It only worked by accident: the fragments matched nothing, so the name was reconstructed unchanged. Make reference handling explicit: - splitUCName treats a ${...} reference as atomic so a reference in any component no longer breaks parsing. - resolveCatalog/resolveSchema leave a component that is already an explicit reference untouched. - findSchema normalizes a catalog reference back to its literal name (via literalCatalogName), so a literal schema is still captured when its catalog is written as a ${resources.catalogs..name} reference. Co-authored-by: Isaac --- .../bundles/capture-uc-deps-references.md | 1 + .../databricks.yml | 17 +++ .../output.txt | 18 +++ .../capture_uc_dependencies.go | 67 +++++++++- .../capture_uc_dependencies_test.go | 122 ++++++++++++++++++ 5 files changed, 221 insertions(+), 4 deletions(-) create mode 100644 .nextchanges/bundles/capture-uc-deps-references.md diff --git a/.nextchanges/bundles/capture-uc-deps-references.md b/.nextchanges/bundles/capture-uc-deps-references.md new file mode 100644 index 0000000000..9576d356ca --- /dev/null +++ b/.nextchanges/bundles/capture-uc-deps-references.md @@ -0,0 +1 @@ +* Preserve existing `${resources...}` references when capturing implicit UC catalog and schema dependencies, and still capture a literal schema when its catalog is written as a reference. diff --git a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml index 97c420f1a5..e43ba9e74a 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml +++ b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml @@ -3,6 +3,9 @@ bundle: # Tests implicit dependency detection for vector search indexes: # - name (format "catalog.schema.index") should resolve the catalog and schema parts +# - a name that already spells the catalog as a ${resources...} reference keeps that +# reference and still captures the literal schema (catalog_ref_index) +# - a fully referenced name is left untouched (full_ref_index) resources: catalogs: my_catalog: @@ -19,6 +22,20 @@ resources: index_type: DIRECT_ACCESS direct_access_index_spec: schema_json: '{"id":"integer","vector":"array"}' + catalog_ref_index: + name: ${resources.catalogs.my_catalog.name}.myschema.myindex + endpoint_name: my-endpoint + primary_key: id + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + full_ref_index: + name: ${resources.catalogs.my_catalog.name}.${resources.schemas.my_schema.name}.myindex + endpoint_name: my-endpoint + primary_key: id + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' targets: dev: diff --git a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt index 862b7e25ea..e9ad04089f 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt +++ b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt @@ -13,6 +13,24 @@ } }, "vector_search_indexes": { + "catalog_ref_index": { + "direct_access_index_spec": { + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" + }, + "endpoint_name": "my-endpoint", + "index_type": "DIRECT_ACCESS", + "name": "${resources.catalogs.my_catalog.name}.${resources.schemas.my_schema.name}.myindex", + "primary_key": "id" + }, + "full_ref_index": { + "direct_access_index_spec": { + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" + }, + "endpoint_name": "my-endpoint", + "index_type": "DIRECT_ACCESS", + "name": "${resources.catalogs.my_catalog.name}.${resources.schemas.my_schema.name}.myindex", + "primary_key": "id" + }, "my_index": { "direct_access_index_spec": { "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index 61b7467df9..12b4a57f94 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn/dynvar" ) type captureUCDependencies struct{} @@ -40,13 +41,35 @@ func catalogNameRef(key string) string { return fmt.Sprintf("${resources.catalogs.%s.name}", key) } +// literalCatalogName returns the concrete catalog name for a catalog value that is +// either a literal name or a ${resources.catalogs..name} reference to a catalog +// defined in the bundle. A reference that does not resolve to a bundle catalog yields +// "", so an unresolvable reference never accidentally matches a schema by name. +func literalCatalogName(b *bundle.Bundle, catalogName string) string { + if !dynvar.ContainsVariableReference(catalogName) { + return catalogName + } + p, ok := dynvar.PureReferenceToPath(catalogName) + if !ok || len(p) != 4 || p[0].Key() != "resources" || p[1].Key() != "catalogs" || p[3].Key() != "name" { + return "" + } + if c := b.Config.Resources.Catalogs[p[2].Key()]; c != nil { + return c.Name + } + return "" +} + func findSchema(b *bundle.Bundle, catalogName, schemaName string) (string, *resources.Schema) { + // A bundle catalog can be addressed either by literal name or by a + // ${resources.catalogs..name} reference; normalize both sides to the literal + // name so a schema is matched regardless of which form its catalog uses. + catalogName = literalCatalogName(b, catalogName) if catalogName == "" || schemaName == "" { return "", nil } for k, s := range b.Config.Resources.Schemas { - if s != nil && s.CatalogName == catalogName && s.Name == schemaName { + if s != nil && literalCatalogName(b, s.CatalogName) == catalogName && s.Name == schemaName { return k, s } } @@ -71,6 +94,11 @@ func findCatalog(b *bundle.Bundle, catalogName string) (string, *resources.Catal // unchanged. Must be called before resolveCatalog on the same resource since // findSchema needs the original (unmutated) catalogName. func resolveSchema(b *bundle.Bundle, catalogName, schemaName string) string { + // A schema already written as an explicit reference is left as is; it need not + // (and cannot) be matched to a bundle schema by name. + if dynvar.ContainsVariableReference(schemaName) { + return schemaName + } k, s := findSchema(b, catalogName, schemaName) if s != nil { return schemaNameRef(k) @@ -81,6 +109,10 @@ func resolveSchema(b *bundle.Bundle, catalogName, schemaName string) string { // resolveCatalog returns the explicit catalog reference if catalogName matches // a catalog defined in the bundle. Otherwise returns catalogName unchanged. func resolveCatalog(b *bundle.Bundle, catalogName string) string { + // A catalog already written as an explicit reference is left as is. + if dynvar.ContainsVariableReference(catalogName) { + return catalogName + } k, c := findCatalog(b, catalogName) if c != nil { return catalogNameRef(k) @@ -88,6 +120,33 @@ func resolveCatalog(b *bundle.Bundle, catalogName string) string { return catalogName } +// splitUCName splits a UC identifier on "." into at most n parts, like +// strings.SplitN, but treats a ${...} reference as atomic so the dots inside a +// reference (e.g. ${resources.catalogs.c.name}) do not create extra splits. +func splitUCName(name string, n int) []string { + var parts []string + depth, start := 0, 0 + for i := range len(name) { + if len(parts) == n-1 { + break + } + switch name[i] { + case '{': + depth++ + case '}': + if depth > 0 { + depth-- + } + case '.': + if depth == 0 { + parts = append(parts, name[start:i]) + start = i + 1 + } + } + } + return append(parts, name[start:]) +} + // resolveParent rewrites a `schemas/{catalog}.{schema}` parent reference so that // a catalog or schema defined in the same bundle becomes an explicit deploy-time // dependency. AI Gateway securables address their parent schema with this @@ -97,7 +156,7 @@ func resolveParent(b *bundle.Bundle, parent string) string { if !ok { return parent } - parts := strings.SplitN(rest, ".", 2) + parts := splitUCName(rest, 2) if len(parts) != 2 { return parent } @@ -145,7 +204,7 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia continue } // OutputSchemaName is a compound "catalog.schema" string. - parts := strings.SplitN(qm.OutputSchemaName, ".", 2) + parts := splitUCName(qm.OutputSchemaName, 2) if len(parts) != 2 { continue } @@ -160,7 +219,7 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia continue } // Name is a three-part "catalog.schema.index" UC identifier. - parts := strings.SplitN(idx.Name, ".", 3) + parts := splitUCName(idx.Name, 3) if len(parts) != 3 { continue } diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go index db4f888c7c..9872659104 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go @@ -60,6 +60,8 @@ func TestResolveSchema(t *testing.T) { {"empty_catalog", "", "foobar", "foobar"}, {"empty_schema", "catalog1", "", ""}, {"both_empty", "", "", ""}, + {"schema_reference_passthrough", "catalog1", "${resources.schemas.schema1.name}", "${resources.schemas.schema1.name}"}, + {"catalog_reference_unresolved", "${resources.catalogs.x.name}", "foobar", "foobar"}, } for _, tc := range tests { @@ -81,6 +83,7 @@ func TestResolveCatalog(t *testing.T) { {"match_catalog2", "catalog2", "${resources.catalogs.prod_catalog.name}"}, {"no_match", "catalogX", "catalogX"}, {"empty", "", ""}, + {"reference_passthrough", "${resources.catalogs.dev_catalog.name}", "${resources.catalogs.dev_catalog.name}"}, } for _, tc := range tests { @@ -347,3 +350,122 @@ func TestCaptureUCDependenciesNilResources(t *testing.T) { d := bundle.Apply(t.Context(), b, CaptureUCDependencies()) require.Nil(t, d) } + +func TestSplitUCName(t *testing.T) { + tests := []struct { + name string + input string + n int + expected []string + }{ + {"three_part", "catalog.schema.index", 3, []string{"catalog", "schema", "index"}}, + {"two_part", "catalog.schema", 2, []string{"catalog", "schema"}}, + {"extra_dots_kept_in_last", "a.b.c.d", 3, []string{"a", "b", "c.d"}}, + {"fewer_parts_than_n", "a.b", 3, []string{"a", "b"}}, + {"single_component", "a", 3, []string{"a"}}, + {"empty", "", 3, []string{""}}, + {"reference_catalog_atomic", "${resources.catalogs.c.name}.schema.index", 3, []string{"${resources.catalogs.c.name}", "schema", "index"}}, + {"reference_catalog_and_schema_atomic", "${resources.catalogs.c.name}.${resources.schemas.s.name}.index", 3, []string{"${resources.catalogs.c.name}", "${resources.schemas.s.name}", "index"}}, + {"reference_in_two_part", "${resources.catalogs.c.name}.schema", 2, []string{"${resources.catalogs.c.name}", "schema"}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, splitUCName(tc.input, tc.n)) + }) + } +} + +func TestLiteralCatalogName(t *testing.T) { + b := bundleWithCatalogs() + + tests := []struct { + name string + input string + expected string + }{ + {"literal", "catalog1", "catalog1"}, + {"reference_resolves", "${resources.catalogs.dev_catalog.name}", "catalog1"}, + {"reference_missing_catalog", "${resources.catalogs.unknown.name}", ""}, + {"reference_wrong_resource_type", "${resources.schemas.dev_catalog.name}", ""}, + {"impure_reference", "prefix-${resources.catalogs.dev_catalog.name}", ""}, + {"empty", "", ""}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, literalCatalogName(b, tc.input)) + }) + } +} + +// A UC name that already contains ${resources...} references is left intact, while +// any literal catalog/schema component is still captured. This covers the mixed case +// where the catalog is a reference but the sibling schema is a literal, in both +// directions (a bundle schema may itself carry its catalog as a literal or reference). +func TestCaptureUCDependenciesVectorSearchIndexReferences(t *testing.T) { + newBundle := func(indexName string) *bundle.Bundle { + return &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Catalogs: map[string]*resources.Catalog{ + "dev_catalog": {CreateCatalog: catalog.CreateCatalog{Name: "catalog1"}}, + }, + Schemas: map[string]*resources.Schema{ + "schema_lit": {CreateSchema: catalog.CreateSchema{CatalogName: "catalog1", Name: "foobar"}}, + "schema_ref": {CreateSchema: catalog.CreateSchema{CatalogName: "${resources.catalogs.dev_catalog.name}", Name: "barbaz"}}, + }, + VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ + "idx": {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{Name: indexName}}, + }, + }, + }, + } + } + + tests := []struct { + name string + input string + expected string + }{ + { + "catalog_reference_literal_schema", + "${resources.catalogs.dev_catalog.name}.foobar.my_index", + "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_lit.name}.my_index", + }, + { + "catalog_reference_schema_whose_catalog_is_a_reference", + "${resources.catalogs.dev_catalog.name}.barbaz.my_index", + "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_ref.name}.my_index", + }, + { + "fully_referenced_passthrough", + "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_lit.name}.my_index", + "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_lit.name}.my_index", + }, + { + "all_literal", + "catalog1.foobar.my_index", + "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_lit.name}.my_index", + }, + { + "literal_catalog_schema_whose_catalog_is_a_reference", + "catalog1.barbaz.my_index", + "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_ref.name}.my_index", + }, + { + "reference_to_unknown_catalog_unchanged", + "${resources.catalogs.unknown.name}.foobar.my_index", + "${resources.catalogs.unknown.name}.foobar.my_index", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + b := newBundle(tc.input) + d := bundle.Apply(t.Context(), b, CaptureUCDependencies()) + require.Nil(t, d) + assert.Equal(t, tc.expected, b.Config.Resources.VectorSearchIndexes["idx"].Name) + }) + } +} From 556516067f30e39eb37a7abda018058807890fc9 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 14 Sep 2026 10:17:04 +0000 Subject: [PATCH 2/4] Add changelog PR link Co-authored-by: Isaac --- .nextchanges/bundles/capture-uc-deps-references.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/capture-uc-deps-references.md b/.nextchanges/bundles/capture-uc-deps-references.md index 9576d356ca..63b49ad876 100644 --- a/.nextchanges/bundles/capture-uc-deps-references.md +++ b/.nextchanges/bundles/capture-uc-deps-references.md @@ -1 +1 @@ -* Preserve existing `${resources...}` references when capturing implicit UC catalog and schema dependencies, and still capture a literal schema when its catalog is written as a reference. +* Preserve existing `${resources...}` references when capturing implicit UC catalog and schema dependencies, and still capture a literal schema when its catalog is written as a reference. ([#6667](https://github.com/databricks/cli/pull/6667)) From 21b301248af09eee731ed931d6939f13c6ef3bbd Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 14 Sep 2026 11:49:24 +0000 Subject: [PATCH 3/4] Be less clever, more strict. no parsing --- .../databricks.yml | 9 +- .../output.txt | 6 +- .../capture_uc_dependencies.go | 93 +++++++------------ .../capture_uc_dependencies_test.go | 91 ++++++------------ 4 files changed, 65 insertions(+), 134 deletions(-) diff --git a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml index e43ba9e74a..77114e26e3 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml +++ b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml @@ -2,10 +2,9 @@ bundle: name: test-bundle # Tests implicit dependency detection for vector search indexes: -# - name (format "catalog.schema.index") should resolve the catalog and schema parts -# - a name that already spells the catalog as a ${resources...} reference keeps that -# reference and still captures the literal schema (catalog_ref_index) -# - a fully referenced name is left untouched (full_ref_index) +# - name (format "catalog.schema.index") resolves the catalog and schema parts (my_index) +# - a name that already contains a ${resources...} reference is left untouched; a mix of +# references and literals is not supported (mixed_ref_index, full_ref_index) resources: catalogs: my_catalog: @@ -22,7 +21,7 @@ resources: index_type: DIRECT_ACCESS direct_access_index_spec: schema_json: '{"id":"integer","vector":"array"}' - catalog_ref_index: + mixed_ref_index: name: ${resources.catalogs.my_catalog.name}.myschema.myindex endpoint_name: my-endpoint primary_key: id diff --git a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt index e9ad04089f..3cdf0b6522 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt +++ b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt @@ -13,7 +13,7 @@ } }, "vector_search_indexes": { - "catalog_ref_index": { + "full_ref_index": { "direct_access_index_spec": { "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" }, @@ -22,13 +22,13 @@ "name": "${resources.catalogs.my_catalog.name}.${resources.schemas.my_schema.name}.myindex", "primary_key": "id" }, - "full_ref_index": { + "mixed_ref_index": { "direct_access_index_spec": { "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" }, "endpoint_name": "my-endpoint", "index_type": "DIRECT_ACCESS", - "name": "${resources.catalogs.my_catalog.name}.${resources.schemas.my_schema.name}.myindex", + "name": "${resources.catalogs.my_catalog.name}.myschema.myindex", "primary_key": "id" }, "my_index": { diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index 12b4a57f94..88b3853925 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -41,35 +41,13 @@ func catalogNameRef(key string) string { return fmt.Sprintf("${resources.catalogs.%s.name}", key) } -// literalCatalogName returns the concrete catalog name for a catalog value that is -// either a literal name or a ${resources.catalogs..name} reference to a catalog -// defined in the bundle. A reference that does not resolve to a bundle catalog yields -// "", so an unresolvable reference never accidentally matches a schema by name. -func literalCatalogName(b *bundle.Bundle, catalogName string) string { - if !dynvar.ContainsVariableReference(catalogName) { - return catalogName - } - p, ok := dynvar.PureReferenceToPath(catalogName) - if !ok || len(p) != 4 || p[0].Key() != "resources" || p[1].Key() != "catalogs" || p[3].Key() != "name" { - return "" - } - if c := b.Config.Resources.Catalogs[p[2].Key()]; c != nil { - return c.Name - } - return "" -} - func findSchema(b *bundle.Bundle, catalogName, schemaName string) (string, *resources.Schema) { - // A bundle catalog can be addressed either by literal name or by a - // ${resources.catalogs..name} reference; normalize both sides to the literal - // name so a schema is matched regardless of which form its catalog uses. - catalogName = literalCatalogName(b, catalogName) if catalogName == "" || schemaName == "" { return "", nil } for k, s := range b.Config.Resources.Schemas { - if s != nil && literalCatalogName(b, s.CatalogName) == catalogName && s.Name == schemaName { + if s != nil && s.CatalogName == catalogName && s.Name == schemaName { return k, s } } @@ -94,11 +72,6 @@ func findCatalog(b *bundle.Bundle, catalogName string) (string, *resources.Catal // unchanged. Must be called before resolveCatalog on the same resource since // findSchema needs the original (unmutated) catalogName. func resolveSchema(b *bundle.Bundle, catalogName, schemaName string) string { - // A schema already written as an explicit reference is left as is; it need not - // (and cannot) be matched to a bundle schema by name. - if dynvar.ContainsVariableReference(schemaName) { - return schemaName - } k, s := findSchema(b, catalogName, schemaName) if s != nil { return schemaNameRef(k) @@ -109,10 +82,6 @@ func resolveSchema(b *bundle.Bundle, catalogName, schemaName string) string { // resolveCatalog returns the explicit catalog reference if catalogName matches // a catalog defined in the bundle. Otherwise returns catalogName unchanged. func resolveCatalog(b *bundle.Bundle, catalogName string) string { - // A catalog already written as an explicit reference is left as is. - if dynvar.ContainsVariableReference(catalogName) { - return catalogName - } k, c := findCatalog(b, catalogName) if c != nil { return catalogNameRef(k) @@ -120,31 +89,16 @@ func resolveCatalog(b *bundle.Bundle, catalogName string) string { return catalogName } -// splitUCName splits a UC identifier on "." into at most n parts, like -// strings.SplitN, but treats a ${...} reference as atomic so the dots inside a -// reference (e.g. ${resources.catalogs.c.name}) do not create extra splits. -func splitUCName(name string, n int) []string { - var parts []string - depth, start := 0, 0 - for i := range len(name) { - if len(parts) == n-1 { - break - } - switch name[i] { - case '{': - depth++ - case '}': - if depth > 0 { - depth-- - } - case '.': - if depth == 0 { - parts = append(parts, name[start:i]) - start = i + 1 - } - } +// splitUCName splits a compound UC identifier into exactly n dot-separated +// components, returning false if it has a different number of components. Callers +// skip names containing a ${...} reference beforehand, so this is a plain split: +// a reference would push the component count past n and be rejected here anyway. +func splitUCName(name string, n int) ([]string, bool) { + parts := strings.Split(name, ".") + if len(parts) != n { + return nil, false } - return append(parts, name[start:]) + return parts, true } // resolveParent rewrites a `schemas/{catalog}.{schema}` parent reference so that @@ -156,8 +110,8 @@ func resolveParent(b *bundle.Bundle, parent string) string { if !ok { return parent } - parts := splitUCName(rest, 2) - if len(parts) != 2 { + parts, ok := splitUCName(rest, 2) + if !ok { return parent } catalogName, schemaName := parts[0], parts[1] @@ -203,9 +157,14 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia if qm == nil || qm.OutputSchemaName == "" { continue } + // A name that already contains a reference is left as is: we only rewrite a + // fully literal name and do not support a mix of references and literals. + if dynvar.ContainsVariableReference(qm.OutputSchemaName) { + continue + } // OutputSchemaName is a compound "catalog.schema" string. - parts := splitUCName(qm.OutputSchemaName, 2) - if len(parts) != 2 { + parts, ok := splitUCName(qm.OutputSchemaName, 2) + if !ok { continue } catalogName, schemaName := parts[0], parts[1] @@ -218,9 +177,14 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia if idx == nil { continue } + // A name that already contains a reference is left as is; a mix of + // references and literals is not supported. + if dynvar.ContainsVariableReference(idx.Name) { + continue + } // Name is a three-part "catalog.schema.index" UC identifier. - parts := splitUCName(idx.Name, 3) - if len(parts) != 3 { + parts, ok := splitUCName(idx.Name, 3) + if !ok { continue } catalogName, schemaName := parts[0], parts[1] @@ -246,6 +210,11 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia if ms == nil { continue } + // A parent that already contains a reference is left as is; a mix of + // references and literals is not supported. + if dynvar.ContainsVariableReference(ms.Parent) { + continue + } ms.Parent = resolveParent(b, ms.Parent) } diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go index 9872659104..481df5d214 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go @@ -60,8 +60,6 @@ func TestResolveSchema(t *testing.T) { {"empty_catalog", "", "foobar", "foobar"}, {"empty_schema", "catalog1", "", ""}, {"both_empty", "", "", ""}, - {"schema_reference_passthrough", "catalog1", "${resources.schemas.schema1.name}", "${resources.schemas.schema1.name}"}, - {"catalog_reference_unresolved", "${resources.catalogs.x.name}", "foobar", "foobar"}, } for _, tc := range tests { @@ -83,7 +81,6 @@ func TestResolveCatalog(t *testing.T) { {"match_catalog2", "catalog2", "${resources.catalogs.prod_catalog.name}"}, {"no_match", "catalogX", "catalogX"}, {"empty", "", ""}, - {"reference_passthrough", "${resources.catalogs.dev_catalog.name}", "${resources.catalogs.dev_catalog.name}"}, } for _, tc := range tests { @@ -357,52 +354,29 @@ func TestSplitUCName(t *testing.T) { input string n int expected []string + ok bool }{ - {"three_part", "catalog.schema.index", 3, []string{"catalog", "schema", "index"}}, - {"two_part", "catalog.schema", 2, []string{"catalog", "schema"}}, - {"extra_dots_kept_in_last", "a.b.c.d", 3, []string{"a", "b", "c.d"}}, - {"fewer_parts_than_n", "a.b", 3, []string{"a", "b"}}, - {"single_component", "a", 3, []string{"a"}}, - {"empty", "", 3, []string{""}}, - {"reference_catalog_atomic", "${resources.catalogs.c.name}.schema.index", 3, []string{"${resources.catalogs.c.name}", "schema", "index"}}, - {"reference_catalog_and_schema_atomic", "${resources.catalogs.c.name}.${resources.schemas.s.name}.index", 3, []string{"${resources.catalogs.c.name}", "${resources.schemas.s.name}", "index"}}, - {"reference_in_two_part", "${resources.catalogs.c.name}.schema", 2, []string{"${resources.catalogs.c.name}", "schema"}}, + {"three_part", "catalog.schema.index", 3, []string{"catalog", "schema", "index"}, true}, + {"two_part", "catalog.schema", 2, []string{"catalog", "schema"}, true}, + {"too_many_components", "a.b.c.d", 3, nil, false}, + {"too_few_components", "a.b", 3, nil, false}, + {"single_component", "a", 3, nil, false}, + {"empty", "", 3, nil, false}, + {"reference_over_counts", "${resources.catalogs.c.name}.schema.index", 3, nil, false}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.expected, splitUCName(tc.input, tc.n)) + parts, ok := splitUCName(tc.input, tc.n) + assert.Equal(t, tc.ok, ok) + assert.Equal(t, tc.expected, parts) }) } } -func TestLiteralCatalogName(t *testing.T) { - b := bundleWithCatalogs() - - tests := []struct { - name string - input string - expected string - }{ - {"literal", "catalog1", "catalog1"}, - {"reference_resolves", "${resources.catalogs.dev_catalog.name}", "catalog1"}, - {"reference_missing_catalog", "${resources.catalogs.unknown.name}", ""}, - {"reference_wrong_resource_type", "${resources.schemas.dev_catalog.name}", ""}, - {"impure_reference", "prefix-${resources.catalogs.dev_catalog.name}", ""}, - {"empty", "", ""}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.expected, literalCatalogName(b, tc.input)) - }) - } -} - -// A UC name that already contains ${resources...} references is left intact, while -// any literal catalog/schema component is still captured. This covers the mixed case -// where the catalog is a reference but the sibling schema is a literal, in both -// directions (a bundle schema may itself carry its catalog as a literal or reference). +// A vector search index name that already contains a ${resources...} reference is left +// untouched: a mix of references and literals is not supported. Only a fully literal name +// with exactly three components has its catalog and schema captured. func TestCaptureUCDependenciesVectorSearchIndexReferences(t *testing.T) { newBundle := func(indexName string) *bundle.Bundle { return &bundle.Bundle{ @@ -412,8 +386,7 @@ func TestCaptureUCDependenciesVectorSearchIndexReferences(t *testing.T) { "dev_catalog": {CreateCatalog: catalog.CreateCatalog{Name: "catalog1"}}, }, Schemas: map[string]*resources.Schema{ - "schema_lit": {CreateSchema: catalog.CreateSchema{CatalogName: "catalog1", Name: "foobar"}}, - "schema_ref": {CreateSchema: catalog.CreateSchema{CatalogName: "${resources.catalogs.dev_catalog.name}", Name: "barbaz"}}, + "my_schema": {CreateSchema: catalog.CreateSchema{CatalogName: "catalog1", Name: "myschema"}}, }, VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ "idx": {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{Name: indexName}}, @@ -429,34 +402,24 @@ func TestCaptureUCDependenciesVectorSearchIndexReferences(t *testing.T) { expected string }{ { - "catalog_reference_literal_schema", - "${resources.catalogs.dev_catalog.name}.foobar.my_index", - "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_lit.name}.my_index", - }, - { - "catalog_reference_schema_whose_catalog_is_a_reference", - "${resources.catalogs.dev_catalog.name}.barbaz.my_index", - "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_ref.name}.my_index", - }, - { - "fully_referenced_passthrough", - "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_lit.name}.my_index", - "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_lit.name}.my_index", + "all_literal_captured", + "catalog1.myschema.myindex", + "${resources.catalogs.dev_catalog.name}.${resources.schemas.my_schema.name}.myindex", }, { - "all_literal", - "catalog1.foobar.my_index", - "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_lit.name}.my_index", + "catalog_reference_with_literal_schema_unsupported", + "${resources.catalogs.dev_catalog.name}.myschema.myindex", + "${resources.catalogs.dev_catalog.name}.myschema.myindex", }, { - "literal_catalog_schema_whose_catalog_is_a_reference", - "catalog1.barbaz.my_index", - "${resources.catalogs.dev_catalog.name}.${resources.schemas.schema_ref.name}.my_index", + "fully_referenced_unchanged", + "${resources.catalogs.dev_catalog.name}.${resources.schemas.my_schema.name}.myindex", + "${resources.catalogs.dev_catalog.name}.${resources.schemas.my_schema.name}.myindex", }, { - "reference_to_unknown_catalog_unchanged", - "${resources.catalogs.unknown.name}.foobar.my_index", - "${resources.catalogs.unknown.name}.foobar.my_index", + "more_than_three_components_skipped", + "catalog1.myschema.my.index", + "catalog1.myschema.my.index", }, } From da87e827c8dce802bfbc8bbe979be57820df4b9a Mon Sep 17 00:00:00 2001 From: Jan N Rose Date: Mon, 14 Sep 2026 13:52:00 +0200 Subject: [PATCH 4/4] delete changelog entry no longer customer facing --- .nextchanges/bundles/capture-uc-deps-references.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .nextchanges/bundles/capture-uc-deps-references.md diff --git a/.nextchanges/bundles/capture-uc-deps-references.md b/.nextchanges/bundles/capture-uc-deps-references.md deleted file mode 100644 index 63b49ad876..0000000000 --- a/.nextchanges/bundles/capture-uc-deps-references.md +++ /dev/null @@ -1 +0,0 @@ -* Preserve existing `${resources...}` references when capturing implicit UC catalog and schema dependencies, and still capture a literal schema when its catalog is written as a reference. ([#6667](https://github.com/databricks/cli/pull/6667))