diff --git a/acceptance/bundle/invariant/configs/volume_external.yml.tmpl b/acceptance/bundle/invariant/configs/volume_external.yml.tmpl new file mode 100644 index 00000000000..9bb123550f5 --- /dev/null +++ b/acceptance/bundle/invariant/configs/volume_external.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + volumes: + foo: + name: test-volume-$UNIQUE_NAME + catalog_name: main + schema_name: default + volume_type: EXTERNAL + storage_location: s3://test-bucket/path/ diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 22d9d4dff31..11aaf584918 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -38,5 +38,6 @@ EnvMatrix.INPUT_CONFIG = [ "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", - "volume.yml.tmpl" + "volume.yml.tmpl", + "volume_external.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 22d9d4dff31..11aaf584918 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -38,5 +38,6 @@ EnvMatrix.INPUT_CONFIG = [ "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", - "volume.yml.tmpl" + "volume.yml.tmpl", + "volume_external.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 22d9d4dff31..11aaf584918 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -38,5 +38,6 @@ EnvMatrix.INPUT_CONFIG = [ "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", - "volume.yml.tmpl" + "volume.yml.tmpl", + "volume_external.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index bb66a393bef..beabef5ef1e 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -57,6 +57,7 @@ EnvMatrix.INPUT_CONFIG = [ "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", "volume.yml.tmpl", + "volume_external.yml.tmpl", ] [EnvMatrixExclude] @@ -71,6 +72,9 @@ no_postgres_endpoint_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_end # which are environment-specific, so we only test locally with the mock server no_external_location_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=external_location.yml.tmpl"] +# External volumes reference external locations; excluded from cloud for the same reason +no_external_volume_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=volume_external.yml.tmpl"] + # Fake SQL endpoint for local tests [[Server]] Pattern = "POST /api/2.0/sql/statements/" diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index e0dcd9b2886..b35357c7c28 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -100,6 +100,7 @@ type ChangeDesc struct { const ( ReasonBackendDefault = "backend_default" ReasonAlias = "alias" + ReasonURLNormalization = "url_normalization" ReasonRemoteAlreadySet = "remote_already_set" ReasonEmpty = "empty" ReasonCustom = "custom" diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 62a08987eea..35196bdb388 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -6,7 +6,9 @@ import ( "strings" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -112,6 +114,35 @@ func (r *ResourceVolume) DoDelete(ctx context.Context, id string) error { return r.client.Volumes.DeleteByName(ctx, id) } +// OverrideChangeDesc suppresses drift for storage_location when the only difference +// is a trailing slash. The UC API strips trailing slashes on create, so remote returns +// "s3://bucket/path" while the config may have "s3://bucket/path/". +// +// This matches the Terraform provider's suppressLocationDiff behavior. +// https://github.com/databricks/terraform-provider-databricks/blob/v1.65.1/catalog/resource_volume.go#L25 +func (*ResourceVolume) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *catalog.VolumeInfo) error { + if change.Action == deployplan.Skip { + return nil + } + + if path.String() != "storage_location" { + return nil + } + + newStr, newOk := change.New.(string) + remoteStr, remoteOk := change.Remote.(string) + if !newOk || !remoteOk { + return nil + } + + if newStr != remoteStr && strings.TrimRight(newStr, "/") == strings.TrimRight(remoteStr, "/") { + change.Action = deployplan.Skip + change.Reason = deployplan.ReasonURLNormalization + } + + return nil +} + func getNameFromID(id string) (string, error) { items := strings.Split(id, ".") if len(items) == 0 { diff --git a/libs/testserver/volumes.go b/libs/testserver/volumes.go index 66e5d047ab8..76b5fdf3e5f 100644 --- a/libs/testserver/volumes.go +++ b/libs/testserver/volumes.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "strings" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/workspace" @@ -21,7 +22,7 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response { volume.FullName = volume.CatalogName + "." + volume.SchemaName + "." + volume.Name - if volume.StorageLocation != "" { + if volume.StorageLocation != "" && volume.VolumeType != catalog.VolumeTypeExternal { return Response{ StatusCode: 400, Body: map[string]string{ @@ -30,8 +31,15 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response { }, } } + volume.VolumeId = nextUUID() - volume.StorageLocation = fmt.Sprintf("s3://%s/metastore/%s/volumes/%s", testMetastoreName, TestMetastore.MetastoreId, volume.VolumeId) + + if volume.StorageLocation != "" { + // Strip trailing slash to mimic UC API normalization behavior. + volume.StorageLocation = strings.TrimRight(volume.StorageLocation, "/") + } else { + volume.StorageLocation = fmt.Sprintf("s3://%s/metastore/%s/volumes/%s", testMetastoreName, TestMetastore.MetastoreId, volume.VolumeId) + } volume.CreatedAt = nowMilli() volume.UpdatedAt = volume.CreatedAt