From 06d91004f4c47457fd0184ce70f1ecf8e40fbd7d Mon Sep 17 00:00:00 2001 From: Hector Castejon Diaz Date: Wed, 13 Sep 2023 11:09:20 +0200 Subject: [PATCH 1/8] Use custom marshaller --- bundle/config/resources/job.go | 11 +++++++++++ bundle/config/resources/mlflow_experiment.go | 11 +++++++++++ bundle/config/resources/mlflow_model.go | 11 +++++++++++ bundle/config/resources/model_serving_endpoint.go | 11 +++++++++++ bundle/config/resources/pipeline.go | 12 ++++++++++++ bundle/config/workspace.go | 11 +++++++++++ go.mod | 2 ++ go.sum | 4 ++-- 8 files changed, 71 insertions(+), 2 deletions(-) diff --git a/bundle/config/resources/job.go b/bundle/config/resources/job.go index 7fc5b7610bf..1252c3c6c9b 100644 --- a/bundle/config/resources/job.go +++ b/bundle/config/resources/job.go @@ -2,6 +2,7 @@ package resources import ( "github.com/databricks/cli/bundle/config/paths" + marshal "github.com/databricks/databricks-sdk-go/json" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/imdario/mergo" ) @@ -15,6 +16,16 @@ type Job struct { *jobs.JobSettings } +func (s *Job) UnmarshalJSON(b []byte) error { + type C Job + return marshal.Unmarshal(b, (*C)(s)) +} + +func (s Job) MarshalJSON() ([]byte, error) { + type C Job + return marshal.Marshal((C)(s)) +} + // MergeJobClusters merges job clusters with the same key. // The job clusters field is a slice, and as such, overrides are appended to it. // We can identify a job cluster by its key, however, so we can use this key diff --git a/bundle/config/resources/mlflow_experiment.go b/bundle/config/resources/mlflow_experiment.go index d843cf226ba..c8c0229e73d 100644 --- a/bundle/config/resources/mlflow_experiment.go +++ b/bundle/config/resources/mlflow_experiment.go @@ -2,6 +2,7 @@ package resources import ( "github.com/databricks/cli/bundle/config/paths" + marshal "github.com/databricks/databricks-sdk-go/json" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -12,3 +13,13 @@ type MlflowExperiment struct { *ml.Experiment } + +func (s *MlflowExperiment) UnmarshalJSON(b []byte) error { + type C MlflowExperiment + return marshal.Unmarshal(b, (*C)(s)) +} + +func (s MlflowExperiment) MarshalJSON() ([]byte, error) { + type C MlflowExperiment + return marshal.Marshal((C)(s)) +} diff --git a/bundle/config/resources/mlflow_model.go b/bundle/config/resources/mlflow_model.go index 92617c95a39..4d364b65c2a 100644 --- a/bundle/config/resources/mlflow_model.go +++ b/bundle/config/resources/mlflow_model.go @@ -2,6 +2,7 @@ package resources import ( "github.com/databricks/cli/bundle/config/paths" + marshal "github.com/databricks/databricks-sdk-go/json" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -12,3 +13,13 @@ type MlflowModel struct { *ml.Model } + +func (s *MlflowModel) UnmarshalJSON(b []byte) error { + type C MlflowModel + return marshal.Unmarshal(b, (*C)(s)) +} + +func (s MlflowModel) MarshalJSON() ([]byte, error) { + type C MlflowModel + return marshal.Marshal((C)(s)) +} diff --git a/bundle/config/resources/model_serving_endpoint.go b/bundle/config/resources/model_serving_endpoint.go index dccecaa6f86..12ac5a631b0 100644 --- a/bundle/config/resources/model_serving_endpoint.go +++ b/bundle/config/resources/model_serving_endpoint.go @@ -2,6 +2,7 @@ package resources import ( "github.com/databricks/cli/bundle/config/paths" + marshal "github.com/databricks/databricks-sdk-go/json" "github.com/databricks/databricks-sdk-go/service/serving" ) @@ -22,3 +23,13 @@ type ModelServingEndpoint struct { // Implementation could be different based on the resource type. Permissions []Permission `json:"permissions,omitempty"` } + +func (s *ModelServingEndpoint) UnmarshalJSON(b []byte) error { + type C ModelServingEndpoint + return marshal.Unmarshal(b, (*C)(s)) +} + +func (s ModelServingEndpoint) MarshalJSON() ([]byte, error) { + type C ModelServingEndpoint + return marshal.Marshal((C)(s)) +} diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 94c0f2b0293..fdfcd126a58 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/databricks/cli/bundle/config/paths" + marshal "github.com/databricks/databricks-sdk-go/json" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/imdario/mergo" ) @@ -17,6 +18,7 @@ type Pipeline struct { *pipelines.PipelineSpec } +<<<<<<< HEAD // MergeClusters merges cluster definitions with same label. // The clusters field is a slice, and as such, overrides are appended to it. // We can identify a cluster by its label, however, so we can use this label @@ -62,4 +64,14 @@ func (p *Pipeline) MergeClusters() error { // Overwrite resulting slice. p.Clusters = output return nil +======= +func (s *Pipeline) UnmarshalJSON(b []byte) error { + type C Pipeline + return marshal.Unmarshal(b, (*C)(s)) +} + +func (s Pipeline) MarshalJSON() ([]byte, error) { + type C Pipeline + return marshal.Marshal((C)(s)) +>>>>>>> ed1d479 ([DO NOT MERGE] Use custom marshaller) } diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index 90cd59c6f32..a8471f3ef64 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" + marshal "github.com/databricks/databricks-sdk-go/json" "github.com/databricks/databricks-sdk-go/service/iam" ) @@ -69,6 +70,16 @@ type User struct { *iam.User } +func (s *User) UnmarshalJSON(b []byte) error { + type C User + return marshal.Unmarshal(b, (*C)(s)) +} + +func (s User) MarshalJSON() ([]byte, error) { + type C User + return marshal.Marshal((C)(s)) +} + func (w *Workspace) Client() (*databricks.WorkspaceClient, error) { cfg := databricks.Config{ // Generic diff --git a/go.mod b/go.mod index fe9bf9ccefd..7d76fe53e38 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/databricks/cli go 1.21 +replace github.com/databricks/databricks-sdk-go => github.com/databricks/databricks-sdk-go v0.19.2-0.20230913090047-4b0ed4e462ad + require ( github.com/briandowns/spinner v1.23.0 // Apache 2.0 github.com/databricks/databricks-sdk-go v0.22.0 // Apache 2.0 diff --git a/go.sum b/go.sum index b040d0dc616..4fbbfff53a8 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,8 @@ github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEM github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/databricks/databricks-sdk-go v0.22.0 h1:CIwNZcOV7wYZmRLl1NWA+07f2j6H9h5L6MhR5O/4dRw= -github.com/databricks/databricks-sdk-go v0.22.0/go.mod h1:COiklTN3IdieazXcs4TnMou5GQFwIM7uhMGrz7nEAAk= +github.com/databricks/databricks-sdk-go v0.19.2-0.20230913090047-4b0ed4e462ad h1:QLo6CwbiK3u68RhB0l+yBT8Zflw/cHMV/AUTVYmwFbU= +github.com/databricks/databricks-sdk-go v0.19.2-0.20230913090047-4b0ed4e462ad/go.mod h1:Bt/3i3ry/rQdE6Y+psvkAENlp+LzJHaQK5PsLIstQb4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From e0e8bfca9a4064431773db117295295457e54be8 Mon Sep 17 00:00:00 2001 From: Hector Castejon Diaz Date: Wed, 13 Sep 2023 13:19:26 +0200 Subject: [PATCH 2/8] Update dependency --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7d76fe53e38..d2d94d38044 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/databricks/cli go 1.21 -replace github.com/databricks/databricks-sdk-go => github.com/databricks/databricks-sdk-go v0.19.2-0.20230913090047-4b0ed4e462ad +replace github.com/databricks/databricks-sdk-go => github.com/databricks/databricks-sdk-go v0.19.2-0.20230913101814-20732bd320e2 require ( github.com/briandowns/spinner v1.23.0 // Apache 2.0 diff --git a/go.sum b/go.sum index 4fbbfff53a8..18792141dac 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/databricks/databricks-sdk-go v0.19.2-0.20230913090047-4b0ed4e462ad h1:QLo6CwbiK3u68RhB0l+yBT8Zflw/cHMV/AUTVYmwFbU= github.com/databricks/databricks-sdk-go v0.19.2-0.20230913090047-4b0ed4e462ad/go.mod h1:Bt/3i3ry/rQdE6Y+psvkAENlp+LzJHaQK5PsLIstQb4= +github.com/databricks/databricks-sdk-go v0.19.2-0.20230913101814-20732bd320e2 h1:uW/UW2ZdC+13XXAVkPPGUCQJiDgxHmDwK+q+P8u1BMU= +github.com/databricks/databricks-sdk-go v0.19.2-0.20230913101814-20732bd320e2/go.mod h1:Bt/3i3ry/rQdE6Y+psvkAENlp+LzJHaQK5PsLIstQb4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 9c377f59480fc2c5006b39c08aa991aaec4604e8 Mon Sep 17 00:00:00 2001 From: Hector Castejon Diaz Date: Wed, 13 Sep 2023 16:06:14 +0200 Subject: [PATCH 3/8] Add integration test --- internal/jobs_test.go | 10 ++++++ .../testjsons/create_job_without_cluster.json | 35 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 internal/jobs_test.go create mode 100644 internal/testjsons/create_job_without_cluster.json diff --git a/internal/jobs_test.go b/internal/jobs_test.go new file mode 100644 index 00000000000..8a3d98cdb29 --- /dev/null +++ b/internal/jobs_test.go @@ -0,0 +1,10 @@ +package internal + +import ( + "testing" +) + +func TestCreateJob(t *testing.T) { + t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) + RequireSuccessfulRun(t, "jobs", "create", "--json", "@testjsons/create_job_without_cluster.json", "--log-level=debug") +} diff --git a/internal/testjsons/create_job_without_cluster.json b/internal/testjsons/create_job_without_cluster.json new file mode 100644 index 00000000000..e92ce82684e --- /dev/null +++ b/internal/testjsons/create_job_without_cluster.json @@ -0,0 +1,35 @@ +{ + "name": "create-job-without-workers", + "job_clusters": [{ + "job_cluster_key": "create-job-without-workers-cluster", + "new_cluster": { + "num_workers": 0, + "spark_version": "10.4.x-scala2.12", + "node_type_id": "Standard_DS3_v2" + } + }], + "tasks": [{ + "job_cluster_key": "create-job-without-workers-cluster", + "task_key": "create-job-without-workers-cluster1", + "libraries": [ + { + "jar": "dbfs:/max/jars/exampleJarTask.jar" + } + ], + "spark_jar_task": { + "main_class_name": "com.databricks.quickstart.exampleTask" + } + }, + { + "job_cluster_key": "create-job-without-workers-cluster", + "task_key": "create-job-without-workers-cluster2", + "libraries": [ + { + "jar": "dbfs:/max/jars/exampleJarTask.jar" + } + ], + "spark_jar_task": { + "main_class_name": "com.databricks.quickstart.exampleTask" + } + }] +} From 316bea540e509e26b8b874d597af10b2346ab66a Mon Sep 17 00:00:00 2001 From: Hector Castejon Diaz Date: Thu, 14 Sep 2023 13:55:23 +0200 Subject: [PATCH 4/8] Address PR comments --- bundle/config/resources/job.go | 8 +++----- bundle/config/resources/mlflow_experiment.go | 8 +++----- bundle/config/resources/mlflow_model.go | 8 +++----- bundle/config/resources/model_serving_endpoint.go | 8 +++----- bundle/config/resources/pipeline.go | 13 +++++-------- bundle/config/workspace.go | 8 +++----- go.mod | 2 +- go.sum | 2 ++ internal/jobs_test.go | 2 +- 9 files changed, 24 insertions(+), 35 deletions(-) diff --git a/bundle/config/resources/job.go b/bundle/config/resources/job.go index 1252c3c6c9b..edda8a92177 100644 --- a/bundle/config/resources/job.go +++ b/bundle/config/resources/job.go @@ -2,7 +2,7 @@ package resources import ( "github.com/databricks/cli/bundle/config/paths" - marshal "github.com/databricks/databricks-sdk-go/json" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/imdario/mergo" ) @@ -17,13 +17,11 @@ type Job struct { } func (s *Job) UnmarshalJSON(b []byte) error { - type C Job - return marshal.Unmarshal(b, (*C)(s)) + return marshal.Unmarshal(b, s) } func (s Job) MarshalJSON() ([]byte, error) { - type C Job - return marshal.Marshal((C)(s)) + return marshal.Marshal(s) } // MergeJobClusters merges job clusters with the same key. diff --git a/bundle/config/resources/mlflow_experiment.go b/bundle/config/resources/mlflow_experiment.go index c8c0229e73d..e4a9a8a887a 100644 --- a/bundle/config/resources/mlflow_experiment.go +++ b/bundle/config/resources/mlflow_experiment.go @@ -2,7 +2,7 @@ package resources import ( "github.com/databricks/cli/bundle/config/paths" - marshal "github.com/databricks/databricks-sdk-go/json" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -15,11 +15,9 @@ type MlflowExperiment struct { } func (s *MlflowExperiment) UnmarshalJSON(b []byte) error { - type C MlflowExperiment - return marshal.Unmarshal(b, (*C)(s)) + return marshal.Unmarshal(b, s) } func (s MlflowExperiment) MarshalJSON() ([]byte, error) { - type C MlflowExperiment - return marshal.Marshal((C)(s)) + return marshal.Marshal(s) } diff --git a/bundle/config/resources/mlflow_model.go b/bundle/config/resources/mlflow_model.go index 4d364b65c2a..51fb0e08722 100644 --- a/bundle/config/resources/mlflow_model.go +++ b/bundle/config/resources/mlflow_model.go @@ -2,7 +2,7 @@ package resources import ( "github.com/databricks/cli/bundle/config/paths" - marshal "github.com/databricks/databricks-sdk-go/json" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -15,11 +15,9 @@ type MlflowModel struct { } func (s *MlflowModel) UnmarshalJSON(b []byte) error { - type C MlflowModel - return marshal.Unmarshal(b, (*C)(s)) + return marshal.Unmarshal(b, s) } func (s MlflowModel) MarshalJSON() ([]byte, error) { - type C MlflowModel - return marshal.Marshal((C)(s)) + return marshal.Marshal(s) } diff --git a/bundle/config/resources/model_serving_endpoint.go b/bundle/config/resources/model_serving_endpoint.go index 12ac5a631b0..3847e6a6cda 100644 --- a/bundle/config/resources/model_serving_endpoint.go +++ b/bundle/config/resources/model_serving_endpoint.go @@ -2,7 +2,7 @@ package resources import ( "github.com/databricks/cli/bundle/config/paths" - marshal "github.com/databricks/databricks-sdk-go/json" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/serving" ) @@ -25,11 +25,9 @@ type ModelServingEndpoint struct { } func (s *ModelServingEndpoint) UnmarshalJSON(b []byte) error { - type C ModelServingEndpoint - return marshal.Unmarshal(b, (*C)(s)) + return marshal.Unmarshal(b, s) } func (s ModelServingEndpoint) MarshalJSON() ([]byte, error) { - type C ModelServingEndpoint - return marshal.Marshal((C)(s)) + return marshal.Marshal(s) } diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index fdfcd126a58..174f6e4d2a4 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -4,7 +4,7 @@ import ( "strings" "github.com/databricks/cli/bundle/config/paths" - marshal "github.com/databricks/databricks-sdk-go/json" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/imdario/mergo" ) @@ -18,7 +18,6 @@ type Pipeline struct { *pipelines.PipelineSpec } -<<<<<<< HEAD // MergeClusters merges cluster definitions with same label. // The clusters field is a slice, and as such, overrides are appended to it. // We can identify a cluster by its label, however, so we can use this label @@ -64,14 +63,12 @@ func (p *Pipeline) MergeClusters() error { // Overwrite resulting slice. p.Clusters = output return nil -======= +} + func (s *Pipeline) UnmarshalJSON(b []byte) error { - type C Pipeline - return marshal.Unmarshal(b, (*C)(s)) + return marshal.Unmarshal(b, s) } func (s Pipeline) MarshalJSON() ([]byte, error) { - type C Pipeline - return marshal.Marshal((C)(s)) ->>>>>>> ed1d479 ([DO NOT MERGE] Use custom marshaller) + return marshal.Marshal(s) } diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index a8471f3ef64..f29d7c562d3 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -7,7 +7,7 @@ import ( "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" - marshal "github.com/databricks/databricks-sdk-go/json" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/iam" ) @@ -71,13 +71,11 @@ type User struct { } func (s *User) UnmarshalJSON(b []byte) error { - type C User - return marshal.Unmarshal(b, (*C)(s)) + return marshal.Unmarshal(b, s) } func (s User) MarshalJSON() ([]byte, error) { - type C User - return marshal.Marshal((C)(s)) + return marshal.Marshal(s) } func (w *Workspace) Client() (*databricks.WorkspaceClient, error) { diff --git a/go.mod b/go.mod index d2d94d38044..aa3fcf1ec60 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/databricks/cli go 1.21 -replace github.com/databricks/databricks-sdk-go => github.com/databricks/databricks-sdk-go v0.19.2-0.20230913101814-20732bd320e2 +replace github.com/databricks/databricks-sdk-go => github.com/databricks/databricks-sdk-go v0.19.2-0.20230914113857-3aa4e5b801a8 require ( github.com/briandowns/spinner v1.23.0 // Apache 2.0 diff --git a/go.sum b/go.sum index 18792141dac..20b05ec7c87 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/databricks/databricks-sdk-go v0.19.2-0.20230913090047-4b0ed4e462ad h1 github.com/databricks/databricks-sdk-go v0.19.2-0.20230913090047-4b0ed4e462ad/go.mod h1:Bt/3i3ry/rQdE6Y+psvkAENlp+LzJHaQK5PsLIstQb4= github.com/databricks/databricks-sdk-go v0.19.2-0.20230913101814-20732bd320e2 h1:uW/UW2ZdC+13XXAVkPPGUCQJiDgxHmDwK+q+P8u1BMU= github.com/databricks/databricks-sdk-go v0.19.2-0.20230913101814-20732bd320e2/go.mod h1:Bt/3i3ry/rQdE6Y+psvkAENlp+LzJHaQK5PsLIstQb4= +github.com/databricks/databricks-sdk-go v0.19.2-0.20230914113857-3aa4e5b801a8 h1:yPN8OzuthpFq/KeP9/N5sIQffOPyZGTCAcb7JM95RNk= +github.com/databricks/databricks-sdk-go v0.19.2-0.20230914113857-3aa4e5b801a8/go.mod h1:Bt/3i3ry/rQdE6Y+psvkAENlp+LzJHaQK5PsLIstQb4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/jobs_test.go b/internal/jobs_test.go index 8a3d98cdb29..6c98c124dc0 100644 --- a/internal/jobs_test.go +++ b/internal/jobs_test.go @@ -4,7 +4,7 @@ import ( "testing" ) -func TestCreateJob(t *testing.T) { +func TestAccCreateJob(t *testing.T) { t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) RequireSuccessfulRun(t, "jobs", "create", "--json", "@testjsons/create_job_without_cluster.json", "--log-level=debug") } From 25d5040efa7c9bfa8e6ffe2c6fa383861a28280a Mon Sep 17 00:00:00 2001 From: Hector Castejon Diaz Date: Wed, 4 Oct 2023 14:00:36 +0200 Subject: [PATCH 5/8] Delete job after test --- internal/acc/debug.go | 3 ++- internal/jobs_test.go | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/internal/acc/debug.go b/internal/acc/debug.go index 467642e2402..11663113248 100644 --- a/internal/acc/debug.go +++ b/internal/acc/debug.go @@ -5,13 +5,14 @@ import ( "os" "path" "path/filepath" + "strings" "testing" ) // Detects if test is run from "debug test" feature in VS Code. func isInDebug() bool { ex, _ := os.Executable() - return path.Base(ex) == "__debug_bin" + return strings.HasPrefix(path.Base(ex), "__debug_bin") } // Loads debug environment from ~/.databricks/debug-env.json. diff --git a/internal/jobs_test.go b/internal/jobs_test.go index 6c98c124dc0..2cd1e2336f6 100644 --- a/internal/jobs_test.go +++ b/internal/jobs_test.go @@ -1,10 +1,24 @@ package internal import ( + "encoding/json" + "fmt" "testing" + + "github.com/databricks/cli/internal/acc" + "github.com/stretchr/testify/assert" ) func TestAccCreateJob(t *testing.T) { - t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) - RequireSuccessfulRun(t, "jobs", "create", "--json", "@testjsons/create_job_without_cluster.json", "--log-level=debug") + acc.WorkspaceTest(t) + env := GetEnvOrSkipTest(t, "CLOUD_ENV") + if env != "azure" { + t.Skipf("Not running test on cloud %s", env) + } + stdout, stderr := RequireSuccessfulRun(t, "jobs", "create", "--json", "@testjsons/create_job_without_cluster.json", "--log-level=debug") + assert.Equal(t, "", stderr.String()) + var output map[string]int + err := json.Unmarshal(stdout.Bytes(), &output) + assert.NoError(t, err) + RequireSuccessfulRun(t, "jobs", "delete", fmt.Sprint(output["job_id"]), "--log-level=debug") } From 7711a9b3efc25cbc4f4809472090f8b918377303 Mon Sep 17 00:00:00 2001 From: Miles Yucht Date: Fri, 13 Oct 2023 09:58:39 +0200 Subject: [PATCH 6/8] Use latest version of the Go SDK --- .codegen/_openapi_sha | 2 +- .gitattributes | 1 - cmd/account/billable-usage/billable-usage.go | 5 +- cmd/account/cmd.go | 2 - cmd/account/network-policy/network-policy.go | 243 ------------------ .../storage-credentials.go | 12 +- cmd/workspace/clusters/clusters.go | 4 +- cmd/workspace/connections/connections.go | 2 + .../instance-pools/instance-pools.go | 7 - cmd/workspace/jobs/jobs.go | 8 +- cmd/workspace/pipelines/pipelines.go | 10 +- .../storage-credentials.go | 13 +- .../workspace-bindings/workspace-bindings.go | 161 +++++++++++- cmd/workspace/workspace/workspace.go | 6 +- go.mod | 4 +- go.sum | 8 +- 16 files changed, 189 insertions(+), 299 deletions(-) delete mode 100755 cmd/account/network-policy/network-policy.go diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 7d4ee2a6780..e36ae5312a8 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -bcbf6e851e3d82fd910940910dd31c10c059746c \ No newline at end of file +493a76554afd3afdd15dc858773d01643f80352a \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index 61527fed69b..f50218feda5 100755 --- a/.gitattributes +++ b/.gitattributes @@ -10,7 +10,6 @@ cmd/account/ip-access-lists/ip-access-lists.go linguist-generated=true cmd/account/log-delivery/log-delivery.go linguist-generated=true cmd/account/metastore-assignments/metastore-assignments.go linguist-generated=true cmd/account/metastores/metastores.go linguist-generated=true -cmd/account/network-policy/network-policy.go linguist-generated=true cmd/account/networks/networks.go linguist-generated=true cmd/account/o-auth-enrollment/o-auth-enrollment.go linguist-generated=true cmd/account/o-auth-published-apps/o-auth-published-apps.go linguist-generated=true diff --git a/cmd/account/billable-usage/billable-usage.go b/cmd/account/billable-usage/billable-usage.go index b5b9749dcad..1dde38f64bd 100755 --- a/cmd/account/billable-usage/billable-usage.go +++ b/cmd/account/billable-usage/billable-usage.go @@ -4,6 +4,7 @@ package billable_usage import ( "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/databricks-sdk-go/service/billing" "github.com/spf13/cobra" ) @@ -80,11 +81,11 @@ func newDownload() *cobra.Command { downloadReq.StartMonth = args[0] downloadReq.EndMonth = args[1] - err = a.BillableUsage.Download(ctx, downloadReq) + response, err := a.BillableUsage.Download(ctx, downloadReq) if err != nil { return err } - return nil + return cmdio.Render(ctx, response) } // Disable completions since they are not applicable. diff --git a/cmd/account/cmd.go b/cmd/account/cmd.go index 38be7314675..744b3670952 100644 --- a/cmd/account/cmd.go +++ b/cmd/account/cmd.go @@ -16,7 +16,6 @@ import ( log_delivery "github.com/databricks/cli/cmd/account/log-delivery" account_metastore_assignments "github.com/databricks/cli/cmd/account/metastore-assignments" account_metastores "github.com/databricks/cli/cmd/account/metastores" - account_network_policy "github.com/databricks/cli/cmd/account/network-policy" networks "github.com/databricks/cli/cmd/account/networks" o_auth_enrollment "github.com/databricks/cli/cmd/account/o-auth-enrollment" o_auth_published_apps "github.com/databricks/cli/cmd/account/o-auth-published-apps" @@ -50,7 +49,6 @@ func New() *cobra.Command { cmd.AddCommand(log_delivery.New()) cmd.AddCommand(account_metastore_assignments.New()) cmd.AddCommand(account_metastores.New()) - cmd.AddCommand(account_network_policy.New()) cmd.AddCommand(networks.New()) cmd.AddCommand(o_auth_enrollment.New()) cmd.AddCommand(o_auth_published_apps.New()) diff --git a/cmd/account/network-policy/network-policy.go b/cmd/account/network-policy/network-policy.go deleted file mode 100755 index 60db933ab66..00000000000 --- a/cmd/account/network-policy/network-policy.go +++ /dev/null @@ -1,243 +0,0 @@ -// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. - -package network_policy - -import ( - "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/flags" - "github.com/databricks/databricks-sdk-go/service/settings" - "github.com/spf13/cobra" -) - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var cmdOverrides []func(*cobra.Command) - -func New() *cobra.Command { - cmd := &cobra.Command{ - Use: "network-policy", - Short: `Network policy is a set of rules that defines what can be accessed from your Databricks network.`, - Long: `Network policy is a set of rules that defines what can be accessed from your - Databricks network. E.g.: You can choose to block your SQL UDF to access - internet from your Databricks serverless clusters. - - There is only one instance of this setting per account. Since this setting has - a default value, this setting is present on all accounts even though it's - never set on a given account. Deletion reverts the value of the setting back - to the default value.`, - GroupID: "settings", - Annotations: map[string]string{ - "package": "settings", - }, - - // This service is being previewed; hide from help output. - Hidden: true, - } - - // Apply optional overrides to this command. - for _, fn := range cmdOverrides { - fn(cmd) - } - - return cmd -} - -// start delete-account-network-policy command - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var deleteAccountNetworkPolicyOverrides []func( - *cobra.Command, - *settings.DeleteAccountNetworkPolicyRequest, -) - -func newDeleteAccountNetworkPolicy() *cobra.Command { - cmd := &cobra.Command{} - - var deleteAccountNetworkPolicyReq settings.DeleteAccountNetworkPolicyRequest - - // TODO: short flags - - cmd.Use = "delete-account-network-policy ETAG" - cmd.Short = `Delete Account Network Policy.` - cmd.Long = `Delete Account Network Policy. - - Reverts back all the account network policies back to default.` - - cmd.Annotations = make(map[string]string) - - cmd.Args = func(cmd *cobra.Command, args []string) error { - check := cobra.ExactArgs(1) - return check(cmd, args) - } - - cmd.PreRunE = root.MustAccountClient - cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() - a := root.AccountClient(ctx) - - deleteAccountNetworkPolicyReq.Etag = args[0] - - response, err := a.NetworkPolicy.DeleteAccountNetworkPolicy(ctx, deleteAccountNetworkPolicyReq) - if err != nil { - return err - } - return cmdio.Render(ctx, response) - } - - // Disable completions since they are not applicable. - // Can be overridden by manual implementation in `override.go`. - cmd.ValidArgsFunction = cobra.NoFileCompletions - - // Apply optional overrides to this command. - for _, fn := range deleteAccountNetworkPolicyOverrides { - fn(cmd, &deleteAccountNetworkPolicyReq) - } - - return cmd -} - -func init() { - cmdOverrides = append(cmdOverrides, func(cmd *cobra.Command) { - cmd.AddCommand(newDeleteAccountNetworkPolicy()) - }) -} - -// start read-account-network-policy command - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var readAccountNetworkPolicyOverrides []func( - *cobra.Command, - *settings.ReadAccountNetworkPolicyRequest, -) - -func newReadAccountNetworkPolicy() *cobra.Command { - cmd := &cobra.Command{} - - var readAccountNetworkPolicyReq settings.ReadAccountNetworkPolicyRequest - - // TODO: short flags - - cmd.Use = "read-account-network-policy ETAG" - cmd.Short = `Get Account Network Policy.` - cmd.Long = `Get Account Network Policy. - - Gets the value of Account level Network Policy.` - - cmd.Annotations = make(map[string]string) - - cmd.Args = func(cmd *cobra.Command, args []string) error { - check := cobra.ExactArgs(1) - return check(cmd, args) - } - - cmd.PreRunE = root.MustAccountClient - cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() - a := root.AccountClient(ctx) - - readAccountNetworkPolicyReq.Etag = args[0] - - response, err := a.NetworkPolicy.ReadAccountNetworkPolicy(ctx, readAccountNetworkPolicyReq) - if err != nil { - return err - } - return cmdio.Render(ctx, response) - } - - // Disable completions since they are not applicable. - // Can be overridden by manual implementation in `override.go`. - cmd.ValidArgsFunction = cobra.NoFileCompletions - - // Apply optional overrides to this command. - for _, fn := range readAccountNetworkPolicyOverrides { - fn(cmd, &readAccountNetworkPolicyReq) - } - - return cmd -} - -func init() { - cmdOverrides = append(cmdOverrides, func(cmd *cobra.Command) { - cmd.AddCommand(newReadAccountNetworkPolicy()) - }) -} - -// start update-account-network-policy command - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var updateAccountNetworkPolicyOverrides []func( - *cobra.Command, - *settings.UpdateAccountNetworkPolicyRequest, -) - -func newUpdateAccountNetworkPolicy() *cobra.Command { - cmd := &cobra.Command{} - - var updateAccountNetworkPolicyReq settings.UpdateAccountNetworkPolicyRequest - var updateAccountNetworkPolicyJson flags.JsonFlag - - // TODO: short flags - cmd.Flags().Var(&updateAccountNetworkPolicyJson, "json", `either inline JSON string or @path/to/file.json with request body`) - - cmd.Flags().BoolVar(&updateAccountNetworkPolicyReq.AllowMissing, "allow-missing", updateAccountNetworkPolicyReq.AllowMissing, `This should always be set to true for Settings RPCs.`) - // TODO: complex arg: setting - - cmd.Use = "update-account-network-policy" - cmd.Short = `Update Account Network Policy.` - cmd.Long = `Update Account Network Policy. - - Updates the policy content of Account level Network Policy.` - - cmd.Annotations = make(map[string]string) - - cmd.Args = func(cmd *cobra.Command, args []string) error { - check := cobra.ExactArgs(0) - if cmd.Flags().Changed("json") { - check = cobra.ExactArgs(0) - } - return check(cmd, args) - } - - cmd.PreRunE = root.MustAccountClient - cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() - a := root.AccountClient(ctx) - - if cmd.Flags().Changed("json") { - err = updateAccountNetworkPolicyJson.Unmarshal(&updateAccountNetworkPolicyReq) - if err != nil { - return err - } - } else { - } - - response, err := a.NetworkPolicy.UpdateAccountNetworkPolicy(ctx, updateAccountNetworkPolicyReq) - if err != nil { - return err - } - return cmdio.Render(ctx, response) - } - - // Disable completions since they are not applicable. - // Can be overridden by manual implementation in `override.go`. - cmd.ValidArgsFunction = cobra.NoFileCompletions - - // Apply optional overrides to this command. - for _, fn := range updateAccountNetworkPolicyOverrides { - fn(cmd, &updateAccountNetworkPolicyReq) - } - - return cmd -} - -func init() { - cmdOverrides = append(cmdOverrides, func(cmd *cobra.Command) { - cmd.AddCommand(newUpdateAccountNetworkPolicy()) - }) -} - -// end service AccountNetworkPolicy diff --git a/cmd/account/storage-credentials/storage-credentials.go b/cmd/account/storage-credentials/storage-credentials.go index 451b7112128..670bb26d750 100755 --- a/cmd/account/storage-credentials/storage-credentials.go +++ b/cmd/account/storage-credentials/storage-credentials.go @@ -128,7 +128,7 @@ func newDelete() *cobra.Command { cmd.Flags().BoolVar(&deleteReq.Force, "force", deleteReq.Force, `Force deletion even if the Storage Credential is not empty.`) - cmd.Use = "delete METASTORE_ID NAME" + cmd.Use = "delete METASTORE_ID STORAGE_CREDENTIAL_NAME" cmd.Short = `Delete a storage credential.` cmd.Long = `Delete a storage credential. @@ -148,7 +148,7 @@ func newDelete() *cobra.Command { a := root.AccountClient(ctx) deleteReq.MetastoreId = args[0] - deleteReq.Name = args[1] + deleteReq.StorageCredentialName = args[1] err = a.StorageCredentials.Delete(ctx, deleteReq) if err != nil { @@ -191,7 +191,7 @@ func newGet() *cobra.Command { // TODO: short flags - cmd.Use = "get METASTORE_ID NAME" + cmd.Use = "get METASTORE_ID STORAGE_CREDENTIAL_NAME" cmd.Short = `Gets the named storage credential.` cmd.Long = `Gets the named storage credential. @@ -212,7 +212,7 @@ func newGet() *cobra.Command { a := root.AccountClient(ctx) getReq.MetastoreId = args[0] - getReq.Name = args[1] + getReq.StorageCredentialName = args[1] response, err := a.StorageCredentials.Get(ctx, getReq) if err != nil { @@ -321,7 +321,7 @@ func newUpdate() *cobra.Command { // TODO: complex arg: credential_info - cmd.Use = "update METASTORE_ID NAME" + cmd.Use = "update METASTORE_ID STORAGE_CREDENTIAL_NAME" cmd.Short = `Updates a storage credential.` cmd.Long = `Updates a storage credential. @@ -348,7 +348,7 @@ func newUpdate() *cobra.Command { } } updateReq.MetastoreId = args[0] - updateReq.Name = args[1] + updateReq.StorageCredentialName = args[1] response, err := a.StorageCredentials.Update(ctx, updateReq) if err != nil { diff --git a/cmd/workspace/clusters/clusters.go b/cmd/workspace/clusters/clusters.go index f14864f02d4..07effe090f7 100755 --- a/cmd/workspace/clusters/clusters.go +++ b/cmd/workspace/clusters/clusters.go @@ -160,7 +160,7 @@ func newCreate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().BoolVar(&createReq.ApplyPolicyDefaultValues, "apply-policy-default-values", createReq.ApplyPolicyDefaultValues, `Note: This field won't be true for webapp requests.`) + cmd.Flags().BoolVar(&createReq.ApplyPolicyDefaultValues, "apply-policy-default-values", createReq.ApplyPolicyDefaultValues, ``) // TODO: complex arg: autoscale cmd.Flags().IntVar(&createReq.AutoterminationMinutes, "autotermination-minutes", createReq.AutoterminationMinutes, `Automatically terminates the cluster after it is inactive for this time in minutes.`) // TODO: complex arg: aws_attributes @@ -387,7 +387,7 @@ func newEdit() *cobra.Command { // TODO: short flags cmd.Flags().Var(&editJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().BoolVar(&editReq.ApplyPolicyDefaultValues, "apply-policy-default-values", editReq.ApplyPolicyDefaultValues, `Note: This field won't be true for webapp requests.`) + cmd.Flags().BoolVar(&editReq.ApplyPolicyDefaultValues, "apply-policy-default-values", editReq.ApplyPolicyDefaultValues, ``) // TODO: complex arg: autoscale cmd.Flags().IntVar(&editReq.AutoterminationMinutes, "autotermination-minutes", editReq.AutoterminationMinutes, `Automatically terminates the cluster after it is inactive for this time in minutes.`) // TODO: complex arg: aws_attributes diff --git a/cmd/workspace/connections/connections.go b/cmd/workspace/connections/connections.go index 917aeda9c25..26a5eec9180 100755 --- a/cmd/workspace/connections/connections.go +++ b/cmd/workspace/connections/connections.go @@ -330,6 +330,8 @@ func newUpdate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of current owner of the connection.`) + cmd.Use = "update" cmd.Short = `Update a connection.` cmd.Long = `Update a connection. diff --git a/cmd/workspace/instance-pools/instance-pools.go b/cmd/workspace/instance-pools/instance-pools.go index 9e7805aef30..b03542c0e55 100755 --- a/cmd/workspace/instance-pools/instance-pools.go +++ b/cmd/workspace/instance-pools/instance-pools.go @@ -239,17 +239,10 @@ func newEdit() *cobra.Command { // TODO: short flags cmd.Flags().Var(&editJson, "json", `either inline JSON string or @path/to/file.json with request body`) - // TODO: complex arg: aws_attributes - // TODO: complex arg: azure_attributes // TODO: map via StringToStringVar: custom_tags - // TODO: complex arg: disk_spec - cmd.Flags().BoolVar(&editReq.EnableElasticDisk, "enable-elastic-disk", editReq.EnableElasticDisk, `Autoscaling Local Storage: when enabled, this instances in this pool will dynamically acquire additional disk space when its Spark workers are running low on disk space.`) - // TODO: complex arg: gcp_attributes cmd.Flags().IntVar(&editReq.IdleInstanceAutoterminationMinutes, "idle-instance-autotermination-minutes", editReq.IdleInstanceAutoterminationMinutes, `Automatically terminates the extra instances in the pool cache after they are inactive for this time in minutes if min_idle_instances requirement is already met.`) cmd.Flags().IntVar(&editReq.MaxCapacity, "max-capacity", editReq.MaxCapacity, `Maximum number of outstanding instances to keep in the pool, including both instances used by clusters and idle instances.`) cmd.Flags().IntVar(&editReq.MinIdleInstances, "min-idle-instances", editReq.MinIdleInstances, `Minimum number of idle instances to keep in the instance pool.`) - // TODO: array: preloaded_docker_images - // TODO: array: preloaded_spark_versions cmd.Use = "edit INSTANCE_POOL_ID INSTANCE_POOL_NAME NODE_TYPE_ID" cmd.Short = `Edit an existing instance pool.` diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go index 9edebb661cf..be0df694b0d 100755 --- a/cmd/workspace/jobs/jobs.go +++ b/cmd/workspace/jobs/jobs.go @@ -1256,11 +1256,11 @@ func newReset() *cobra.Command { cmd.Flags().Var(&resetJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Use = "reset" - cmd.Short = `Overwrites all settings for a job.` - cmd.Long = `Overwrites all settings for a job. + cmd.Short = `Overwrite all settings for a job.` + cmd.Long = `Overwrite all settings for a job. - Overwrites all the settings for a specific job. Use the Update endpoint to - update job settings partially.` + Overwrite all settings for the given job. Use the Update endpoint to update + job settings partially.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go index 06d904d372d..d24606cd2da 100755 --- a/cmd/workspace/pipelines/pipelines.go +++ b/cmd/workspace/pipelines/pipelines.go @@ -919,10 +919,11 @@ func newStartUpdate() *cobra.Command { // TODO: array: refresh_selection cmd.Use = "start-update PIPELINE_ID" - cmd.Short = `Queue a pipeline update.` - cmd.Long = `Queue a pipeline update. + cmd.Short = `Start a pipeline.` + cmd.Long = `Start a pipeline. - Starts or queues a pipeline update.` + Starts a new update for the pipeline. If there is already an active update for + the pipeline, the request will fail and the active update will remain running.` cmd.Annotations = make(map[string]string) @@ -1006,7 +1007,8 @@ func newStop() *cobra.Command { cmd.Short = `Stop a pipeline.` cmd.Long = `Stop a pipeline. - Stops a pipeline.` + Stops the pipeline by canceling the active update. If there is no active + update for the pipeline, this request is a no-op.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/storage-credentials/storage-credentials.go b/cmd/workspace/storage-credentials/storage-credentials.go index b5dd5141b31..9754d0ff0ac 100755 --- a/cmd/workspace/storage-credentials/storage-credentials.go +++ b/cmd/workspace/storage-credentials/storage-credentials.go @@ -77,14 +77,7 @@ func newCreate() *cobra.Command { cmd.Short = `Create a storage credential.` cmd.Long = `Create a storage credential. - Creates a new storage credential. The request object is specific to the cloud: - - * **AwsIamRole** for AWS credentials. * **AzureServicePrincipal** for Azure - credentials. * **AzureManagedIdentity** for Azure managed credentials. * - **DatabricksGcpServiceAccount** for GCP managed credentials. - - The caller must be a metastore admin and have the - **CREATE_STORAGE_CREDENTIAL** privilege on the metastore.` + Creates a new storage credential.` cmd.Annotations = make(map[string]string) @@ -371,9 +364,7 @@ func newUpdate() *cobra.Command { cmd.Short = `Update a credential.` cmd.Long = `Update a credential. - Updates a storage credential on the metastore. The caller must be the owner of - the storage credential or a metastore admin. If the caller is a metastore - admin, only the __owner__ credential can be changed.` + Updates a storage credential on the metastore.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/workspace-bindings/workspace-bindings.go b/cmd/workspace/workspace-bindings/workspace-bindings.go index 3d7fa677cce..2d2bb5ed9dc 100755 --- a/cmd/workspace/workspace-bindings/workspace-bindings.go +++ b/cmd/workspace/workspace-bindings/workspace-bindings.go @@ -17,13 +17,25 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "workspace-bindings", - Short: `A catalog in Databricks can be configured as __OPEN__ or __ISOLATED__.`, - Long: `A catalog in Databricks can be configured as __OPEN__ or __ISOLATED__. An - __OPEN__ catalog can be accessed from any workspace, while an __ISOLATED__ - catalog can only be access from a configured list of workspaces. + Short: `A securable in Databricks can be configured as __OPEN__ or __ISOLATED__.`, + Long: `A securable in Databricks can be configured as __OPEN__ or __ISOLATED__. An + __OPEN__ securable can be accessed from any workspace, while an __ISOLATED__ + securable can only be accessed from a configured list of workspaces. This API + allows you to configure (bind) securables to workspaces. - A catalog's workspace bindings can be configured by a metastore admin or the - owner of the catalog.`, + NOTE: The __isolation_mode__ is configured for the securable itself (using its + Update method) and the workspace bindings are only consulted when the + securable's __isolation_mode__ is set to __ISOLATED__. + + A securable's workspace bindings can be configured by a metastore admin or the + owner of the securable. + + The original path (/api/2.1/unity-catalog/workspace-bindings/catalogs/{name}) + is deprecated. Please use the new path + (/api/2.1/unity-catalog/bindings/{securable_type}/{securable_name}) which + introduces the ability to bind a securable in READ_ONLY mode (catalogs only). + + Securables that support binding: - catalog`, GroupID: "catalog", Annotations: map[string]string{ "package": "catalog", @@ -100,6 +112,69 @@ func init() { }) } +// start get-bindings command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getBindingsOverrides []func( + *cobra.Command, + *catalog.GetBindingsRequest, +) + +func newGetBindings() *cobra.Command { + cmd := &cobra.Command{} + + var getBindingsReq catalog.GetBindingsRequest + + // TODO: short flags + + cmd.Use = "get-bindings SECURABLE_TYPE SECURABLE_NAME" + cmd.Short = `Get securable workspace bindings.` + cmd.Long = `Get securable workspace bindings. + + Gets workspace bindings of the securable. The caller must be a metastore admin + or an owner of the securable.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := cobra.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getBindingsReq.SecurableType = args[0] + getBindingsReq.SecurableName = args[1] + + response, err := w.WorkspaceBindings.GetBindings(ctx, getBindingsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getBindingsOverrides { + fn(cmd, &getBindingsReq) + } + + return cmd +} + +func init() { + cmdOverrides = append(cmdOverrides, func(cmd *cobra.Command) { + cmd.AddCommand(newGetBindings()) + }) +} + // start update command // Slice with functions to override default command behavior. @@ -173,4 +248,78 @@ func init() { }) } +// start update-bindings command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateBindingsOverrides []func( + *cobra.Command, + *catalog.UpdateWorkspaceBindingsParameters, +) + +func newUpdateBindings() *cobra.Command { + cmd := &cobra.Command{} + + var updateBindingsReq catalog.UpdateWorkspaceBindingsParameters + var updateBindingsJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateBindingsJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: add + // TODO: array: remove + + cmd.Use = "update-bindings SECURABLE_TYPE SECURABLE_NAME" + cmd.Short = `Update securable workspace bindings.` + cmd.Long = `Update securable workspace bindings. + + Updates workspace bindings of the securable. The caller must be a metastore + admin or an owner of the securable.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := cobra.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateBindingsJson.Unmarshal(&updateBindingsReq) + if err != nil { + return err + } + } + updateBindingsReq.SecurableType = args[0] + updateBindingsReq.SecurableName = args[1] + + response, err := w.WorkspaceBindings.UpdateBindings(ctx, updateBindingsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateBindingsOverrides { + fn(cmd, &updateBindingsReq) + } + + return cmd +} + +func init() { + cmdOverrides = append(cmdOverrides, func(cmd *cobra.Command) { + cmd.AddCommand(newUpdateBindings()) + }) +} + // end service WorkspaceBindings diff --git a/cmd/workspace/workspace/workspace.go b/cmd/workspace/workspace/workspace.go index 124680f0bdf..2541c8e34b8 100755 --- a/cmd/workspace/workspace/workspace.go +++ b/cmd/workspace/workspace/workspace.go @@ -428,8 +428,10 @@ func newImport() *cobra.Command { Imports a workspace object (for example, a notebook or file) or the contents of an entire directory. If path already exists and overwrite is set to - false, this call returns an error RESOURCE_ALREADY_EXISTS. One can only - use DBC format to import a directory.` + false, this call returns an error RESOURCE_ALREADY_EXISTS. To import a + directory, you can use either the DBC format or the SOURCE format with the + language field unset. To import a single file as SOURCE, you must set the + language field.` cmd.Annotations = make(map[string]string) diff --git a/go.mod b/go.mod index aa3fcf1ec60..6fe69f795ea 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ replace github.com/databricks/databricks-sdk-go => github.com/databricks/databri require ( github.com/briandowns/spinner v1.23.0 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.22.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.23.0 // Apache 2.0 github.com/fatih/color v1.15.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.3.1 // BSD-3-Clause @@ -56,7 +56,7 @@ require ( golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/api v0.143.0 // indirect + google.golang.org/api v0.146.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/grpc v1.58.2 // indirect diff --git a/go.sum b/go.sum index 20b05ec7c87..aa7d65569f3 100644 --- a/go.sum +++ b/go.sum @@ -29,10 +29,6 @@ github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEM github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/databricks/databricks-sdk-go v0.19.2-0.20230913090047-4b0ed4e462ad h1:QLo6CwbiK3u68RhB0l+yBT8Zflw/cHMV/AUTVYmwFbU= -github.com/databricks/databricks-sdk-go v0.19.2-0.20230913090047-4b0ed4e462ad/go.mod h1:Bt/3i3ry/rQdE6Y+psvkAENlp+LzJHaQK5PsLIstQb4= -github.com/databricks/databricks-sdk-go v0.19.2-0.20230913101814-20732bd320e2 h1:uW/UW2ZdC+13XXAVkPPGUCQJiDgxHmDwK+q+P8u1BMU= -github.com/databricks/databricks-sdk-go v0.19.2-0.20230913101814-20732bd320e2/go.mod h1:Bt/3i3ry/rQdE6Y+psvkAENlp+LzJHaQK5PsLIstQb4= github.com/databricks/databricks-sdk-go v0.19.2-0.20230914113857-3aa4e5b801a8 h1:yPN8OzuthpFq/KeP9/N5sIQffOPyZGTCAcb7JM95RNk= github.com/databricks/databricks-sdk-go v0.19.2-0.20230914113857-3aa4e5b801a8/go.mod h1:Bt/3i3ry/rQdE6Y+psvkAENlp+LzJHaQK5PsLIstQb4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -250,8 +246,8 @@ golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.143.0 h1:o8cekTkqhywkbZT6p1UHJPZ9+9uuCAJs/KYomxZB8fA= -google.golang.org/api v0.143.0/go.mod h1:FoX9DO9hT7DLNn97OuoZAGSDuNAXdJRuGK98rSUgurk= +google.golang.org/api v0.146.0 h1:9aBYT4vQXt9dhCuLNfwfd3zpwu8atg0yPkjBymwSrOM= +google.golang.org/api v0.146.0/go.mod h1:OARJqIfoYjXJj4C1AiBSXYZt03qsoz8FQYU6fBEfrHM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= From 90e48c88ff5bede281041cfc5a73556c53ee2b0f Mon Sep 17 00:00:00 2001 From: Hector Castejon Diaz Date: Fri, 13 Oct 2023 11:54:02 +0200 Subject: [PATCH 7/8] Improve test --- bundle/config/resources/pipeline.go | 16 ++++++++-------- go.mod | 2 -- go.sum | 4 ++-- internal/jobs_test.go | 5 +++-- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 174f6e4d2a4..5c741f8af60 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -18,6 +18,14 @@ type Pipeline struct { *pipelines.PipelineSpec } +func (s *Pipeline) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s Pipeline) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + // MergeClusters merges cluster definitions with same label. // The clusters field is a slice, and as such, overrides are appended to it. // We can identify a cluster by its label, however, so we can use this label @@ -64,11 +72,3 @@ func (p *Pipeline) MergeClusters() error { p.Clusters = output return nil } - -func (s *Pipeline) UnmarshalJSON(b []byte) error { - return marshal.Unmarshal(b, s) -} - -func (s Pipeline) MarshalJSON() ([]byte, error) { - return marshal.Marshal(s) -} diff --git a/go.mod b/go.mod index 6fe69f795ea..ffa44f74192 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/databricks/cli go 1.21 -replace github.com/databricks/databricks-sdk-go => github.com/databricks/databricks-sdk-go v0.19.2-0.20230914113857-3aa4e5b801a8 - require ( github.com/briandowns/spinner v1.23.0 // Apache 2.0 github.com/databricks/databricks-sdk-go v0.23.0 // Apache 2.0 diff --git a/go.sum b/go.sum index aa7d65569f3..c9e4cd1fde0 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,8 @@ github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEM github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/databricks/databricks-sdk-go v0.19.2-0.20230914113857-3aa4e5b801a8 h1:yPN8OzuthpFq/KeP9/N5sIQffOPyZGTCAcb7JM95RNk= -github.com/databricks/databricks-sdk-go v0.19.2-0.20230914113857-3aa4e5b801a8/go.mod h1:Bt/3i3ry/rQdE6Y+psvkAENlp+LzJHaQK5PsLIstQb4= +github.com/databricks/databricks-sdk-go v0.23.0 h1:rdLMA7cDUPJiCSMyuUSufzDDmugqyp79SNiY/vc7kMI= +github.com/databricks/databricks-sdk-go v0.23.0/go.mod h1:a6rErRNh5bz+IJbO07nwW70iGyvtWidy1p/S5thepXI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/jobs_test.go b/internal/jobs_test.go index 2cd1e2336f6..37d5ddc2ed0 100644 --- a/internal/jobs_test.go +++ b/internal/jobs_test.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/internal/acc" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestAccCreateJob(t *testing.T) { @@ -16,9 +17,9 @@ func TestAccCreateJob(t *testing.T) { t.Skipf("Not running test on cloud %s", env) } stdout, stderr := RequireSuccessfulRun(t, "jobs", "create", "--json", "@testjsons/create_job_without_cluster.json", "--log-level=debug") - assert.Equal(t, "", stderr.String()) + assert.Empty(t, stderr.String()) var output map[string]int err := json.Unmarshal(stdout.Bytes(), &output) - assert.NoError(t, err) + require.NoError(t, err) RequireSuccessfulRun(t, "jobs", "delete", fmt.Sprint(output["job_id"]), "--log-level=debug") } From 98df23db6c3556d31eec7d1fbd06047d4de8fa48 Mon Sep 17 00:00:00 2001 From: Hector Castejon Diaz Date: Mon, 16 Oct 2023 08:37:59 +0200 Subject: [PATCH 8/8] Rename test file --- internal/jobs_test.go | 2 +- ...job_without_cluster.json => create_job_without_workers.json} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename internal/testjsons/{create_job_without_cluster.json => create_job_without_workers.json} (100%) diff --git a/internal/jobs_test.go b/internal/jobs_test.go index 37d5ddc2ed0..8513168c823 100644 --- a/internal/jobs_test.go +++ b/internal/jobs_test.go @@ -16,7 +16,7 @@ func TestAccCreateJob(t *testing.T) { if env != "azure" { t.Skipf("Not running test on cloud %s", env) } - stdout, stderr := RequireSuccessfulRun(t, "jobs", "create", "--json", "@testjsons/create_job_without_cluster.json", "--log-level=debug") + stdout, stderr := RequireSuccessfulRun(t, "jobs", "create", "--json", "@testjsons/create_job_without_workers.json", "--log-level=debug") assert.Empty(t, stderr.String()) var output map[string]int err := json.Unmarshal(stdout.Bytes(), &output) diff --git a/internal/testjsons/create_job_without_cluster.json b/internal/testjsons/create_job_without_workers.json similarity index 100% rename from internal/testjsons/create_job_without_cluster.json rename to internal/testjsons/create_job_without_workers.json