From 1a431dd26c02dbb611c20395b236928185798e1d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 15 May 2023 11:25:01 +0200 Subject: [PATCH 01/60] wip --- cmd/init/Untitled-1.json | 21 ++++++++++++ cmd/init/file_tree.go | 74 ++++++++++++++++++++++++++++++++++++++++ cmd/init/helpers.go | 31 +++++++++++++++++ cmd/init/init.go | 45 ++++++++++++++++++++++++ main.go | 1 + 5 files changed, 172 insertions(+) create mode 100644 cmd/init/Untitled-1.json create mode 100644 cmd/init/file_tree.go create mode 100644 cmd/init/helpers.go create mode 100644 cmd/init/init.go diff --git a/cmd/init/Untitled-1.json b/cmd/init/Untitled-1.json new file mode 100644 index 00000000000..35c17247f6b --- /dev/null +++ b/cmd/init/Untitled-1.json @@ -0,0 +1,21 @@ +{ + "foo": { + "type": "string", + "default": "abc", + "validation": ["regex ^[abcd]*$"] + }, + "bar": { + "type": "integer", + "default": 123, + "validation": ["greaterThan 5", "lessThan 10"] + }, + "isAws": { + "type": "boolean", + "default": true + }, + "project_name": { + "type": "string", + "default": "my_project", + "validation": ["startsWith my_"] + } +} diff --git a/cmd/init/file_tree.go b/cmd/init/file_tree.go new file mode 100644 index 00000000000..1f8b87912c8 --- /dev/null +++ b/cmd/init/file_tree.go @@ -0,0 +1,74 @@ +package init + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "text/template" +) + +// TODO: cleanup if template initialization fails +// TODO: need robust way to clean up half generated files +// TODO: define default files +// TODO: self reference for + +func walkFileTree(config map[string]interface{}, templatePath string, instancePath string) error { + enteries, err := os.ReadDir(templatePath) + if err != nil { + return err + } + for _, entry := range enteries { + if entry.Name() == SchemaFileName { + continue + } + fileName := entry.Name() + tmpl, err := template.New("filename").Parse(fileName) + if err != nil { + return err + } + result := strings.Builder{} + err = tmpl.Execute(&result, config) + if err != nil { + return err + } + resolvedFileName := result.String() + fmt.Println(resolvedFileName) + if entry.IsDir() { + err := os.Mkdir(resolvedFileName, os.ModePerm) + if err != nil { + return err + } + err = walkFileTree(config, filepath.Join(templatePath, fileName), filepath.Join(instancePath, resolvedFileName)) + if err != nil { + return err + } + } else { + f, err := os.Create(filepath.Join(instancePath, resolvedFileName)) + if err != nil { + return err + } + b, err := os.ReadFile(filepath.Join(templatePath, fileName)) + if err != nil { + return err + } + // TODO: Might be able to use ParseFiles or ParseFS. Might be more suited + contentTmpl, err := template.New("content").Funcs(HelperFuncs).Parse(string(b)) + if err != nil { + return err + } + err = contentTmpl.Execute(f, config) + + // Make this assertion more robust + if err != nil && strings.Contains(err.Error(), ErrSkipThisFile.Error()) { + err := os.Remove(filepath.Join(instancePath, resolvedFileName)) + if err != nil { + return err + } + } else if err != nil { + return err + } + } + } + return nil +} diff --git a/cmd/init/helpers.go b/cmd/init/helpers.go new file mode 100644 index 00000000000..b1df69fb341 --- /dev/null +++ b/cmd/init/helpers.go @@ -0,0 +1,31 @@ +package init + +import ( + "errors" + "fmt" + "strings" + "text/template" +) + +var ErrSkipThisFile = errors.New("skip generating this file") + +var HelperFuncs = template.FuncMap{ + "skipThisFile": func() error { + panic(ErrSkipThisFile) + }, + "eqString": func(a string, b string) bool { + return a == b + }, + "eqNumber": func(a float64, b int) bool { + return int(a) == b + }, + "validationError": func(message string) error { + panic(fmt.Errorf(message)) + }, + "assertStartsWith": func(s string, substr string) error { + if !strings.HasPrefix(s, substr) { + panic(fmt.Errorf("%s does not start with %s.", s, substr)) + } + return nil + }, +} diff --git a/cmd/init/init.go b/cmd/init/init.go new file mode 100644 index 00000000000..fed7321b568 --- /dev/null +++ b/cmd/init/init.go @@ -0,0 +1,45 @@ +package init + +import ( + "encoding/json" + "os" + + "github.com/databricks/bricks/cmd/root" + "github.com/spf13/cobra" +) + +const SchemaFileName = "config.json" + +// root template defination at schema.json +// decide on semantics of defination later + +// initCmd represents the fs command +var initCmd = &cobra.Command{ + Use: "init", + Short: "Initialize Template", + Long: `Initialize bundle template`, + RunE: func(cmd *cobra.Command, args []string) error { + var config map[string]interface{} + b, err := os.ReadFile(SchemaFileName) + if err != nil { + return err + } + err = json.Unmarshal(b, &config) + if err != nil { + return err + } + err = walkFileTree(config, ".", ".") + if err != nil { + err2 := os.RemoveAll("favela") + if err2 != nil { + return err2 + } + return err + } + return nil + }, +} + +func init() { + root.RootCmd.AddCommand(initCmd) +} diff --git a/main.go b/main.go index fd5d31f3ad8..1ef34ccce47 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,7 @@ import ( _ "github.com/databricks/bricks/cmd/bundle/debug" _ "github.com/databricks/bricks/cmd/configure" _ "github.com/databricks/bricks/cmd/fs" + _ "github.com/databricks/bricks/cmd/init" "github.com/databricks/bricks/cmd/root" _ "github.com/databricks/bricks/cmd/sync" _ "github.com/databricks/bricks/cmd/version" From 9ef3547a3a8deadd9202a9b232e12b12bb83f5f4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 02:43:41 +0200 Subject: [PATCH 02/60] First draft --- cmd/init/Untitled-1.json | 21 -------- cmd/init/execute.go | 101 ++++++++++++++++++++++++++++++++++++++ cmd/init/file_tree.go | 74 ---------------------------- cmd/init/helpers.go | 17 ------- cmd/init/init.go | 47 +++++++++++++----- cmd/init/schema.go | 102 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 237 insertions(+), 125 deletions(-) delete mode 100644 cmd/init/Untitled-1.json create mode 100644 cmd/init/execute.go delete mode 100644 cmd/init/file_tree.go create mode 100644 cmd/init/schema.go diff --git a/cmd/init/Untitled-1.json b/cmd/init/Untitled-1.json deleted file mode 100644 index 35c17247f6b..00000000000 --- a/cmd/init/Untitled-1.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "foo": { - "type": "string", - "default": "abc", - "validation": ["regex ^[abcd]*$"] - }, - "bar": { - "type": "integer", - "default": 123, - "validation": ["greaterThan 5", "lessThan 10"] - }, - "isAws": { - "type": "boolean", - "default": true - }, - "project_name": { - "type": "string", - "default": "my_project", - "validation": ["startsWith my_"] - } -} diff --git a/cmd/init/execute.go b/cmd/init/execute.go new file mode 100644 index 00000000000..73b08e7d62e --- /dev/null +++ b/cmd/init/execute.go @@ -0,0 +1,101 @@ +package init + +import ( + "os" + "path/filepath" + "strings" + "text/template" +) + +// Executes the template by appling config on it. Returns the materialized config +// as a string +func executeTemplate(config map[string]any, templateDefination string) (string, error) { + // configure template with helper functions + tmpl, err := template.New("foo").Funcs(HelperFuncs).Parse(templateDefination) + if err != nil { + return "", err + } + + // execute template + result := strings.Builder{} + err = tmpl.Execute(&result, config) + if err != nil { + return "", err + } + return result.String(), nil +} + +// TODO: allow skipping directories +func generateDirectory(config map[string]any, parentDir, nameTempate string) (string, error) { + dirName, err := executeTemplate(config, nameTempate) + if err != nil { + return "", err + } + err = os.Mkdir(filepath.Join(parentDir, dirName), 0755) + if err != nil { + return "", err + } + return dirName, nil +} + +func generateFile(config map[string]any, parentDir, nameTempate, contentTemplate string) error { + // compute file content + fileContent, err := executeTemplate(config, contentTemplate) + // TODO: maybe we need string matching here to make this work + if err != nil && err == ErrSkipThisFile { + return nil + } + if err != nil { + return err + } + + // create the file by executing the templatized file name + fileName, err := executeTemplate(config, nameTempate) + if err != nil { + return err + } + f, err := os.Create(filepath.Join(parentDir, fileName)) + if err != nil { + return err + } + + // write to file the computed content + _, err = f.Write([]byte(fileContent)) + return err +} + +func walkFileTree(config map[string]any, templatePath, instancePath string) error { + enteries, err := os.ReadDir(templatePath) + if err != nil { + return err + } + for _, entry := range enteries { + if entry.IsDir() { + // case: materialize a template directory + dirName, err := generateDirectory(config, instancePath, entry.Name()) + if err != nil { + return err + } + + // recusive generate files and directories inside inside our newly generated + // directory from the template defination + err = walkFileTree(config, filepath.Join(templatePath, entry.Name()), filepath.Join(instancePath, dirName)) + if err != nil { + return err + } + } else { + // case: materialize a template file with it's contents + b, err := os.ReadFile(filepath.Join(templatePath, entry.Name())) + if err != nil { + return err + } + contentTemplate := string(b) + fileNameTemplate := entry.Name() + err = generateFile(config, instancePath, fileNameTemplate, contentTemplate) + if err != nil { + return err + } + } + } + return nil +} diff --git a/cmd/init/file_tree.go b/cmd/init/file_tree.go deleted file mode 100644 index 1f8b87912c8..00000000000 --- a/cmd/init/file_tree.go +++ /dev/null @@ -1,74 +0,0 @@ -package init - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "text/template" -) - -// TODO: cleanup if template initialization fails -// TODO: need robust way to clean up half generated files -// TODO: define default files -// TODO: self reference for - -func walkFileTree(config map[string]interface{}, templatePath string, instancePath string) error { - enteries, err := os.ReadDir(templatePath) - if err != nil { - return err - } - for _, entry := range enteries { - if entry.Name() == SchemaFileName { - continue - } - fileName := entry.Name() - tmpl, err := template.New("filename").Parse(fileName) - if err != nil { - return err - } - result := strings.Builder{} - err = tmpl.Execute(&result, config) - if err != nil { - return err - } - resolvedFileName := result.String() - fmt.Println(resolvedFileName) - if entry.IsDir() { - err := os.Mkdir(resolvedFileName, os.ModePerm) - if err != nil { - return err - } - err = walkFileTree(config, filepath.Join(templatePath, fileName), filepath.Join(instancePath, resolvedFileName)) - if err != nil { - return err - } - } else { - f, err := os.Create(filepath.Join(instancePath, resolvedFileName)) - if err != nil { - return err - } - b, err := os.ReadFile(filepath.Join(templatePath, fileName)) - if err != nil { - return err - } - // TODO: Might be able to use ParseFiles or ParseFS. Might be more suited - contentTmpl, err := template.New("content").Funcs(HelperFuncs).Parse(string(b)) - if err != nil { - return err - } - err = contentTmpl.Execute(f, config) - - // Make this assertion more robust - if err != nil && strings.Contains(err.Error(), ErrSkipThisFile.Error()) { - err := os.Remove(filepath.Join(instancePath, resolvedFileName)) - if err != nil { - return err - } - } else if err != nil { - return err - } - } - } - return nil -} diff --git a/cmd/init/helpers.go b/cmd/init/helpers.go index b1df69fb341..89f236fdfb6 100644 --- a/cmd/init/helpers.go +++ b/cmd/init/helpers.go @@ -2,8 +2,6 @@ package init import ( "errors" - "fmt" - "strings" "text/template" ) @@ -13,19 +11,4 @@ var HelperFuncs = template.FuncMap{ "skipThisFile": func() error { panic(ErrSkipThisFile) }, - "eqString": func(a string, b string) bool { - return a == b - }, - "eqNumber": func(a float64, b int) bool { - return int(a) == b - }, - "validationError": func(message string) error { - panic(fmt.Errorf(message)) - }, - "assertStartsWith": func(s string, substr string) error { - if !strings.HasPrefix(s, substr) { - panic(fmt.Errorf("%s does not start with %s.", s, substr)) - } - return nil - }, } diff --git a/cmd/init/init.go b/cmd/init/init.go index fed7321b568..71b7870b4eb 100644 --- a/cmd/init/init.go +++ b/cmd/init/init.go @@ -3,24 +3,38 @@ package init import ( "encoding/json" "os" + "path/filepath" "github.com/databricks/bricks/cmd/root" "github.com/spf13/cobra" ) -const SchemaFileName = "config.json" +const ConfigFileName = "config.json" +const SchemaFileName = "schema.json" +const TemplateDirname = "template" -// root template defination at schema.json -// decide on semantics of defination later - -// initCmd represents the fs command var initCmd = &cobra.Command{ Use: "init", Short: "Initialize Template", - Long: `Initialize bundle template`, + Long: `Initialize template`, + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + templateLocation := args[0] + + // read the file containing schema for template input parameters + schemaBytes, err := os.ReadFile(filepath.Join(templateLocation, SchemaFileName)) + if err != nil { + return err + } + schema := Schema{} + err = json.Unmarshal(schemaBytes, &schema) + if err != nil { + return err + } + + // read user config to initalize the template with var config map[string]interface{} - b, err := os.ReadFile(SchemaFileName) + b, err := os.ReadFile(ConfigFileName) if err != nil { return err } @@ -28,15 +42,22 @@ var initCmd = &cobra.Command{ if err != nil { return err } - err = walkFileTree(config, ".", ".") + + // cast any fields that are supported to be integers. The json unmarshalling + // for a generic map converts all numbers to floating point + err = schema.CastFloatToInt(config) if err != nil { - err2 := os.RemoveAll("favela") - if err2 != nil { - return err2 - } return err } - return nil + + // validate config according to schema + err = schema.ValidateConfig(config) + if err != nil { + return err + } + + // materialize the template + return walkFileTree(config, filepath.Join(args[0], TemplateDirname), ".") }, } diff --git a/cmd/init/schema.go b/cmd/init/schema.go new file mode 100644 index 00000000000..78a2eef0b64 --- /dev/null +++ b/cmd/init/schema.go @@ -0,0 +1,102 @@ +package init + +import ( + "fmt" + "reflect" +) + +type Schema map[string]FieldInfo + +type FieldType string + +const ( + FieldTypeString = FieldType("string") + FieldTypeInt = FieldType("integer") + FieldTypeFloat = FieldType("float") + FieldTypeBoolean = FieldType("boolean") +) + +type FieldInfo struct { + Type FieldType `json:"type"` + Description string `json:"description"` + Validation string `json:"validate"` +} + +// function to check whether a float value represents an integer +func isIntegerValue(v float64) bool { + return v == float64(int(v)) +} + +// cast value to integer for config values that are floats but are supposed to be +// integeres according to the schema +// +// Needed because the default json unmarshaller for maps converts all numbers to floats +func (schema Schema) CastFloatToInt(config map[string]any) error { + for k, v := range config { + // error because all config keys should be defined in schema too + if _, ok := schema[k]; !ok { + return fmt.Errorf("%s is not defined as an input parameter for the template", k) + } + + // skip non integer fields + fieldInfo := schema[k] + if fieldInfo.Type != FieldTypeInt { + continue + } + + // convert floating point type values to integer + valueType := reflect.TypeOf(v) + switch valueType.Kind() { + case reflect.Float32: + floatVal := v.(float32) + if !isIntegerValue(float64(floatVal)) { + return fmt.Errorf("expected %s to have integer value but it is %v", k, v) + } + config[k] = int(floatVal) + case reflect.Float64: + floatVal := v.(float64) + if !isIntegerValue(floatVal) { + return fmt.Errorf("expected %s to have integer value but it is %v", k, v) + } + config[k] = int(floatVal) + } + } + return nil +} + +func validateType(v any, fieldType FieldType) error { + switch fieldType { + case FieldTypeString: + if _, ok := v.(string); !ok { + return fmt.Errorf("expected type string, but value is %#v", v) + } + case FieldTypeInt: + if _, ok := v.(int); !ok { + return fmt.Errorf("expected type integer, but value is %#v", v) + } + case FieldTypeFloat: + if _, ok := v.(float64); !ok { + return fmt.Errorf("expected type float, but value is %#v", v) + } + case FieldTypeBoolean: + if _, ok := v.(bool); !ok { + return fmt.Errorf("expected type boolean, but value is %#v", v) + } + } + return nil +} + +// TODO: add validation check for regex for string types +func (schema Schema) ValidateConfig(config map[string]any) error { + for k, v := range config { + fieldMetadata, ok := schema[k] + if !ok { + return fmt.Errorf("%s is not defined as an input parameter for the template", k) + } + err := validateType(v, fieldMetadata.Type) + if err != nil { + return fmt.Errorf("incorrect type for %s. %w", k, err) + } + } + return nil +} From 4b7a606d72dafcf896cdc9ea41751b2a93b84bbc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 11:53:13 +0200 Subject: [PATCH 03/60] move walker to a separate library --- cmd/init/init.go | 5 +++-- {cmd/init => libs/template}/execute.go | 6 +++--- {cmd/init => libs/template}/helpers.go | 2 +- {cmd/init => libs/template}/schema.go | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) rename {cmd/init => libs/template}/execute.go (94%) rename {cmd/init => libs/template}/helpers.go (92%) rename {cmd/init => libs/template}/schema.go (99%) diff --git a/cmd/init/init.go b/cmd/init/init.go index 71b7870b4eb..7839728f000 100644 --- a/cmd/init/init.go +++ b/cmd/init/init.go @@ -6,6 +6,7 @@ import ( "path/filepath" "github.com/databricks/bricks/cmd/root" + "github.com/databricks/bricks/libs/template" "github.com/spf13/cobra" ) @@ -26,7 +27,7 @@ var initCmd = &cobra.Command{ if err != nil { return err } - schema := Schema{} + schema := template.Schema{} err = json.Unmarshal(schemaBytes, &schema) if err != nil { return err @@ -57,7 +58,7 @@ var initCmd = &cobra.Command{ } // materialize the template - return walkFileTree(config, filepath.Join(args[0], TemplateDirname), ".") + return template.WalkFileTree(config, filepath.Join(args[0], TemplateDirname), ".") }, } diff --git a/cmd/init/execute.go b/libs/template/execute.go similarity index 94% rename from cmd/init/execute.go rename to libs/template/execute.go index 73b08e7d62e..133dd0c7eb6 100644 --- a/cmd/init/execute.go +++ b/libs/template/execute.go @@ -1,4 +1,4 @@ -package init +package template import ( "os" @@ -64,7 +64,7 @@ func generateFile(config map[string]any, parentDir, nameTempate, contentTemplate return err } -func walkFileTree(config map[string]any, templatePath, instancePath string) error { +func WalkFileTree(config map[string]any, templatePath, instancePath string) error { enteries, err := os.ReadDir(templatePath) if err != nil { return err @@ -79,7 +79,7 @@ func walkFileTree(config map[string]any, templatePath, instancePath string) erro // recusive generate files and directories inside inside our newly generated // directory from the template defination - err = walkFileTree(config, filepath.Join(templatePath, entry.Name()), filepath.Join(instancePath, dirName)) + err = WalkFileTree(config, filepath.Join(templatePath, entry.Name()), filepath.Join(instancePath, dirName)) if err != nil { return err } diff --git a/cmd/init/helpers.go b/libs/template/helpers.go similarity index 92% rename from cmd/init/helpers.go rename to libs/template/helpers.go index 89f236fdfb6..ff8ae539eda 100644 --- a/cmd/init/helpers.go +++ b/libs/template/helpers.go @@ -1,4 +1,4 @@ -package init +package template import ( "errors" diff --git a/cmd/init/schema.go b/libs/template/schema.go similarity index 99% rename from cmd/init/schema.go rename to libs/template/schema.go index 78a2eef0b64..7b3bcc2f5b1 100644 --- a/cmd/init/schema.go +++ b/libs/template/schema.go @@ -1,4 +1,4 @@ -package init +package template import ( "fmt" From 603a68badef6d371e21135028514d46b3f6e0f05 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 13:41:15 +0200 Subject: [PATCH 04/60] Added e2e tests --- cmd/init/init.go | 7 +- internal/init_test.go | 79 +++++++++++++++++++ .../init/templateDefinition/schema.json | 17 ++++ .../template/{{.project_name}}/.{{.ci_type}} | 5 ++ .../template/{{.project_name}}/aws_file | 4 + .../template/{{.project_name}}/azure_file | 4 + libs/template/execute.go | 6 +- 7 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 internal/init_test.go create mode 100644 internal/testdata/init/templateDefinition/schema.json create mode 100644 internal/testdata/init/templateDefinition/template/{{.project_name}}/.{{.ci_type}} create mode 100644 internal/testdata/init/templateDefinition/template/{{.project_name}}/aws_file create mode 100644 internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file diff --git a/cmd/init/init.go b/cmd/init/init.go index 7839728f000..a2f2f2e8f25 100644 --- a/cmd/init/init.go +++ b/cmd/init/init.go @@ -35,7 +35,7 @@ var initCmd = &cobra.Command{ // read user config to initalize the template with var config map[string]interface{} - b, err := os.ReadFile(ConfigFileName) + b, err := os.ReadFile(filepath.Join(targetDir, ConfigFileName)) if err != nil { return err } @@ -58,10 +58,13 @@ var initCmd = &cobra.Command{ } // materialize the template - return template.WalkFileTree(config, filepath.Join(args[0], TemplateDirname), ".") + return template.WalkFileTree(config, filepath.Join(args[0], TemplateDirname), targetDir) }, } +var targetDir string + func init() { + initCmd.Flags().StringVar(&targetDir, "target-dir", ".", "path to directory template will be initialized in") root.RootCmd.AddCommand(initCmd) } diff --git a/internal/init_test.go b/internal/init_test.go new file mode 100644 index 00000000000..ba4737e79e8 --- /dev/null +++ b/internal/init_test.go @@ -0,0 +1,79 @@ +package internal + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + _ "github.com/databricks/bricks/cmd/init" + "github.com/databricks/bricks/cmd/root" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func assertFileContains(t *testing.T, path string, substr string) { + b, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(b), substr) +} + +func TestTemplateInitialization(t *testing.T) { + // create target directory with the input config + tmp := t.TempDir() + f, err := os.Create(filepath.Join(tmp, "config.json")) + require.NoError(t, err) + _, err = f.WriteString(` + { + "project_name": "development_project", + "cloud_type": "AWS", + "ci_type": "github", + "is_production": false + } + `) + require.NoError(t, err) + + // materialize the template + cmd := root.RootCmd + cmd.SetArgs([]string{"init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", tmp}) + err = cmd.Execute() + require.NoError(t, err) + + // assert on materialized template + assert.FileExists(t, filepath.Join(tmp, "development_project", "aws_file")) + assert.FileExists(t, filepath.Join(tmp, "development_project", ".github")) + assert.NoFileExists(t, filepath.Join(tmp, "development_project", "azure_file")) + assertFileContains(t, filepath.Join(tmp, "development_project", "aws_file"), "This file should only be generated for AWS") + assertFileContains(t, filepath.Join(tmp, "development_project", ".github"), "This is a development project") +} + +func TestTemplateInitialization2(t *testing.T) { + // create target directory with the input config + tmp := t.TempDir() + f, err := os.Create(filepath.Join(tmp, "config.json")) + require.NoError(t, err) + _, err = f.WriteString(` + { + "project_name": "production_project", + "cloud_type": "Azure", + "ci_type": "azure_devops", + "is_production": true + } + `) + require.NoError(t, err) + + // materialize the template + cmd := root.RootCmd + childCommands := cmd.Commands() + fmt.Println(childCommands) + cmd.SetArgs([]string{"init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", tmp}) + err = cmd.Execute() + require.NoError(t, err) + + // assert on materialized template + assert.FileExists(t, filepath.Join(tmp, "production_project", "azure_file")) + assert.FileExists(t, filepath.Join(tmp, "production_project", ".azure_devops")) + assert.NoFileExists(t, filepath.Join(tmp, "production_project", "aws_file")) + assertFileContains(t, filepath.Join(tmp, "production_project", "azure_file"), "This file should only be generated for Azure") + assertFileContains(t, filepath.Join(tmp, "production_project", ".azure_devops"), "This is a production project") +} diff --git a/internal/testdata/init/templateDefinition/schema.json b/internal/testdata/init/templateDefinition/schema.json new file mode 100644 index 00000000000..5f927011b0f --- /dev/null +++ b/internal/testdata/init/templateDefinition/schema.json @@ -0,0 +1,17 @@ +{ + "project_name": { + "description": "Name of the project", + "type": "string" + }, + "cloud_type": { + "description": "type of the cloud for the project", + "type": "string" + }, + "is_production": { + "type": "boolean" + }, + "ci_type": { + "type": "string", + "description": "type of the CI runner, eg: github, azure devops" + } +} diff --git a/internal/testdata/init/templateDefinition/template/{{.project_name}}/.{{.ci_type}} b/internal/testdata/init/templateDefinition/template/{{.project_name}}/.{{.ci_type}} new file mode 100644 index 00000000000..b5c2d444085 --- /dev/null +++ b/internal/testdata/init/templateDefinition/template/{{.project_name}}/.{{.ci_type}} @@ -0,0 +1,5 @@ +{{if .is_production}} +This is a production project +{{else}} +This is a development project +{{end}} diff --git a/internal/testdata/init/templateDefinition/template/{{.project_name}}/aws_file b/internal/testdata/init/templateDefinition/template/{{.project_name}}/aws_file new file mode 100644 index 00000000000..a5f33d20e4f --- /dev/null +++ b/internal/testdata/init/templateDefinition/template/{{.project_name}}/aws_file @@ -0,0 +1,4 @@ +This file should only be generated for AWS +{{if ne .cloud_type "AWS"}} +{{skipThisFile}} +{{end}} diff --git a/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file b/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file new file mode 100644 index 00000000000..e1be17cb50a --- /dev/null +++ b/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file @@ -0,0 +1,4 @@ +This file should only be generated for Azure +{{if ne .cloud_type "Azure"}} +{{skipThisFile}} +{{end}} diff --git a/libs/template/execute.go b/libs/template/execute.go index 133dd0c7eb6..d5fb88fb381 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -41,8 +41,10 @@ func generateDirectory(config map[string]any, parentDir, nameTempate string) (st func generateFile(config map[string]any, parentDir, nameTempate, contentTemplate string) error { // compute file content fileContent, err := executeTemplate(config, contentTemplate) - // TODO: maybe we need string matching here to make this work - if err != nil && err == ErrSkipThisFile { + // We do a substring match here because on errors the template library prepends + // some additional information about the callsite from which the ErrSkipThisFile + // error was returned + if err != nil && strings.Contains(err.Error(), ErrSkipThisFile.Error()) { return nil } if err != nil { From 970d05157c5affc6574e8b6745812c1af62a37ce Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 13:43:27 +0200 Subject: [PATCH 05/60] nit --- internal/init_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/init_test.go b/internal/init_test.go index ba4737e79e8..457490920d8 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -18,7 +18,7 @@ func assertFileContains(t *testing.T, path string, substr string) { assert.Contains(t, string(b), substr) } -func TestTemplateInitialization(t *testing.T) { +func TestTemplateInitializationForDevConfig(t *testing.T) { // create target directory with the input config tmp := t.TempDir() f, err := os.Create(filepath.Join(tmp, "config.json")) @@ -47,7 +47,7 @@ func TestTemplateInitialization(t *testing.T) { assertFileContains(t, filepath.Join(tmp, "development_project", ".github"), "This is a development project") } -func TestTemplateInitialization2(t *testing.T) { +func TestTemplateInitializationForProdConfig(t *testing.T) { // create target directory with the input config tmp := t.TempDir() f, err := os.Create(filepath.Join(tmp, "config.json")) From 05b3b6ca2e85f612dd017d8ad2a94ab229364788 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 14:02:41 +0200 Subject: [PATCH 06/60] added tests for castfloat --- libs/template/schema_test.go | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 libs/template/schema_test.go diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go new file mode 100644 index 00000000000..962262b1ac3 --- /dev/null +++ b/libs/template/schema_test.go @@ -0,0 +1,68 @@ +package template + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTemplateSchematIsInterger(t *testing.T) { + assert.False(t, isIntegerValue(1.1)) + assert.False(t, isIntegerValue(0.1)) + assert.False(t, isIntegerValue(-0.1)) + + assert.True(t, isIntegerValue(-1.0)) + assert.True(t, isIntegerValue(0.0)) + assert.True(t, isIntegerValue(2.0)) +} + +func TestTemplateSchemaCastFloatToInt(t *testing.T) { + // define schema for config + schemaJson := `{ + "int_val": { + "type": "integer" + }, + "float_val": { + "type": "float" + }, + "bool_val": { + "type": "boolean" + }, + "string_val": { + "type": "string" + } + }` + var schema Schema + err := json.Unmarshal([]byte(schemaJson), &schema) + require.NoError(t, err) + + // define the config + configJson := `{ + "int_val": 1, + "float_val": 2, + "bool_val": true, + "string_val": "main hoon na" + }` + var config map[string]any + err = json.Unmarshal([]byte(configJson), &config) + require.NoError(t, err) + + // assert types before casting, checking that the integer was indeed loaded + // as a floating point + assert.IsType(t, float64(0), config["int_val"]) + assert.IsType(t, float64(0), config["float_val"]) + assert.IsType(t, true, config["bool_val"]) + assert.IsType(t, "abc", config["string_val"]) + + err = schema.CastFloatToInt(config) + require.NoError(t, err) + + // assert type after casting, that the float value was converted to an integer + // for int_val. + assert.IsType(t, int(0), config["int_val"]) + assert.IsType(t, float64(0), config["float_val"]) + assert.IsType(t, true, config["bool_val"]) + assert.IsType(t, "abc", config["string_val"]) +} From 742edb5215b6d7b7e2d31b16043b56e72b709399 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 14:05:06 +0200 Subject: [PATCH 07/60] added unknown type test --- libs/template/schema_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go index 962262b1ac3..cbb007ac6c0 100644 --- a/libs/template/schema_test.go +++ b/libs/template/schema_test.go @@ -66,3 +66,26 @@ func TestTemplateSchemaCastFloatToInt(t *testing.T) { assert.IsType(t, true, config["bool_val"]) assert.IsType(t, "abc", config["string_val"]) } + +func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { + // define schema for config + schemaJson := `{ + "foo": { + "type": "integer" + } + }` + var schema Schema + err := json.Unmarshal([]byte(schemaJson), &schema) + require.NoError(t, err) + + // define the config + configJson := `{ + "bar": true + }` + var config map[string]any + err = json.Unmarshal([]byte(configJson), &config) + require.NoError(t, err) + + err = schema.CastFloatToInt(config) + assert.ErrorContains(t, err, "bar is not defined as an input parameter for the template") +} From a086574a69b3a80c84acc88ba146afd1a5af6e8e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 14:07:57 +0200 Subject: [PATCH 08/60] merge complete --- cmd/init/init.go | 4 ++-- internal/init_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/init/init.go b/cmd/init/init.go index a2f2f2e8f25..8ffb129a327 100644 --- a/cmd/init/init.go +++ b/cmd/init/init.go @@ -5,8 +5,8 @@ import ( "os" "path/filepath" - "github.com/databricks/bricks/cmd/root" - "github.com/databricks/bricks/libs/template" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/template" "github.com/spf13/cobra" ) diff --git a/internal/init_test.go b/internal/init_test.go index 457490920d8..869be19dec8 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -6,8 +6,8 @@ import ( "path/filepath" "testing" - _ "github.com/databricks/bricks/cmd/init" - "github.com/databricks/bricks/cmd/root" + _ "github.com/databricks/cli/cmd/init" + "github.com/databricks/cli/cmd/root" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) From 1d5e09bd39e130cea81ed9cd47a33bf16d10019f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 14:26:35 +0200 Subject: [PATCH 09/60] correct validateType to use reflection and added test for it --- libs/template/schema.go | 8 +++- libs/template/schema_test.go | 79 ++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/libs/template/schema.go b/libs/template/schema.go index 7b3bcc2f5b1..b5fb35aa926 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -3,6 +3,8 @@ package template import ( "fmt" "reflect" + + "golang.org/x/exp/slices" ) type Schema map[string]FieldInfo @@ -71,11 +73,13 @@ func validateType(v any, fieldType FieldType) error { return fmt.Errorf("expected type string, but value is %#v", v) } case FieldTypeInt: - if _, ok := v.(int); !ok { + if !slices.Contains([]reflect.Kind{reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64}, + reflect.TypeOf(v).Kind()) { return fmt.Errorf("expected type integer, but value is %#v", v) } case FieldTypeFloat: - if _, ok := v.(float64); !ok { + if !slices.Contains([]reflect.Kind{reflect.Float32, reflect.Float64}, + reflect.TypeOf(v).Kind()) { return fmt.Errorf("expected type float, but value is %#v", v) } case FieldTypeBoolean: diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go index cbb007ac6c0..ad1a69c5fa8 100644 --- a/libs/template/schema_test.go +++ b/libs/template/schema_test.go @@ -89,3 +89,82 @@ func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { err = schema.CastFloatToInt(config) assert.ErrorContains(t, err, "bar is not defined as an input parameter for the template") } + +func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { + // define schema for config + schemaJson := `{ + "foo": { + "type": "integer" + } + }` + var schema Schema + err := json.Unmarshal([]byte(schemaJson), &schema) + require.NoError(t, err) + + // define the config + configJson := `{ + "foo": 1.1 + }` + var config map[string]any + err = json.Unmarshal([]byte(configJson), &config) + require.NoError(t, err) + + err = schema.CastFloatToInt(config) + assert.ErrorContains(t, err, "expected foo to have integer value but it is 1.1") +} + +func TestTemplateSchemaValidateType(t *testing.T) { + // assert validation passing + err := validateType(int(0), FieldTypeInt) + assert.NoError(t, err) + + err = validateType(int32(1), FieldTypeInt) + assert.NoError(t, err) + + err = validateType(int64(1), FieldTypeInt) + assert.NoError(t, err) + + err = validateType(float32(1.1), FieldTypeFloat) + assert.NoError(t, err) + + err = validateType(float64(1.2), FieldTypeFloat) + assert.NoError(t, err) + + err = validateType(false, FieldTypeBoolean) + assert.NoError(t, err) + + err = validateType("abc", FieldTypeString) + assert.NoError(t, err) + + // assert validation failing for integers + err = validateType(float64(1.2), FieldTypeInt) + assert.ErrorContains(t, err, "expected type integer, but value is 1.2") + err = validateType(true, FieldTypeInt) + assert.ErrorContains(t, err, "expected type integer, but value is true") + err = validateType("abc", FieldTypeInt) + assert.ErrorContains(t, err, "expected type integer, but value is \"abc\"") + + // assert validation failing for floats + err = validateType(int(1), FieldTypeFloat) + assert.ErrorContains(t, err, "expected type float, but value is 1") + err = validateType(true, FieldTypeFloat) + assert.ErrorContains(t, err, "expected type float, but value is true") + err = validateType("abc", FieldTypeFloat) + assert.ErrorContains(t, err, "expected type float, but value is \"abc\"") + + // assert validation failing for boolean + err = validateType(int(1), FieldTypeBoolean) + assert.ErrorContains(t, err, "expected type boolean, but value is 1") + err = validateType(float64(1), FieldTypeBoolean) + assert.ErrorContains(t, err, "expected type boolean, but value is 1") + err = validateType("abc", FieldTypeBoolean) + assert.ErrorContains(t, err, "expected type boolean, but value is \"abc\"") + + // assert validation failing for string + err = validateType(int(1), FieldTypeString) + assert.ErrorContains(t, err, "expected type string, but value is 1") + err = validateType(float64(1), FieldTypeString) + assert.ErrorContains(t, err, "expected type string, but value is 1") + err = validateType(false, FieldTypeString) + assert.ErrorContains(t, err, "expected type string, but value is false") +} From fb2da25f285141b2875717248884018b80acc7a9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 14:42:24 +0200 Subject: [PATCH 10/60] Add tests for validate config --- libs/template/schema.go | 8 +++ libs/template/schema_test.go | 119 +++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/libs/template/schema.go b/libs/template/schema.go index b5fb35aa926..e8187fa540b 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -92,6 +92,14 @@ func validateType(v any, fieldType FieldType) error { // TODO: add validation check for regex for string types func (schema Schema) ValidateConfig(config map[string]any) error { + // assert all fields are defined in + for k := range schema { + if _, ok := config[k]; !ok { + return fmt.Errorf("input parameter %s is not defined in config", k) + } + } + + // validate types defined in config for k, v := range config { fieldMetadata, ok := schema[k] if !ok { diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go index ad1a69c5fa8..123e9c9baa1 100644 --- a/libs/template/schema_test.go +++ b/libs/template/schema_test.go @@ -168,3 +168,122 @@ func TestTemplateSchemaValidateType(t *testing.T) { err = validateType(false, FieldTypeString) assert.ErrorContains(t, err, "expected type string, but value is false") } + +func TestTemplateSchemaValidateConfig(t *testing.T) { + // define schema for config + schemaJson := `{ + "int_val": { + "type": "integer" + }, + "float_val": { + "type": "float" + }, + "bool_val": { + "type": "boolean" + }, + "string_val": { + "type": "string" + } + }` + var schema Schema + err := json.Unmarshal([]byte(schemaJson), &schema) + require.NoError(t, err) + + // define the config + config := map[string]any{ + "int_val": 1, + "float_val": 1.1, + "bool_val": true, + "string_val": "abc", + } + + err = schema.ValidateConfig(config) + assert.NoError(t, err) +} + +func TestTemplateSchemaValidateConfigFailsForUnknownField(t *testing.T) { + // define schema for config + schemaJson := `{ + "int_val": { + "type": "integer" + }, + "float_val": { + "type": "float" + }, + "bool_val": { + "type": "boolean" + }, + "string_val": { + "type": "string" + } + }` + var schema Schema + err := json.Unmarshal([]byte(schemaJson), &schema) + require.NoError(t, err) + + // define the config + config := map[string]any{ + "foo": 1, + "float_val": 1.1, + "bool_val": true, + "string_val": "abc", + } + + err = schema.ValidateConfig(config) + assert.ErrorContains(t, err, "foo is not defined as an input parameter for the template") +} + +func TestTemplateSchemaValidateConfigFailsForWhenIncorrectTypes(t *testing.T) { + // define schema for config + schemaJson := `{ + "int_val": { + "type": "integer" + }, + "float_val": { + "type": "float" + }, + "bool_val": { + "type": "boolean" + }, + "string_val": { + "type": "string" + } + }` + var schema Schema + err := json.Unmarshal([]byte(schemaJson), &schema) + require.NoError(t, err) + + // define the config + config := map[string]any{ + "int_val": 1, + "float_val": 1.1, + "bool_val": "true", + "string_val": "abc", + } + + err = schema.ValidateConfig(config) + assert.ErrorContains(t, err, "incorrect type for bool_val. expected type boolean, but value is \"true\"") +} + +func TestTemplateSchemaValidateConfigFailsForWhenMissingInputParams(t *testing.T) { + // define schema for config + schemaJson := `{ + "int_val": { + "type": "integer" + }, + "string_val": { + "type": "string" + } + }` + var schema Schema + err := json.Unmarshal([]byte(schemaJson), &schema) + require.NoError(t, err) + + // define the config + config := map[string]any{ + "int_val": 1, + } + + err = schema.ValidateConfig(config) + assert.ErrorContains(t, err, "input parameter string_val is not defined in config") +} From 5771b62afdc2655620061cede14d4f18a518c73d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 14:49:03 +0200 Subject: [PATCH 11/60] add close for files --- internal/init_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/init_test.go b/internal/init_test.go index 869be19dec8..b6f74252b69 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -23,6 +23,7 @@ func TestTemplateInitializationForDevConfig(t *testing.T) { tmp := t.TempDir() f, err := os.Create(filepath.Join(tmp, "config.json")) require.NoError(t, err) + defer f.Close() _, err = f.WriteString(` { "project_name": "development_project", @@ -52,6 +53,7 @@ func TestTemplateInitializationForProdConfig(t *testing.T) { tmp := t.TempDir() f, err := os.Create(filepath.Join(tmp, "config.json")) require.NoError(t, err) + defer f.Close() _, err = f.WriteString(` { "project_name": "production_project", From 9a70861e4bc8fa43b374c55ad603d12429032d2c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 15:00:27 +0200 Subject: [PATCH 12/60] address some comments --- cmd/init/init.go | 4 ++-- libs/template/execute.go | 4 ++-- libs/template/schema.go | 13 ++++++------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/init/init.go b/cmd/init/init.go index 8ffb129a327..07d9ec7b9a2 100644 --- a/cmd/init/init.go +++ b/cmd/init/init.go @@ -12,7 +12,7 @@ import ( const ConfigFileName = "config.json" const SchemaFileName = "schema.json" -const TemplateDirname = "template" +const TemplateDirName = "template" var initCmd = &cobra.Command{ Use: "init", @@ -58,7 +58,7 @@ var initCmd = &cobra.Command{ } // materialize the template - return template.WalkFileTree(config, filepath.Join(args[0], TemplateDirname), targetDir) + return template.WalkFileTree(config, filepath.Join(args[0], TemplateDirName), targetDir) }, } diff --git a/libs/template/execute.go b/libs/template/execute.go index d5fb88fb381..09cf8c59b43 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -7,11 +7,11 @@ import ( "text/template" ) -// Executes the template by appling config on it. Returns the materialized config +// Executes the template by applying config on it. Returns the materialized config // as a string func executeTemplate(config map[string]any, templateDefination string) (string, error) { // configure template with helper functions - tmpl, err := template.New("foo").Funcs(HelperFuncs).Parse(templateDefination) + tmpl, err := template.New("").Funcs(HelperFuncs).Parse(templateDefination) if err != nil { return "", err } diff --git a/libs/template/schema.go b/libs/template/schema.go index e8187fa540b..2023eaf906e 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -92,13 +92,6 @@ func validateType(v any, fieldType FieldType) error { // TODO: add validation check for regex for string types func (schema Schema) ValidateConfig(config map[string]any) error { - // assert all fields are defined in - for k := range schema { - if _, ok := config[k]; !ok { - return fmt.Errorf("input parameter %s is not defined in config", k) - } - } - // validate types defined in config for k, v := range config { fieldMetadata, ok := schema[k] @@ -110,5 +103,11 @@ func (schema Schema) ValidateConfig(config map[string]any) error { return fmt.Errorf("incorrect type for %s. %w", k, err) } } + // assert all fields are defined in + for k := range schema { + if _, ok := config[k]; !ok { + return fmt.Errorf("input parameter %s is not defined in config", k) + } + } return nil } From ecc07e893f4a7c56dbc2ff33e0c977028b3f5124 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 15:27:03 +0200 Subject: [PATCH 13/60] use errors is --- libs/template/execute.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libs/template/execute.go b/libs/template/execute.go index 09cf8c59b43..84da57b2675 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -1,6 +1,7 @@ package template import ( + "errors" "os" "path/filepath" "strings" @@ -41,10 +42,7 @@ func generateDirectory(config map[string]any, parentDir, nameTempate string) (st func generateFile(config map[string]any, parentDir, nameTempate, contentTemplate string) error { // compute file content fileContent, err := executeTemplate(config, contentTemplate) - // We do a substring match here because on errors the template library prepends - // some additional information about the callsite from which the ErrSkipThisFile - // error was returned - if err != nil && strings.Contains(err.Error(), ErrSkipThisFile.Error()) { + if errors.Is(err, ErrSkipThisFile) { return nil } if err != nil { From 9a8205b3dfab267b507f6c03db8b78e44b90d640 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 17 May 2023 15:33:04 +0200 Subject: [PATCH 14/60] address comments plus close file --- internal/init_test.go | 4 ++-- libs/template/execute.go | 14 ++++---------- libs/template/helpers.go | 4 ++-- libs/template/schema.go | 2 +- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/internal/init_test.go b/internal/init_test.go index b6f74252b69..793ee3a5e09 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -23,7 +23,6 @@ func TestTemplateInitializationForDevConfig(t *testing.T) { tmp := t.TempDir() f, err := os.Create(filepath.Join(tmp, "config.json")) require.NoError(t, err) - defer f.Close() _, err = f.WriteString(` { "project_name": "development_project", @@ -32,6 +31,7 @@ func TestTemplateInitializationForDevConfig(t *testing.T) { "is_production": false } `) + f.Close() require.NoError(t, err) // materialize the template @@ -53,7 +53,6 @@ func TestTemplateInitializationForProdConfig(t *testing.T) { tmp := t.TempDir() f, err := os.Create(filepath.Join(tmp, "config.json")) require.NoError(t, err) - defer f.Close() _, err = f.WriteString(` { "project_name": "production_project", @@ -62,6 +61,7 @@ func TestTemplateInitializationForProdConfig(t *testing.T) { "is_production": true } `) + f.Close() require.NoError(t, err) // materialize the template diff --git a/libs/template/execute.go b/libs/template/execute.go index 84da57b2675..bec58884f84 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -42,7 +42,7 @@ func generateDirectory(config map[string]any, parentDir, nameTempate string) (st func generateFile(config map[string]any, parentDir, nameTempate, contentTemplate string) error { // compute file content fileContent, err := executeTemplate(config, contentTemplate) - if errors.Is(err, ErrSkipThisFile) { + if errors.Is(err, errSkipThisFile) { return nil } if err != nil { @@ -54,22 +54,16 @@ func generateFile(config map[string]any, parentDir, nameTempate, contentTemplate if err != nil { return err } - f, err := os.Create(filepath.Join(parentDir, fileName)) - if err != nil { - return err - } - // write to file the computed content - _, err = f.Write([]byte(fileContent)) - return err + return os.WriteFile(filepath.Join(parentDir, fileName), []byte(fileContent), 0644) } func WalkFileTree(config map[string]any, templatePath, instancePath string) error { - enteries, err := os.ReadDir(templatePath) + entries, err := os.ReadDir(templatePath) if err != nil { return err } - for _, entry := range enteries { + for _, entry := range entries { if entry.IsDir() { // case: materialize a template directory dirName, err := generateDirectory(config, instancePath, entry.Name()) diff --git a/libs/template/helpers.go b/libs/template/helpers.go index ff8ae539eda..8e072bba135 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -5,10 +5,10 @@ import ( "text/template" ) -var ErrSkipThisFile = errors.New("skip generating this file") +var errSkipThisFile = errors.New("skip generating this file") var HelperFuncs = template.FuncMap{ "skipThisFile": func() error { - panic(ErrSkipThisFile) + panic(errSkipThisFile) }, } diff --git a/libs/template/schema.go b/libs/template/schema.go index 2023eaf906e..b536dc335c2 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -21,7 +21,7 @@ const ( type FieldInfo struct { Type FieldType `json:"type"` Description string `json:"description"` - Validation string `json:"validate"` + Validation string `json:"validation"` } // function to check whether a float value represents an integer From f5384fc5ed482503064b1d8525b9f0b9211993ad Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 19 May 2023 15:39:55 +0200 Subject: [PATCH 15/60] added a map for validators, and moved init under bundle --- cmd/{init => bundle}/init.go | 5 ++-- internal/init_test.go | 6 ++--- libs/template/schema.go | 26 +++---------------- libs/template/validators.go | 48 ++++++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 28 deletions(-) rename cmd/{init => bundle}/init.go (95%) create mode 100644 libs/template/validators.go diff --git a/cmd/init/init.go b/cmd/bundle/init.go similarity index 95% rename from cmd/init/init.go rename to cmd/bundle/init.go index 07d9ec7b9a2..498fad8d1f0 100644 --- a/cmd/init/init.go +++ b/cmd/bundle/init.go @@ -1,11 +1,10 @@ -package init +package bundle import ( "encoding/json" "os" "path/filepath" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/template" "github.com/spf13/cobra" ) @@ -66,5 +65,5 @@ var targetDir string func init() { initCmd.Flags().StringVar(&targetDir, "target-dir", ".", "path to directory template will be initialized in") - root.RootCmd.AddCommand(initCmd) + AddCommand(initCmd) } diff --git a/internal/init_test.go b/internal/init_test.go index 793ee3a5e09..5e679ea6f66 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - _ "github.com/databricks/cli/cmd/init" + _ "github.com/databricks/cli/cmd/bundle" "github.com/databricks/cli/cmd/root" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,7 +36,7 @@ func TestTemplateInitializationForDevConfig(t *testing.T) { // materialize the template cmd := root.RootCmd - cmd.SetArgs([]string{"init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", tmp}) + cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", tmp}) err = cmd.Execute() require.NoError(t, err) @@ -68,7 +68,7 @@ func TestTemplateInitializationForProdConfig(t *testing.T) { cmd := root.RootCmd childCommands := cmd.Commands() fmt.Println(childCommands) - cmd.SetArgs([]string{"init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", tmp}) + cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", tmp}) err = cmd.Execute() require.NoError(t, err) diff --git a/libs/template/schema.go b/libs/template/schema.go index b536dc335c2..9a9f2fe31e3 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -3,8 +3,6 @@ package template import ( "fmt" "reflect" - - "golang.org/x/exp/slices" ) type Schema map[string]FieldInfo @@ -67,27 +65,11 @@ func (schema Schema) CastFloatToInt(config map[string]any) error { } func validateType(v any, fieldType FieldType) error { - switch fieldType { - case FieldTypeString: - if _, ok := v.(string); !ok { - return fmt.Errorf("expected type string, but value is %#v", v) - } - case FieldTypeInt: - if !slices.Contains([]reflect.Kind{reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64}, - reflect.TypeOf(v).Kind()) { - return fmt.Errorf("expected type integer, but value is %#v", v) - } - case FieldTypeFloat: - if !slices.Contains([]reflect.Kind{reflect.Float32, reflect.Float64}, - reflect.TypeOf(v).Kind()) { - return fmt.Errorf("expected type float, but value is %#v", v) - } - case FieldTypeBoolean: - if _, ok := v.(bool); !ok { - return fmt.Errorf("expected type boolean, but value is %#v", v) - } + validateFunc, ok := validators[fieldType] + if !ok { + return nil } - return nil + return validateFunc(v) } // TODO: add validation check for regex for string types diff --git a/libs/template/validators.go b/libs/template/validators.go new file mode 100644 index 00000000000..39e7fe9c45b --- /dev/null +++ b/libs/template/validators.go @@ -0,0 +1,48 @@ +package template + +import ( + "fmt" + "reflect" + + "golang.org/x/exp/slices" +) + +type Validator func(v any) error + +// TODO: refactor tests into individual tests for individual validators +func validateString(v any) error { + if _, ok := v.(string); !ok { + return fmt.Errorf("expected type string, but value is %#v", v) + } + return nil +} + +func validateBoolean(v any) error { + if _, ok := v.(bool); !ok { + return fmt.Errorf("expected type boolean, but value is %#v", v) + } + return nil +} + +func validateFloat(v any) error { + if !slices.Contains([]reflect.Kind{reflect.Float32, reflect.Float64}, + reflect.TypeOf(v).Kind()) { + return fmt.Errorf("expected type float, but value is %#v", v) + } + return nil +} + +func validateInteger(v any) error { + if !slices.Contains([]reflect.Kind{reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64}, + reflect.TypeOf(v).Kind()) { + return fmt.Errorf("expected type integer, but value is %#v", v) + } + return nil +} + +var validators map[FieldType]Validator = map[FieldType]Validator{ + FieldTypeString: validateString, + FieldTypeBoolean: validateBoolean, + FieldTypeInt: validateInteger, + FieldTypeFloat: validateFloat, +} From a21fbdd3835cf0a97d7fe6f23a30ed87d5033761 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 21 May 2023 22:37:38 +0200 Subject: [PATCH 16/60] refactor logic out of the cmd --- cmd/bundle/init.go | 48 +----------------------------------- libs/template/materialize.go | 24 ++++++++++++++++++ libs/template/schema.go | 43 +++++++++++++++++++++++++++++++- libs/template/schema_test.go | 6 ++--- libs/template/validators.go | 1 + 5 files changed, 71 insertions(+), 51 deletions(-) create mode 100644 libs/template/materialize.go diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index 498fad8d1f0..973c709e486 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -1,63 +1,17 @@ package bundle import ( - "encoding/json" - "os" - "path/filepath" - "github.com/databricks/cli/libs/template" "github.com/spf13/cobra" ) -const ConfigFileName = "config.json" -const SchemaFileName = "schema.json" -const TemplateDirName = "template" - var initCmd = &cobra.Command{ Use: "init", Short: "Initialize Template", Long: `Initialize template`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - templateLocation := args[0] - - // read the file containing schema for template input parameters - schemaBytes, err := os.ReadFile(filepath.Join(templateLocation, SchemaFileName)) - if err != nil { - return err - } - schema := template.Schema{} - err = json.Unmarshal(schemaBytes, &schema) - if err != nil { - return err - } - - // read user config to initalize the template with - var config map[string]interface{} - b, err := os.ReadFile(filepath.Join(targetDir, ConfigFileName)) - if err != nil { - return err - } - err = json.Unmarshal(b, &config) - if err != nil { - return err - } - - // cast any fields that are supported to be integers. The json unmarshalling - // for a generic map converts all numbers to floating point - err = schema.CastFloatToInt(config) - if err != nil { - return err - } - - // validate config according to schema - err = schema.ValidateConfig(config) - if err != nil { - return err - } - - // materialize the template - return template.WalkFileTree(config, filepath.Join(args[0], TemplateDirName), targetDir) + return template.Materialize(args[0], targetDir) }, } diff --git a/libs/template/materialize.go b/libs/template/materialize.go new file mode 100644 index 00000000000..8501616c094 --- /dev/null +++ b/libs/template/materialize.go @@ -0,0 +1,24 @@ +package template + +import "path/filepath" + +const ConfigFileName = "config.json" +const SchemaFileName = "schema.json" +const TemplateDirName = "template" + +func Materialize(templatePath, instancePath string) error { + // read the file containing schema for template input parameters + schema, err := ReadSchema(filepath.Join(templatePath, SchemaFileName)) + if err != nil { + return err + } + + // read user config to initalize the template with + config, err := schema.ReadConfig(filepath.Join(instancePath, ConfigFileName)) + if err != nil { + return err + } + + // materialize the template + return WalkFileTree(config, filepath.Join(templatePath, TemplateDirName), instancePath) +} diff --git a/libs/template/schema.go b/libs/template/schema.go index 9a9f2fe31e3..0efbad5545a 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -1,7 +1,9 @@ package template import ( + "encoding/json" "fmt" + "os" "reflect" ) @@ -31,7 +33,7 @@ func isIntegerValue(v float64) bool { // integeres according to the schema // // Needed because the default json unmarshaller for maps converts all numbers to floats -func (schema Schema) CastFloatToInt(config map[string]any) error { +func castFloatToInt(config map[string]any, schema Schema) error { for k, v := range config { // error because all config keys should be defined in schema too if _, ok := schema[k]; !ok { @@ -93,3 +95,42 @@ func (schema Schema) ValidateConfig(config map[string]any) error { } return nil } + +func ReadSchema(path string) (Schema, error) { + schemaBytes, err := os.ReadFile(path) + if err != nil { + return nil, err + } + schema := Schema{} + err = json.Unmarshal(schemaBytes, &schema) + if err != nil { + return nil, err + } + return schema, nil +} + +func (schema Schema) ReadConfig(path string) (map[string]any, error) { + var config map[string]any + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + err = json.Unmarshal(b, &config) + if err != nil { + return nil, err + } + + // cast any fields that are supposed to be integers. The json unmarshalling + // for a generic map converts all numbers to floating point + err = castFloatToInt(config, schema) + if err != nil { + return nil, err + } + + // validate config according to schema + err = schema.ValidateConfig(config) + if err != nil { + return nil, err + } + return config, nil +} diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go index 123e9c9baa1..75d8f50346f 100644 --- a/libs/template/schema_test.go +++ b/libs/template/schema_test.go @@ -56,7 +56,7 @@ func TestTemplateSchemaCastFloatToInt(t *testing.T) { assert.IsType(t, true, config["bool_val"]) assert.IsType(t, "abc", config["string_val"]) - err = schema.CastFloatToInt(config) + err = castFloatToInt(config, schema) require.NoError(t, err) // assert type after casting, that the float value was converted to an integer @@ -86,7 +86,7 @@ func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { err = json.Unmarshal([]byte(configJson), &config) require.NoError(t, err) - err = schema.CastFloatToInt(config) + err = castFloatToInt(config, schema) assert.ErrorContains(t, err, "bar is not defined as an input parameter for the template") } @@ -109,7 +109,7 @@ func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { err = json.Unmarshal([]byte(configJson), &config) require.NoError(t, err) - err = schema.CastFloatToInt(config) + err = castFloatToInt(config, schema) assert.ErrorContains(t, err, "expected foo to have integer value but it is 1.1") } diff --git a/libs/template/validators.go b/libs/template/validators.go index 39e7fe9c45b..00034416739 100644 --- a/libs/template/validators.go +++ b/libs/template/validators.go @@ -46,3 +46,4 @@ var validators map[FieldType]Validator = map[FieldType]Validator{ FieldTypeInt: validateInteger, FieldTypeFloat: validateFloat, } + From 350053487a3b73d5d94f9f47f61fe5985cd1b8a3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 21 May 2023 22:48:38 +0200 Subject: [PATCH 17/60] added flag for config file location --- cmd/bundle/init.go | 8 ++++++++ libs/template/materialize.go | 12 +++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index 973c709e486..2cc2ffbed1c 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -1,6 +1,8 @@ package bundle import ( + "path/filepath" + "github.com/databricks/cli/libs/template" "github.com/spf13/cobra" ) @@ -11,13 +13,19 @@ var initCmd = &cobra.Command{ Long: `Initialize template`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + // initialize default value for config file path + if configFile == "" { + configFile = filepath.Join(targetDir, template.ConfigFileName) + } return template.Materialize(args[0], targetDir) }, } var targetDir string +var configFile string func init() { initCmd.Flags().StringVar(&targetDir, "target-dir", ".", "path to directory template will be initialized in") + initCmd.Flags().StringVar(&configFile, "config-file", "", "path to config to use for template initialization") AddCommand(initCmd) } diff --git a/libs/template/materialize.go b/libs/template/materialize.go index 8501616c094..57963d126b4 100644 --- a/libs/template/materialize.go +++ b/libs/template/materialize.go @@ -1,14 +1,16 @@ package template -import "path/filepath" +import ( + "path/filepath" +) const ConfigFileName = "config.json" -const SchemaFileName = "schema.json" -const TemplateDirName = "template" +const schemaFileName = "schema.json" +const templateDirName = "template" func Materialize(templatePath, instancePath string) error { // read the file containing schema for template input parameters - schema, err := ReadSchema(filepath.Join(templatePath, SchemaFileName)) + schema, err := ReadSchema(filepath.Join(templatePath, schemaFileName)) if err != nil { return err } @@ -20,5 +22,5 @@ func Materialize(templatePath, instancePath string) error { } // materialize the template - return WalkFileTree(config, filepath.Join(templatePath, TemplateDirName), instancePath) + return WalkFileTree(config, filepath.Join(templatePath, templateDirName), instancePath) } From 479e2b4e32ce303bb7aaabba9231bc75f5188367 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 22 May 2023 05:58:05 +0200 Subject: [PATCH 18/60] Add config-file flag --- cmd/bundle/init.go | 3 ++- internal/init_test.go | 24 +++++++++++++++++------- libs/template/materialize.go | 4 ++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index 2cc2ffbed1c..bfca7735dc4 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -14,10 +14,11 @@ var initCmd = &cobra.Command{ Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { // initialize default value for config file path + // TODO: add a test for this if configFile == "" { configFile = filepath.Join(targetDir, template.ConfigFileName) } - return template.Materialize(args[0], targetDir) + return template.Materialize(args[0], targetDir, configFile) }, } diff --git a/internal/init_test.go b/internal/init_test.go index 5e679ea6f66..98c378b5918 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -51,7 +51,12 @@ func TestTemplateInitializationForDevConfig(t *testing.T) { func TestTemplateInitializationForProdConfig(t *testing.T) { // create target directory with the input config tmp := t.TempDir() - f, err := os.Create(filepath.Join(tmp, "config.json")) + + // create target directory to with the input config + configDir := filepath.Join(tmp, "dir-with-config") + err := os.Mkdir(configDir, os.ModePerm) + require.NoError(t, err) + f, err := os.Create(filepath.Join(configDir, "my_config.json")) require.NoError(t, err) _, err = f.WriteString(` { @@ -64,18 +69,23 @@ func TestTemplateInitializationForProdConfig(t *testing.T) { f.Close() require.NoError(t, err) + // create directory to initialize the template instance within + instanceDir := filepath.Join(tmp, "dir-with-instance") + err = os.Mkdir(instanceDir, os.ModePerm) + require.NoError(t, err) + // materialize the template cmd := root.RootCmd childCommands := cmd.Commands() fmt.Println(childCommands) - cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", tmp}) + cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", instanceDir, "--config-file", filepath.Join(configDir, "my_config.json")}) err = cmd.Execute() require.NoError(t, err) // assert on materialized template - assert.FileExists(t, filepath.Join(tmp, "production_project", "azure_file")) - assert.FileExists(t, filepath.Join(tmp, "production_project", ".azure_devops")) - assert.NoFileExists(t, filepath.Join(tmp, "production_project", "aws_file")) - assertFileContains(t, filepath.Join(tmp, "production_project", "azure_file"), "This file should only be generated for Azure") - assertFileContains(t, filepath.Join(tmp, "production_project", ".azure_devops"), "This is a production project") + assert.FileExists(t, filepath.Join(instanceDir, "production_project", "azure_file")) + assert.FileExists(t, filepath.Join(instanceDir, "production_project", ".azure_devops")) + assert.NoFileExists(t, filepath.Join(instanceDir, "production_project", "aws_file")) + assertFileContains(t, filepath.Join(instanceDir, "production_project", "azure_file"), "This file should only be generated for Azure") + assertFileContains(t, filepath.Join(instanceDir, "production_project", ".azure_devops"), "This is a production project") } diff --git a/libs/template/materialize.go b/libs/template/materialize.go index 57963d126b4..07d2f05782f 100644 --- a/libs/template/materialize.go +++ b/libs/template/materialize.go @@ -8,7 +8,7 @@ const ConfigFileName = "config.json" const schemaFileName = "schema.json" const templateDirName = "template" -func Materialize(templatePath, instancePath string) error { +func Materialize(templatePath, instancePath, configPath string) error { // read the file containing schema for template input parameters schema, err := ReadSchema(filepath.Join(templatePath, schemaFileName)) if err != nil { @@ -16,7 +16,7 @@ func Materialize(templatePath, instancePath string) error { } // read user config to initalize the template with - config, err := schema.ReadConfig(filepath.Join(instancePath, ConfigFileName)) + config, err := schema.ReadConfig(configPath) if err != nil { return err } From f6fc63a09e7844eadf01f8cbee5d66be9b15d1c4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 22 May 2023 09:46:26 +0200 Subject: [PATCH 19/60] lazy generate dirs --- libs/template/execute.go | 39 +++++++++++++++++------------------- libs/template/materialize.go | 2 +- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/libs/template/execute.go b/libs/template/execute.go index bec58884f84..8563c14a708 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -27,53 +27,50 @@ func executeTemplate(config map[string]any, templateDefination string) (string, } // TODO: allow skipping directories -func generateDirectory(config map[string]any, parentDir, nameTempate string) (string, error) { - dirName, err := executeTemplate(config, nameTempate) - if err != nil { - return "", err - } - err = os.Mkdir(filepath.Join(parentDir, dirName), 0755) - if err != nil { - return "", err - } - return dirName, nil -} -func generateFile(config map[string]any, parentDir, nameTempate, contentTemplate string) error { +func generateFile(config map[string]any, pathTemplate, contentTemplate string) error { // compute file content fileContent, err := executeTemplate(config, contentTemplate) if errors.Is(err, errSkipThisFile) { + // skip this file return nil } if err != nil { return err } - // create the file by executing the templatized file name - fileName, err := executeTemplate(config, nameTempate) + // compute the path for this file + path, err := executeTemplate(config, pathTemplate) + if err != nil { + return err + } + // create any intermediate directories required. Directories are lazily generated + // only when they are required for a file. + err = os.MkdirAll(filepath.Dir(path), 0755) if err != nil { return err } - return os.WriteFile(filepath.Join(parentDir, fileName), []byte(fileContent), 0644) + // write content to file + return os.WriteFile(path, []byte(fileContent), 0644) } -func WalkFileTree(config map[string]any, templatePath, instancePath string) error { +func walkFileTree(config map[string]any, templatePath, instancePath string) error { entries, err := os.ReadDir(templatePath) if err != nil { return err } for _, entry := range entries { if entry.IsDir() { - // case: materialize a template directory - dirName, err := generateDirectory(config, instancePath, entry.Name()) + // compute directory name + dirName, err := executeTemplate(config, entry.Name()) if err != nil { return err } - // recusive generate files and directories inside inside our newly generated + // recusively generate files and directories inside inside our newly generated // directory from the template defination - err = WalkFileTree(config, filepath.Join(templatePath, entry.Name()), filepath.Join(instancePath, dirName)) + err = walkFileTree(config, filepath.Join(templatePath, entry.Name()), filepath.Join(instancePath, dirName)) if err != nil { return err } @@ -85,7 +82,7 @@ func WalkFileTree(config map[string]any, templatePath, instancePath string) erro } contentTemplate := string(b) fileNameTemplate := entry.Name() - err = generateFile(config, instancePath, fileNameTemplate, contentTemplate) + err = generateFile(config, filepath.Join(instancePath, fileNameTemplate), contentTemplate) if err != nil { return err } diff --git a/libs/template/materialize.go b/libs/template/materialize.go index 07d2f05782f..15bb1a0e111 100644 --- a/libs/template/materialize.go +++ b/libs/template/materialize.go @@ -22,5 +22,5 @@ func Materialize(templatePath, instancePath, configPath string) error { } // materialize the template - return WalkFileTree(config, filepath.Join(templatePath, templateDirName), instancePath) + return walkFileTree(config, filepath.Join(templatePath, templateDirName), instancePath) } From 2511c183527959a77f18016065460ea0c6a0a9c5 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 22 May 2023 10:16:40 +0200 Subject: [PATCH 20/60] add test for dir gen skip --- libs/template/materialize_test.go | 54 +++++++++++++++++++ libs/template/testdata/skip_dir/schema.json | 14 +++++ .../skip_dir/template/{{.a}}/.gitkeep | 0 .../testdata/skip_dir/template/{{.c}}/foo | 4 ++ 4 files changed, 72 insertions(+) create mode 100644 libs/template/materialize_test.go create mode 100644 libs/template/testdata/skip_dir/schema.json create mode 100644 libs/template/testdata/skip_dir/template/{{.a}}/.gitkeep create mode 100644 libs/template/testdata/skip_dir/template/{{.c}}/foo diff --git a/libs/template/materialize_test.go b/libs/template/materialize_test.go new file mode 100644 index 00000000000..dff678fa619 --- /dev/null +++ b/libs/template/materialize_test.go @@ -0,0 +1,54 @@ +package template + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupConfig(t *testing.T, config string) string { + // create target directory with the input config + tmp := t.TempDir() + f, err := os.Create(filepath.Join(tmp, "config.json")) + require.NoError(t, err) + _, err = f.WriteString(config) + f.Close() + require.NoError(t, err) + return tmp +} + +func TestMaterializeEmptyDirsAreNotGenerated(t *testing.T) { + tmp := setupConfig(t, ` + { + "a": "dir-with-file", + "b": "empty-dir", + "c": "dir-with-skipped-file", + "d": "skipping" + }`) + err := Materialize("./testdata/skip_dir", tmp, filepath.Join(tmp, "config.json")) + require.NoError(t, err) + + assert.DirExists(t, filepath.Join(tmp, "dir-with-file")) + assert.FileExists(t, filepath.Join(tmp, "dir-with-file/.gitkeep")) + assert.NoDirExists(t, filepath.Join(tmp, "empty-dir")) + assert.NoDirExists(t, filepath.Join(tmp, "dir-with-skipped-file")) + + tmp2 := setupConfig(t, ` + { + "a": "dir-with-file", + "b": "empty-dir", + "c": "dir-not-skipped-this-time", + "d": "not-skipping" + }`) + err = Materialize("./testdata/skip_dir", tmp2, filepath.Join(tmp2, "config.json")) + require.NoError(t, err) + + assert.DirExists(t, filepath.Join(tmp2, "dir-with-file")) + assert.FileExists(t, filepath.Join(tmp2, "dir-with-file/.gitkeep")) + assert.NoDirExists(t, filepath.Join(tmp2, "empty-dir")) + assert.DirExists(t, filepath.Join(tmp2, "dir-not-skipped-this-time")) + assert.FileExists(t, filepath.Join(tmp2, "dir-not-skipped-this-time/foo")) +} diff --git a/libs/template/testdata/skip_dir/schema.json b/libs/template/testdata/skip_dir/schema.json new file mode 100644 index 00000000000..9687bce7661 --- /dev/null +++ b/libs/template/testdata/skip_dir/schema.json @@ -0,0 +1,14 @@ +{ + "a": { + "type": "string" + }, + "b": { + "type": "string" + }, + "c": { + "type": "string" + }, + "d": { + "type": "string" + } +} diff --git a/libs/template/testdata/skip_dir/template/{{.a}}/.gitkeep b/libs/template/testdata/skip_dir/template/{{.a}}/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/libs/template/testdata/skip_dir/template/{{.c}}/foo b/libs/template/testdata/skip_dir/template/{{.c}}/foo new file mode 100644 index 00000000000..9925ff1dce6 --- /dev/null +++ b/libs/template/testdata/skip_dir/template/{{.c}}/foo @@ -0,0 +1,4 @@ +{{if eq .d "skipping"}} +{{skipThisFile}} +{{end}} +Hello! From 2125f256e5b83e069c8181f9eebb514d4cde130c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 22 May 2023 10:30:27 +0200 Subject: [PATCH 21/60] lint --- libs/template/validators.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/template/validators.go b/libs/template/validators.go index 00034416739..39e7fe9c45b 100644 --- a/libs/template/validators.go +++ b/libs/template/validators.go @@ -46,4 +46,3 @@ var validators map[FieldType]Validator = map[FieldType]Validator{ FieldTypeInt: validateInteger, FieldTypeFloat: validateFloat, } - From 8a9e59df3e0362311c2b00e68fde0dacb2d4ef9a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 22 May 2023 10:43:43 +0200 Subject: [PATCH 22/60] add tests for individual validators --- cmd/bundle/init.go | 1 - libs/template/execute.go | 2 - libs/template/schema.go | 1 - libs/template/validators.go | 1 - libs/template/validators_test.go | 75 ++++++++++++++++++++++++++++++++ 5 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 libs/template/validators_test.go diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index bfca7735dc4..236554784fc 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -14,7 +14,6 @@ var initCmd = &cobra.Command{ Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { // initialize default value for config file path - // TODO: add a test for this if configFile == "" { configFile = filepath.Join(targetDir, template.ConfigFileName) } diff --git a/libs/template/execute.go b/libs/template/execute.go index 8563c14a708..9d247132171 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -26,8 +26,6 @@ func executeTemplate(config map[string]any, templateDefination string) (string, return result.String(), nil } -// TODO: allow skipping directories - func generateFile(config map[string]any, pathTemplate, contentTemplate string) error { // compute file content fileContent, err := executeTemplate(config, contentTemplate) diff --git a/libs/template/schema.go b/libs/template/schema.go index 0efbad5545a..9759eee6660 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -74,7 +74,6 @@ func validateType(v any, fieldType FieldType) error { return validateFunc(v) } -// TODO: add validation check for regex for string types func (schema Schema) ValidateConfig(config map[string]any) error { // validate types defined in config for k, v := range config { diff --git a/libs/template/validators.go b/libs/template/validators.go index 39e7fe9c45b..4442173f00c 100644 --- a/libs/template/validators.go +++ b/libs/template/validators.go @@ -9,7 +9,6 @@ import ( type Validator func(v any) error -// TODO: refactor tests into individual tests for individual validators func validateString(v any) error { if _, ok := v.(string); !ok { return fmt.Errorf("expected type string, but value is %#v", v) diff --git a/libs/template/validators_test.go b/libs/template/validators_test.go new file mode 100644 index 00000000000..5621ef9285d --- /dev/null +++ b/libs/template/validators_test.go @@ -0,0 +1,75 @@ +package template + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidatorString(t *testing.T) { + err := validateString("abc") + assert.NoError(t, err) + + err = validateString(1) + assert.ErrorContains(t, err, "expected type string, but value is 1") + + err = validateString(true) + assert.ErrorContains(t, err, "expected type string, but value is true") + + err = validateString("false") + assert.NoError(t, err) +} + +func TestValidatorBoolean(t *testing.T) { + err := validateBoolean(true) + assert.NoError(t, err) + + err = validateBoolean(1) + assert.ErrorContains(t, err, "expected type boolean, but value is 1") + + err = validateBoolean("abc") + assert.ErrorContains(t, err, "expected type boolean, but value is \"abc\"") + + err = validateBoolean("false") + assert.ErrorContains(t, err, "expected type boolean, but value is \"false\"") +} + +func TestValidatorFloat(t *testing.T) { + err := validateFloat(true) + assert.ErrorContains(t, err, "expected type float, but value is true") + + err = validateFloat(int32(1)) + assert.ErrorContains(t, err, "expected type float, but value is 1") + + err = validateFloat(int64(1)) + assert.ErrorContains(t, err, "expected type float, but value is 1") + + err = validateFloat(float32(1)) + assert.NoError(t, err) + + err = validateFloat(float64(1)) + assert.NoError(t, err) + + err = validateFloat("abc") + assert.ErrorContains(t, err, "expected type float, but value is \"abc\"") +} + +func TestValidatorInt(t *testing.T) { + err := validateInteger(true) + assert.ErrorContains(t, err, "expected type integer, but value is true") + + err = validateInteger(int32(1)) + assert.NoError(t, err) + + err = validateInteger(int64(1)) + assert.NoError(t, err) + + err = validateInteger(float32(1)) + assert.ErrorContains(t, err, "expected type integer, but value is 1") + + err = validateInteger(float64(1)) + assert.ErrorContains(t, err, "expected type integer, but value is 1") + + err = validateInteger("abc") + assert.ErrorContains(t, err, "expected type integer, but value is \"abc\"") +} From 89d02cda6bbb73e3ce3b2d9dab0359f91a3dedb3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 22 May 2023 10:48:32 +0200 Subject: [PATCH 23/60] nit --- libs/template/materialize_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libs/template/materialize_test.go b/libs/template/materialize_test.go index dff678fa619..d5149eede47 100644 --- a/libs/template/materialize_test.go +++ b/libs/template/materialize_test.go @@ -24,7 +24,7 @@ func TestMaterializeEmptyDirsAreNotGenerated(t *testing.T) { tmp := setupConfig(t, ` { "a": "dir-with-file", - "b": "empty-dir", + "b": "foo", "c": "dir-with-skipped-file", "d": "skipping" }`) @@ -39,7 +39,7 @@ func TestMaterializeEmptyDirsAreNotGenerated(t *testing.T) { tmp2 := setupConfig(t, ` { "a": "dir-with-file", - "b": "empty-dir", + "b": "foo", "c": "dir-not-skipped-this-time", "d": "not-skipping" }`) @@ -48,7 +48,6 @@ func TestMaterializeEmptyDirsAreNotGenerated(t *testing.T) { assert.DirExists(t, filepath.Join(tmp2, "dir-with-file")) assert.FileExists(t, filepath.Join(tmp2, "dir-with-file/.gitkeep")) - assert.NoDirExists(t, filepath.Join(tmp2, "empty-dir")) assert.DirExists(t, filepath.Join(tmp2, "dir-not-skipped-this-time")) assert.FileExists(t, filepath.Join(tmp2, "dir-not-skipped-this-time/foo")) } From 601e16e9e8e2a3c0b74aec303ceec9078e4f0c86 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 22 May 2023 11:06:10 +0200 Subject: [PATCH 24/60] add version of schema --- .../init/templateDefinition/schema.json | 31 ++++---- libs/template/schema.go | 30 ++++--- libs/template/schema_test.go | 78 ++++++++++++------- libs/template/testdata/skip_dir/schema.json | 25 +++--- 4 files changed, 98 insertions(+), 66 deletions(-) diff --git a/internal/testdata/init/templateDefinition/schema.json b/internal/testdata/init/templateDefinition/schema.json index 5f927011b0f..7a682f6da1f 100644 --- a/internal/testdata/init/templateDefinition/schema.json +++ b/internal/testdata/init/templateDefinition/schema.json @@ -1,17 +1,20 @@ { - "project_name": { - "description": "Name of the project", - "type": "string" - }, - "cloud_type": { - "description": "type of the cloud for the project", - "type": "string" - }, - "is_production": { - "type": "boolean" - }, - "ci_type": { - "type": "string", - "description": "type of the CI runner, eg: github, azure devops" + "version": 0, + "properties": { + "project_name": { + "description": "Name of the project", + "type": "string" + }, + "cloud_type": { + "description": "type of the cloud for the project", + "type": "string" + }, + "is_production": { + "type": "boolean" + }, + "ci_type": { + "type": "string", + "description": "type of the CI runner, eg: github, azure devops" + } } } diff --git a/libs/template/schema.go b/libs/template/schema.go index 9759eee6660..14bacf0ee40 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -7,7 +7,15 @@ import ( "reflect" ) -type Schema map[string]FieldInfo +const LatestSchemaVersion = 0 + +type Schema struct { + // A version for the template schema + Version int `json:"version"` + + // A list of properties that can be used in the config + Properties map[string]FieldInfo `json:"properties"` +} type FieldType string @@ -33,15 +41,15 @@ func isIntegerValue(v float64) bool { // integeres according to the schema // // Needed because the default json unmarshaller for maps converts all numbers to floats -func castFloatToInt(config map[string]any, schema Schema) error { +func castFloatToInt(config map[string]any, schema *Schema) error { for k, v := range config { // error because all config keys should be defined in schema too - if _, ok := schema[k]; !ok { + if _, ok := schema.Properties[k]; !ok { return fmt.Errorf("%s is not defined as an input parameter for the template", k) } // skip non integer fields - fieldInfo := schema[k] + fieldInfo := schema.Properties[k] if fieldInfo.Type != FieldTypeInt { continue } @@ -74,10 +82,10 @@ func validateType(v any, fieldType FieldType) error { return validateFunc(v) } -func (schema Schema) ValidateConfig(config map[string]any) error { +func (schema *Schema) ValidateConfig(config map[string]any) error { // validate types defined in config for k, v := range config { - fieldMetadata, ok := schema[k] + fieldMetadata, ok := schema.Properties[k] if !ok { return fmt.Errorf("%s is not defined as an input parameter for the template", k) } @@ -87,7 +95,7 @@ func (schema Schema) ValidateConfig(config map[string]any) error { } } // assert all fields are defined in - for k := range schema { + for k := range schema.Properties { if _, ok := config[k]; !ok { return fmt.Errorf("input parameter %s is not defined in config", k) } @@ -95,20 +103,20 @@ func (schema Schema) ValidateConfig(config map[string]any) error { return nil } -func ReadSchema(path string) (Schema, error) { +func ReadSchema(path string) (*Schema, error) { schemaBytes, err := os.ReadFile(path) if err != nil { return nil, err } - schema := Schema{} - err = json.Unmarshal(schemaBytes, &schema) + schema := &Schema{} + err = json.Unmarshal(schemaBytes, schema) if err != nil { return nil, err } return schema, nil } -func (schema Schema) ReadConfig(path string) (map[string]any, error) { +func (schema *Schema) ReadConfig(path string) (map[string]any, error) { var config map[string]any b, err := os.ReadFile(path) if err != nil { diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go index 75d8f50346f..cac1da814b5 100644 --- a/libs/template/schema_test.go +++ b/libs/template/schema_test.go @@ -21,17 +21,20 @@ func TestTemplateSchematIsInterger(t *testing.T) { func TestTemplateSchemaCastFloatToInt(t *testing.T) { // define schema for config schemaJson := `{ - "int_val": { - "type": "integer" - }, - "float_val": { - "type": "float" - }, - "bool_val": { - "type": "boolean" - }, - "string_val": { - "type": "string" + "version": 0, + "properties": { + "int_val": { + "type": "integer" + }, + "float_val": { + "type": "float" + }, + "bool_val": { + "type": "boolean" + }, + "string_val": { + "type": "string" + } } }` var schema Schema @@ -56,7 +59,7 @@ func TestTemplateSchemaCastFloatToInt(t *testing.T) { assert.IsType(t, true, config["bool_val"]) assert.IsType(t, "abc", config["string_val"]) - err = castFloatToInt(config, schema) + err = castFloatToInt(config, &schema) require.NoError(t, err) // assert type after casting, that the float value was converted to an integer @@ -70,8 +73,11 @@ func TestTemplateSchemaCastFloatToInt(t *testing.T) { func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { // define schema for config schemaJson := `{ - "foo": { - "type": "integer" + "version": 0, + "properties": { + "foo": { + "type": "integer" + } } }` var schema Schema @@ -86,15 +92,18 @@ func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { err = json.Unmarshal([]byte(configJson), &config) require.NoError(t, err) - err = castFloatToInt(config, schema) + err = castFloatToInt(config, &schema) assert.ErrorContains(t, err, "bar is not defined as an input parameter for the template") } func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { // define schema for config schemaJson := `{ - "foo": { - "type": "integer" + "version": 0, + "properties": { + "foo": { + "type": "integer" + } } }` var schema Schema @@ -109,7 +118,7 @@ func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { err = json.Unmarshal([]byte(configJson), &config) require.NoError(t, err) - err = castFloatToInt(config, schema) + err = castFloatToInt(config, &schema) assert.ErrorContains(t, err, "expected foo to have integer value but it is 1.1") } @@ -172,17 +181,20 @@ func TestTemplateSchemaValidateType(t *testing.T) { func TestTemplateSchemaValidateConfig(t *testing.T) { // define schema for config schemaJson := `{ - "int_val": { - "type": "integer" - }, - "float_val": { - "type": "float" - }, - "bool_val": { - "type": "boolean" - }, - "string_val": { - "type": "string" + "version": 0, + "properties": { + "int_val": { + "type": "integer" + }, + "float_val": { + "type": "float" + }, + "bool_val": { + "type": "boolean" + }, + "string_val": { + "type": "string" + } } }` var schema Schema @@ -236,6 +248,8 @@ func TestTemplateSchemaValidateConfigFailsForUnknownField(t *testing.T) { func TestTemplateSchemaValidateConfigFailsForWhenIncorrectTypes(t *testing.T) { // define schema for config schemaJson := `{ + "version": 0, + "properties": { "int_val": { "type": "integer" }, @@ -248,7 +262,8 @@ func TestTemplateSchemaValidateConfigFailsForWhenIncorrectTypes(t *testing.T) { "string_val": { "type": "string" } - }` + } + }` var schema Schema err := json.Unmarshal([]byte(schemaJson), &schema) require.NoError(t, err) @@ -268,12 +283,15 @@ func TestTemplateSchemaValidateConfigFailsForWhenIncorrectTypes(t *testing.T) { func TestTemplateSchemaValidateConfigFailsForWhenMissingInputParams(t *testing.T) { // define schema for config schemaJson := `{ + "version": 0, + "properties": { "int_val": { "type": "integer" }, "string_val": { "type": "string" } + } }` var schema Schema err := json.Unmarshal([]byte(schemaJson), &schema) diff --git a/libs/template/testdata/skip_dir/schema.json b/libs/template/testdata/skip_dir/schema.json index 9687bce7661..945b7df8447 100644 --- a/libs/template/testdata/skip_dir/schema.json +++ b/libs/template/testdata/skip_dir/schema.json @@ -1,14 +1,17 @@ { - "a": { - "type": "string" - }, - "b": { - "type": "string" - }, - "c": { - "type": "string" - }, - "d": { - "type": "string" + "version": 0, + "properties": { + "a": { + "type": "string" + }, + "b": { + "type": "string" + }, + "c": { + "type": "string" + }, + "d": { + "type": "string" + } } } From 74627160cb88e5e18e922d5faa15d06d05e7cfa3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 22 May 2023 12:31:18 +0200 Subject: [PATCH 25/60] - --- libs/template/schema_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go index cac1da814b5..41afc0bba71 100644 --- a/libs/template/schema_test.go +++ b/libs/template/schema_test.go @@ -216,6 +216,8 @@ func TestTemplateSchemaValidateConfig(t *testing.T) { func TestTemplateSchemaValidateConfigFailsForUnknownField(t *testing.T) { // define schema for config schemaJson := `{ + "version": 0, + "properties": { "int_val": { "type": "integer" }, @@ -228,7 +230,8 @@ func TestTemplateSchemaValidateConfigFailsForUnknownField(t *testing.T) { "string_val": { "type": "string" } - }` + } + }` var schema Schema err := json.Unmarshal([]byte(schemaJson), &schema) require.NoError(t, err) From 4ce4b3aea10f1d29cb0bbc4a57240f65c14d4b56 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 23 May 2023 15:47:48 +0200 Subject: [PATCH 26/60] using stdlib walk and added passing file bits --- libs/template/execute.go | 75 ++++++++++++++++--------------- libs/template/materialize.go | 6 +-- libs/template/materialize_test.go | 71 +++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 39 deletions(-) diff --git a/libs/template/execute.go b/libs/template/execute.go index 9d247132171..6f63a7acf92 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -2,6 +2,7 @@ package template import ( "errors" + "io/fs" "os" "path/filepath" "strings" @@ -10,9 +11,10 @@ import ( // Executes the template by applying config on it. Returns the materialized config // as a string -func executeTemplate(config map[string]any, templateDefination string) (string, error) { +// TODO: test this function +func executeTemplate(config map[string]any, templateDefinition string) (string, error) { // configure template with helper functions - tmpl, err := template.New("").Funcs(HelperFuncs).Parse(templateDefination) + tmpl, err := template.New("").Funcs(HelperFuncs).Parse(templateDefinition) if err != nil { return "", err } @@ -26,7 +28,8 @@ func executeTemplate(config map[string]any, templateDefination string) (string, return result.String(), nil } -func generateFile(config map[string]any, pathTemplate, contentTemplate string) error { +// TODO: test this function +func generateFile(config map[string]any, pathTemplate, contentTemplate string, perm fs.FileMode) error { // compute file content fileContent, err := executeTemplate(config, contentTemplate) if errors.Is(err, errSkipThisFile) { @@ -50,41 +53,41 @@ func generateFile(config map[string]any, pathTemplate, contentTemplate string) e } // write content to file - return os.WriteFile(path, []byte(fileContent), 0644) + return os.WriteFile(path, []byte(fileContent), perm) } -func walkFileTree(config map[string]any, templatePath, instancePath string) error { - entries, err := os.ReadDir(templatePath) - if err != nil { - return err - } - for _, entry := range entries { - if entry.IsDir() { - // compute directory name - dirName, err := executeTemplate(config, entry.Name()) - if err != nil { - return err - } +func walkFileTree(config map[string]any, templateRoot, instanceRoot string) error { + return filepath.WalkDir(templateRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } - // recusively generate files and directories inside inside our newly generated - // directory from the template defination - err = walkFileTree(config, filepath.Join(templatePath, entry.Name()), filepath.Join(instancePath, dirName)) - if err != nil { - return err - } - } else { - // case: materialize a template file with it's contents - b, err := os.ReadFile(filepath.Join(templatePath, entry.Name())) - if err != nil { - return err - } - contentTemplate := string(b) - fileNameTemplate := entry.Name() - err = generateFile(config, filepath.Join(instancePath, fileNameTemplate), contentTemplate) - if err != nil { - return err - } + // skip if current entry is a directory + if d.IsDir() { + return nil } - } - return nil + + // read template file to get the templatized content for the file + b, err := os.ReadFile(path) + if err != nil { + return err + } + contentTemplate := string(b) + + // get relative path to the template file, This forms the template for the + // path to the file + relPathTemplate, err := filepath.Rel(templateRoot, path) + if err != nil { + return err + } + + // Get info about the template file. Used to ensure instance path also + // has the same permission bits + info, err := d.Info() + if err != nil { + return err + } + + return generateFile(config, filepath.Join(instanceRoot, relPathTemplate), contentTemplate, info.Mode().Perm()) + }) } diff --git a/libs/template/materialize.go b/libs/template/materialize.go index 15bb1a0e111..b50b7a4f200 100644 --- a/libs/template/materialize.go +++ b/libs/template/materialize.go @@ -8,9 +8,9 @@ const ConfigFileName = "config.json" const schemaFileName = "schema.json" const templateDirName = "template" -func Materialize(templatePath, instancePath, configPath string) error { +func Materialize(templateRoot, instanceRoot, configPath string) error { // read the file containing schema for template input parameters - schema, err := ReadSchema(filepath.Join(templatePath, schemaFileName)) + schema, err := ReadSchema(filepath.Join(templateRoot, schemaFileName)) if err != nil { return err } @@ -22,5 +22,5 @@ func Materialize(templatePath, instancePath, configPath string) error { } // materialize the template - return walkFileTree(config, filepath.Join(templatePath, templateDirName), instancePath) + return walkFileTree(config, filepath.Join(templateRoot, templateDirName), instanceRoot) } diff --git a/libs/template/materialize_test.go b/libs/template/materialize_test.go index d5149eede47..07fac5184ac 100644 --- a/libs/template/materialize_test.go +++ b/libs/template/materialize_test.go @@ -1,6 +1,7 @@ package template import ( + "io/fs" "os" "path/filepath" "testing" @@ -20,6 +21,12 @@ func setupConfig(t *testing.T, config string) string { return tmp } +func assertFilePerm(t *testing.T, path string, perm fs.FileMode) { + stat, err := os.Stat(path) + require.NoError(t, err) + assert.Equal(t, stat.Mode().Perm(), perm) +} + func TestMaterializeEmptyDirsAreNotGenerated(t *testing.T) { tmp := setupConfig(t, ` { @@ -51,3 +58,67 @@ func TestMaterializeEmptyDirsAreNotGenerated(t *testing.T) { assert.DirExists(t, filepath.Join(tmp2, "dir-not-skipped-this-time")) assert.FileExists(t, filepath.Join(tmp2, "dir-not-skipped-this-time/foo")) } + +func TestMaterializedTemplatesHaveIdenticalFilePermissionsAsTemplate(t *testing.T) { + // create template + tmp := t.TempDir() + err := os.Mkdir(filepath.Join(tmp, "my_tmpl"), 0777) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "schema.json"), []byte(` + { + "version": 0, + "properties": { + "a": { + "type": "string" + }, + "b": { + "type": "string" + } + } + }`), 0644) + require.NoError(t, err) + + // A normal file with the executable bit not flipped + err = os.Mkdir(filepath.Join(tmp, "my_tmpl", "template"), 0777) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "{{.a}}"), []byte("abc"), 0600) + require.NoError(t, err) + + // A read only file + err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "{{.b}}"), []byte("def"), 0400) + require.NoError(t, err) + + // A read only executable file + err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "foo"), []byte("ghi"), 0500) + require.NoError(t, err) + + // An executable script, accessable by non user access classes + err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "bar"), []byte("ghi"), 0755) + require.NoError(t, err) + + // create config.json file + err = os.Mkdir(filepath.Join(tmp, "config"), 0777) + require.NoError(t, err) + configPath := filepath.Join(tmp, "config", "config.json") + err = os.WriteFile(configPath, []byte(` + { + "a": "Amsterdam", + "b": "Hague" + }`), 0644) + require.NoError(t, err) + + // create directory to initialize the template in + instanceRoot := filepath.Join(tmp, "instance") + err = os.Mkdir(instanceRoot, 0777) + require.NoError(t, err) + + // materialize the template + err = Materialize(filepath.Join(tmp, "my_tmpl"), instanceRoot, configPath) + require.NoError(t, err) + + // assert template files have the correct permission bits set + assertFilePerm(t, filepath.Join(instanceRoot, "Amsterdam"), 0600) + assertFilePerm(t, filepath.Join(instanceRoot, "Hague"), 0400) + assertFilePerm(t, filepath.Join(instanceRoot, "foo"), 0500) + assertFilePerm(t, filepath.Join(instanceRoot, "bar"), 0755) +} From 0b62f6d25815a51b6fe7abe4d6a3d746f1d4c6a2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 23 May 2023 18:44:09 +0200 Subject: [PATCH 27/60] removed versoin from schema, this will be a separate config --- libs/template/materialize_test.go | 102 ++++++++++++++---- libs/template/schema.go | 30 +++--- libs/template/schema_test.go | 53 ++++----- .../testdata/skip_dir/template/{{.c}}/abc | 4 + .../testdata/skip_dir/template/{{.c}}/foo | 4 - libs/template/validators.go | 10 +- 6 files changed, 128 insertions(+), 75 deletions(-) create mode 100644 libs/template/testdata/skip_dir/template/{{.c}}/abc delete mode 100644 libs/template/testdata/skip_dir/template/{{.c}}/foo diff --git a/libs/template/materialize_test.go b/libs/template/materialize_test.go index 07fac5184ac..e00d535beaa 100644 --- a/libs/template/materialize_test.go +++ b/libs/template/materialize_test.go @@ -4,6 +4,7 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -24,49 +25,52 @@ func setupConfig(t *testing.T, config string) string { func assertFilePerm(t *testing.T, path string, perm fs.FileMode) { stat, err := os.Stat(path) require.NoError(t, err) - assert.Equal(t, stat.Mode().Perm(), perm) + assert.Equal(t, perm, stat.Mode().Perm()) } func TestMaterializeEmptyDirsAreNotGenerated(t *testing.T) { tmp := setupConfig(t, ` { - "a": "dir-with-file", - "b": "foo", - "c": "dir-with-skipped-file", - "d": "skipping" + "a": "this directory is created because it contains a file", + "b": "this variable is not used anywhere", + "c": "this directory will be skipped if d=foo", + "d": "foo" }`) err := Materialize("./testdata/skip_dir", tmp, filepath.Join(tmp, "config.json")) require.NoError(t, err) - assert.DirExists(t, filepath.Join(tmp, "dir-with-file")) - assert.FileExists(t, filepath.Join(tmp, "dir-with-file/.gitkeep")) - assert.NoDirExists(t, filepath.Join(tmp, "empty-dir")) - assert.NoDirExists(t, filepath.Join(tmp, "dir-with-skipped-file")) + assert.DirExists(t, filepath.Join(tmp, "this directory is created because it contains a file")) + assert.FileExists(t, filepath.Join(tmp, "this directory is created because it contains a file/.gitkeep")) + assert.NoDirExists(t, filepath.Join(tmp, "this directory will be skipped if d=foo")) tmp2 := setupConfig(t, ` { - "a": "dir-with-file", - "b": "foo", - "c": "dir-not-skipped-this-time", - "d": "not-skipping" + "a": "this directory is created because it contains a file", + "b": "this variable is not used anywhere", + "c": "this directory will be skipped if d=foo", + "d": "bar" }`) err = Materialize("./testdata/skip_dir", tmp2, filepath.Join(tmp2, "config.json")) require.NoError(t, err) - assert.DirExists(t, filepath.Join(tmp2, "dir-with-file")) - assert.FileExists(t, filepath.Join(tmp2, "dir-with-file/.gitkeep")) - assert.DirExists(t, filepath.Join(tmp2, "dir-not-skipped-this-time")) - assert.FileExists(t, filepath.Join(tmp2, "dir-not-skipped-this-time/foo")) + assert.DirExists(t, filepath.Join(tmp2, "this directory is created because it contains a file")) + assert.FileExists(t, filepath.Join(tmp2, "this directory is created because it contains a file/.gitkeep")) + assert.DirExists(t, filepath.Join(tmp2, "this directory will be skipped if d=foo")) + assert.FileExists(t, filepath.Join(tmp2, "this directory will be skipped if d=foo/abc")) } -func TestMaterializedTemplatesHaveIdenticalFilePermissionsAsTemplate(t *testing.T) { - // create template +func TestMaterializeFilePermissionsAreCopiedForUnix(t *testing.T) { + if runtime.GOOS == "windows" { + t.SkipNow() + } + tmp := t.TempDir() + + // create template schema in temp directory err := os.Mkdir(filepath.Join(tmp, "my_tmpl"), 0777) require.NoError(t, err) err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "schema.json"), []byte(` { - "version": 0, "properties": { "a": { "type": "string" @@ -122,3 +126,61 @@ func TestMaterializedTemplatesHaveIdenticalFilePermissionsAsTemplate(t *testing. assertFilePerm(t, filepath.Join(instanceRoot, "foo"), 0500) assertFilePerm(t, filepath.Join(instanceRoot, "bar"), 0755) } + +func TestMaterializeFilePermissionsAreCopiedForWindows(t *testing.T) { + if runtime.GOOS != "windows" { + t.SkipNow() + } + + tmp := t.TempDir() + + // create template in temp directory + err := os.Mkdir(filepath.Join(tmp, "my_tmpl"), 0777) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "schema.json"), []byte(` + { + "properties": { + "a": { + "type": "string" + }, + "b": { + "type": "string" + } + } + }`), 0644) + require.NoError(t, err) + + // A normal file with the executable bit not flipped + err = os.Mkdir(filepath.Join(tmp, "my_tmpl", "template"), 0777) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "{{.a}}"), []byte("abc"), 0666) + require.NoError(t, err) + + // A read only file + err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "{{.b}}"), []byte("def"), 0444) + require.NoError(t, err) + + // create config.json file + err = os.Mkdir(filepath.Join(tmp, "config"), 0777) + require.NoError(t, err) + configPath := filepath.Join(tmp, "config", "config.json") + err = os.WriteFile(configPath, []byte(` + { + "a": "Amsterdam", + "b": "Hague" + }`), 0644) + require.NoError(t, err) + + // create directory to initialize the template in + instanceRoot := filepath.Join(tmp, "instance") + err = os.Mkdir(instanceRoot, 0777) + require.NoError(t, err) + + // materialize the template + err = Materialize(filepath.Join(tmp, "my_tmpl"), instanceRoot, configPath) + require.NoError(t, err) + + // assert template files have the correct permission bits set + assertFilePerm(t, filepath.Join(instanceRoot, "Amsterdam"), 0600) + assertFilePerm(t, filepath.Join(instanceRoot, "Hague"), 0400) +} diff --git a/libs/template/schema.go b/libs/template/schema.go index 14bacf0ee40..4edcc0d4b32 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -9,27 +9,25 @@ import ( const LatestSchemaVersion = 0 +// This is a JSON Schema compliant struct that we use to do validation checks on +// the provided configuration type Schema struct { - // A version for the template schema - Version int `json:"version"` - // A list of properties that can be used in the config - Properties map[string]FieldInfo `json:"properties"` + Properties map[string]Property `json:"properties"` } -type FieldType string +type PropertyType string const ( - FieldTypeString = FieldType("string") - FieldTypeInt = FieldType("integer") - FieldTypeFloat = FieldType("float") - FieldTypeBoolean = FieldType("boolean") + PropertyTypeString = PropertyType("string") + PropertyTypeInt = PropertyType("integer") + PropertyTypeNumber = PropertyType("number") + PropertyTypeBoolean = PropertyType("boolean") ) -type FieldInfo struct { - Type FieldType `json:"type"` - Description string `json:"description"` - Validation string `json:"validation"` +type Property struct { + Type PropertyType `json:"type"` + Description string `json:"description"` } // function to check whether a float value represents an integer @@ -38,7 +36,7 @@ func isIntegerValue(v float64) bool { } // cast value to integer for config values that are floats but are supposed to be -// integeres according to the schema +// integers according to the schema // // Needed because the default json unmarshaller for maps converts all numbers to floats func castFloatToInt(config map[string]any, schema *Schema) error { @@ -50,7 +48,7 @@ func castFloatToInt(config map[string]any, schema *Schema) error { // skip non integer fields fieldInfo := schema.Properties[k] - if fieldInfo.Type != FieldTypeInt { + if fieldInfo.Type != PropertyTypeInt { continue } @@ -74,7 +72,7 @@ func castFloatToInt(config map[string]any, schema *Schema) error { return nil } -func validateType(v any, fieldType FieldType) error { +func validateType(v any, fieldType PropertyType) error { validateFunc, ok := validators[fieldType] if !ok { return nil diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go index 41afc0bba71..fd9395bf54f 100644 --- a/libs/template/schema_test.go +++ b/libs/template/schema_test.go @@ -21,13 +21,12 @@ func TestTemplateSchematIsInterger(t *testing.T) { func TestTemplateSchemaCastFloatToInt(t *testing.T) { // define schema for config schemaJson := `{ - "version": 0, "properties": { "int_val": { "type": "integer" }, "float_val": { - "type": "float" + "type": "number" }, "bool_val": { "type": "boolean" @@ -73,7 +72,6 @@ func TestTemplateSchemaCastFloatToInt(t *testing.T) { func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { // define schema for config schemaJson := `{ - "version": 0, "properties": { "foo": { "type": "integer" @@ -99,7 +97,6 @@ func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { // define schema for config schemaJson := `{ - "version": 0, "properties": { "foo": { "type": "integer" @@ -124,70 +121,69 @@ func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { func TestTemplateSchemaValidateType(t *testing.T) { // assert validation passing - err := validateType(int(0), FieldTypeInt) + err := validateType(int(0), PropertyTypeInt) assert.NoError(t, err) - err = validateType(int32(1), FieldTypeInt) + err = validateType(int32(1), PropertyTypeInt) assert.NoError(t, err) - err = validateType(int64(1), FieldTypeInt) + err = validateType(int64(1), PropertyTypeInt) assert.NoError(t, err) - err = validateType(float32(1.1), FieldTypeFloat) + err = validateType(float32(1.1), PropertyTypeNumber) assert.NoError(t, err) - err = validateType(float64(1.2), FieldTypeFloat) + err = validateType(float64(1.2), PropertyTypeNumber) assert.NoError(t, err) - err = validateType(false, FieldTypeBoolean) + err = validateType(false, PropertyTypeBoolean) assert.NoError(t, err) - err = validateType("abc", FieldTypeString) + err = validateType("abc", PropertyTypeString) assert.NoError(t, err) // assert validation failing for integers - err = validateType(float64(1.2), FieldTypeInt) + err = validateType(float64(1.2), PropertyTypeInt) assert.ErrorContains(t, err, "expected type integer, but value is 1.2") - err = validateType(true, FieldTypeInt) + err = validateType(true, PropertyTypeInt) assert.ErrorContains(t, err, "expected type integer, but value is true") - err = validateType("abc", FieldTypeInt) + err = validateType("abc", PropertyTypeInt) assert.ErrorContains(t, err, "expected type integer, but value is \"abc\"") // assert validation failing for floats - err = validateType(int(1), FieldTypeFloat) + err = validateType(int(1), PropertyTypeNumber) assert.ErrorContains(t, err, "expected type float, but value is 1") - err = validateType(true, FieldTypeFloat) + err = validateType(true, PropertyTypeNumber) assert.ErrorContains(t, err, "expected type float, but value is true") - err = validateType("abc", FieldTypeFloat) + err = validateType("abc", PropertyTypeNumber) assert.ErrorContains(t, err, "expected type float, but value is \"abc\"") // assert validation failing for boolean - err = validateType(int(1), FieldTypeBoolean) + err = validateType(int(1), PropertyTypeBoolean) assert.ErrorContains(t, err, "expected type boolean, but value is 1") - err = validateType(float64(1), FieldTypeBoolean) + err = validateType(float64(1), PropertyTypeBoolean) assert.ErrorContains(t, err, "expected type boolean, but value is 1") - err = validateType("abc", FieldTypeBoolean) + err = validateType("abc", PropertyTypeBoolean) assert.ErrorContains(t, err, "expected type boolean, but value is \"abc\"") // assert validation failing for string - err = validateType(int(1), FieldTypeString) + err = validateType(int(1), PropertyTypeString) assert.ErrorContains(t, err, "expected type string, but value is 1") - err = validateType(float64(1), FieldTypeString) + err = validateType(float64(1), PropertyTypeString) assert.ErrorContains(t, err, "expected type string, but value is 1") - err = validateType(false, FieldTypeString) + err = validateType(false, PropertyTypeString) assert.ErrorContains(t, err, "expected type string, but value is false") } func TestTemplateSchemaValidateConfig(t *testing.T) { // define schema for config schemaJson := `{ - "version": 0, "properties": { "int_val": { "type": "integer" }, "float_val": { - "type": "float" + "type": "number" }, "bool_val": { "type": "boolean" @@ -216,13 +212,12 @@ func TestTemplateSchemaValidateConfig(t *testing.T) { func TestTemplateSchemaValidateConfigFailsForUnknownField(t *testing.T) { // define schema for config schemaJson := `{ - "version": 0, "properties": { "int_val": { "type": "integer" }, "float_val": { - "type": "float" + "type": "number" }, "bool_val": { "type": "boolean" @@ -251,13 +246,12 @@ func TestTemplateSchemaValidateConfigFailsForUnknownField(t *testing.T) { func TestTemplateSchemaValidateConfigFailsForWhenIncorrectTypes(t *testing.T) { // define schema for config schemaJson := `{ - "version": 0, "properties": { "int_val": { "type": "integer" }, "float_val": { - "type": "float" + "type": "number" }, "bool_val": { "type": "boolean" @@ -286,7 +280,6 @@ func TestTemplateSchemaValidateConfigFailsForWhenIncorrectTypes(t *testing.T) { func TestTemplateSchemaValidateConfigFailsForWhenMissingInputParams(t *testing.T) { // define schema for config schemaJson := `{ - "version": 0, "properties": { "int_val": { "type": "integer" diff --git a/libs/template/testdata/skip_dir/template/{{.c}}/abc b/libs/template/testdata/skip_dir/template/{{.c}}/abc new file mode 100644 index 00000000000..6af8c5850c9 --- /dev/null +++ b/libs/template/testdata/skip_dir/template/{{.c}}/abc @@ -0,0 +1,4 @@ +{{if eq .d "foo"}} +{{skipThisFile}} +{{end}} +Hello, World diff --git a/libs/template/testdata/skip_dir/template/{{.c}}/foo b/libs/template/testdata/skip_dir/template/{{.c}}/foo deleted file mode 100644 index 9925ff1dce6..00000000000 --- a/libs/template/testdata/skip_dir/template/{{.c}}/foo +++ /dev/null @@ -1,4 +0,0 @@ -{{if eq .d "skipping"}} -{{skipThisFile}} -{{end}} -Hello! diff --git a/libs/template/validators.go b/libs/template/validators.go index 4442173f00c..c67c6eddd2b 100644 --- a/libs/template/validators.go +++ b/libs/template/validators.go @@ -39,9 +39,9 @@ func validateInteger(v any) error { return nil } -var validators map[FieldType]Validator = map[FieldType]Validator{ - FieldTypeString: validateString, - FieldTypeBoolean: validateBoolean, - FieldTypeInt: validateInteger, - FieldTypeFloat: validateFloat, +var validators map[PropertyType]Validator = map[PropertyType]Validator{ + PropertyTypeString: validateString, + PropertyTypeBoolean: validateBoolean, + PropertyTypeInt: validateInteger, + PropertyTypeNumber: validateFloat, } From 8b8320554c3ec98c615261da05eadd69e23e72fd Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 23 May 2023 18:49:15 +0200 Subject: [PATCH 28/60] remove default for config file --- cmd/bundle/init.go | 7 +------ libs/template/materialize.go | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index 236554784fc..40f9829b112 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -1,8 +1,6 @@ package bundle import ( - "path/filepath" - "github.com/databricks/cli/libs/template" "github.com/spf13/cobra" ) @@ -13,10 +11,6 @@ var initCmd = &cobra.Command{ Long: `Initialize template`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - // initialize default value for config file path - if configFile == "" { - configFile = filepath.Join(targetDir, template.ConfigFileName) - } return template.Materialize(args[0], targetDir, configFile) }, } @@ -27,5 +21,6 @@ var configFile string func init() { initCmd.Flags().StringVar(&targetDir, "target-dir", ".", "path to directory template will be initialized in") initCmd.Flags().StringVar(&configFile, "config-file", "", "path to config to use for template initialization") + initCmd.MarkFlagRequired("config-file") AddCommand(initCmd) } diff --git a/libs/template/materialize.go b/libs/template/materialize.go index b50b7a4f200..7bfe9cb03ca 100644 --- a/libs/template/materialize.go +++ b/libs/template/materialize.go @@ -4,7 +4,6 @@ import ( "path/filepath" ) -const ConfigFileName = "config.json" const schemaFileName = "schema.json" const templateDirName = "template" From 2cafe2e766ca8c5aa9cf949d16c9395cad179c97 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 23 May 2023 18:51:08 +0200 Subject: [PATCH 29/60] fix testS --- libs/template/materialize_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/template/materialize_test.go b/libs/template/materialize_test.go index e00d535beaa..6596d58d3c2 100644 --- a/libs/template/materialize_test.go +++ b/libs/template/materialize_test.go @@ -181,6 +181,6 @@ func TestMaterializeFilePermissionsAreCopiedForWindows(t *testing.T) { require.NoError(t, err) // assert template files have the correct permission bits set - assertFilePerm(t, filepath.Join(instanceRoot, "Amsterdam"), 0600) - assertFilePerm(t, filepath.Join(instanceRoot, "Hague"), 0400) + assertFilePerm(t, filepath.Join(instanceRoot, "Amsterdam"), 0666) + assertFilePerm(t, filepath.Join(instanceRoot, "Hague"), 0444) } From 3626588511f6d1bb00a53465512f90b521af93b0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 23 May 2023 18:52:27 +0200 Subject: [PATCH 30/60] remove version in testdata --- internal/testdata/init/templateDefinition/schema.json | 1 - libs/template/testdata/skip_dir/schema.json | 1 - 2 files changed, 2 deletions(-) diff --git a/internal/testdata/init/templateDefinition/schema.json b/internal/testdata/init/templateDefinition/schema.json index 7a682f6da1f..5e8b40e83e5 100644 --- a/internal/testdata/init/templateDefinition/schema.json +++ b/internal/testdata/init/templateDefinition/schema.json @@ -1,5 +1,4 @@ { - "version": 0, "properties": { "project_name": { "description": "Name of the project", diff --git a/libs/template/testdata/skip_dir/schema.json b/libs/template/testdata/skip_dir/schema.json index 945b7df8447..e3d5253b26a 100644 --- a/libs/template/testdata/skip_dir/schema.json +++ b/libs/template/testdata/skip_dir/schema.json @@ -1,5 +1,4 @@ { - "version": 0, "properties": { "a": { "type": "string" From 95d4da6dc095fafdc9ff0b62ac822f26610a8e9c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 23 May 2023 19:02:09 +0200 Subject: [PATCH 31/60] fix and added execute template tests --- internal/init_test.go | 2 +- libs/template/execute.go | 1 - libs/template/execute_test.go | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 libs/template/execute_test.go diff --git a/internal/init_test.go b/internal/init_test.go index 98c378b5918..58c48a03cdd 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -36,7 +36,7 @@ func TestTemplateInitializationForDevConfig(t *testing.T) { // materialize the template cmd := root.RootCmd - cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", tmp}) + cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", tmp, "--config-file", filepath.Join(tmp, "config.json")}) err = cmd.Execute() require.NoError(t, err) diff --git a/libs/template/execute.go b/libs/template/execute.go index 6f63a7acf92..82f80c5284a 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -11,7 +11,6 @@ import ( // Executes the template by applying config on it. Returns the materialized config // as a string -// TODO: test this function func executeTemplate(config map[string]any, templateDefinition string) (string, error) { // configure template with helper functions tmpl, err := template.New("").Funcs(HelperFuncs).Parse(templateDefinition) diff --git a/libs/template/execute_test.go b/libs/template/execute_test.go new file mode 100644 index 00000000000..906c886ac54 --- /dev/null +++ b/libs/template/execute_test.go @@ -0,0 +1,34 @@ +package template + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExecuteTemplate(t *testing.T) { + templateText := + `"{{.count}} items are made of {{.Material}}". +{{if eq .Animal "sheep" }} +Sheep wool is the best! +{{else}} +{{.Animal}} wool is not too bad... +{{end}} +` + statement, err := executeTemplate(map[string]any{ + "Material": "wool", + "count": 1, + "Animal": "sheep", + }, templateText) + require.NoError(t, err) + assert.Equal(t, "\"1 items are made of wool\".\n\nSheep wool is the best!\n\n", statement) + + statement, err = executeTemplate(map[string]any{ + "Material": "wool", + "count": 1, + "Animal": "cat", + }, templateText) + require.NoError(t, err) + assert.Equal(t, "\"1 items are made of wool\".\n\ncat wool is not too bad...\n\n", statement) +} From 4fa8c9c06dd1438df6e975e4461c8ca8921884dc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 23 May 2023 19:15:20 +0200 Subject: [PATCH 32/60] added test for gen file --- libs/template/execute.go | 1 - libs/template/execute_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/libs/template/execute.go b/libs/template/execute.go index 82f80c5284a..6096ccc094d 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -27,7 +27,6 @@ func executeTemplate(config map[string]any, templateDefinition string) (string, return result.String(), nil } -// TODO: test this function func generateFile(config map[string]any, pathTemplate, contentTemplate string, perm fs.FileMode) error { // compute file content fileContent, err := executeTemplate(config, contentTemplate) diff --git a/libs/template/execute_test.go b/libs/template/execute_test.go index 906c886ac54..55ac03a2aae 100644 --- a/libs/template/execute_test.go +++ b/libs/template/execute_test.go @@ -1,6 +1,8 @@ package template import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -32,3 +34,35 @@ Sheep wool is the best! require.NoError(t, err) assert.Equal(t, "\"1 items are made of wool\".\n\ncat wool is not too bad...\n\n", statement) } + +func TestGenerateFile(t *testing.T) { + tmp := t.TempDir() + + pathTemplate := filepath.Join(tmp, "{{.Animal}}", "{{.Material}}", "foo", "{{.count}}.txt") + contentTemplate := `"{{.count}} items are made of {{.Material}}". + {{if eq .Animal "sheep" }} + Sheep wool is the best! + {{else}} + {{.Animal}} wool is not too bad... + {{end}} + ` + err := generateFile(map[string]any{ + "Material": "wool", + "count": 1, + "Animal": "cat", + }, pathTemplate, contentTemplate, 0444) + require.NoError(t, err) + + // assert file exists + assert.FileExists(t, filepath.Join(tmp, "cat", "wool", "foo", "1.txt")) + + // assert file content is created correctly + b, err := os.ReadFile(filepath.Join(tmp, "cat", "wool", "foo", "1.txt")) + require.NoError(t, err) + assert.Equal(t, "\"1 items are made of wool\".\n\t\n\tcat wool is not too bad...\n\t\n\t", string(b)) + + // assert file permissions are correctly assigned + stat, err := os.Stat(filepath.Join(tmp, "cat", "wool", "foo", "1.txt")) + require.NoError(t, err) + assert.Equal(t, uint(0444), uint(stat.Mode().Perm())) +} From 2c02fdf2fbd2c79785200c513ac46d1df21c3823 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 23 May 2023 19:25:51 +0200 Subject: [PATCH 33/60] fixed validators for number --- libs/template/schema_test.go | 7 ++----- libs/template/validators.go | 6 +++--- libs/template/validators_test.go | 17 +++++++++-------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go index fd9395bf54f..f3b69a0e677 100644 --- a/libs/template/schema_test.go +++ b/libs/template/schema_test.go @@ -123,18 +123,17 @@ func TestTemplateSchemaValidateType(t *testing.T) { // assert validation passing err := validateType(int(0), PropertyTypeInt) assert.NoError(t, err) - err = validateType(int32(1), PropertyTypeInt) assert.NoError(t, err) - err = validateType(int64(1), PropertyTypeInt) assert.NoError(t, err) err = validateType(float32(1.1), PropertyTypeNumber) assert.NoError(t, err) - err = validateType(float64(1.2), PropertyTypeNumber) assert.NoError(t, err) + err = validateType(int(1), PropertyTypeNumber) + assert.NoError(t, err) err = validateType(false, PropertyTypeBoolean) assert.NoError(t, err) @@ -151,8 +150,6 @@ func TestTemplateSchemaValidateType(t *testing.T) { assert.ErrorContains(t, err, "expected type integer, but value is \"abc\"") // assert validation failing for floats - err = validateType(int(1), PropertyTypeNumber) - assert.ErrorContains(t, err, "expected type float, but value is 1") err = validateType(true, PropertyTypeNumber) assert.ErrorContains(t, err, "expected type float, but value is true") err = validateType("abc", PropertyTypeNumber) diff --git a/libs/template/validators.go b/libs/template/validators.go index c67c6eddd2b..96995560355 100644 --- a/libs/template/validators.go +++ b/libs/template/validators.go @@ -23,8 +23,8 @@ func validateBoolean(v any) error { return nil } -func validateFloat(v any) error { - if !slices.Contains([]reflect.Kind{reflect.Float32, reflect.Float64}, +func validateNumber(v any) error { + if !slices.Contains([]reflect.Kind{reflect.Float32, reflect.Float64, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64}, reflect.TypeOf(v).Kind()) { return fmt.Errorf("expected type float, but value is %#v", v) } @@ -43,5 +43,5 @@ var validators map[PropertyType]Validator = map[PropertyType]Validator{ PropertyTypeString: validateString, PropertyTypeBoolean: validateBoolean, PropertyTypeInt: validateInteger, - PropertyTypeNumber: validateFloat, + PropertyTypeNumber: validateNumber, } diff --git a/libs/template/validators_test.go b/libs/template/validators_test.go index 5621ef9285d..3573592ffe5 100644 --- a/libs/template/validators_test.go +++ b/libs/template/validators_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestValidatorString(t *testing.T) { @@ -35,22 +36,22 @@ func TestValidatorBoolean(t *testing.T) { } func TestValidatorFloat(t *testing.T) { - err := validateFloat(true) + err := validateNumber(true) assert.ErrorContains(t, err, "expected type float, but value is true") - err = validateFloat(int32(1)) - assert.ErrorContains(t, err, "expected type float, but value is 1") + err = validateNumber(int32(1)) + require.NoError(t, err) - err = validateFloat(int64(1)) - assert.ErrorContains(t, err, "expected type float, but value is 1") + err = validateNumber(int64(1)) + require.NoError(t, err) - err = validateFloat(float32(1)) + err = validateNumber(float32(1)) assert.NoError(t, err) - err = validateFloat(float64(1)) + err = validateNumber(float64(1)) assert.NoError(t, err) - err = validateFloat("abc") + err = validateNumber("abc") assert.ErrorContains(t, err, "expected type float, but value is \"abc\"") } From c02b28069142a29bfdeea956b5b8f873d3fbfcc0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 23 May 2023 19:31:02 +0200 Subject: [PATCH 34/60] nit --- libs/template/validators_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/template/validators_test.go b/libs/template/validators_test.go index 3573592ffe5..f0cbf8a14e1 100644 --- a/libs/template/validators_test.go +++ b/libs/template/validators_test.go @@ -35,7 +35,7 @@ func TestValidatorBoolean(t *testing.T) { assert.ErrorContains(t, err, "expected type boolean, but value is \"false\"") } -func TestValidatorFloat(t *testing.T) { +func TestValidatorNumber(t *testing.T) { err := validateNumber(true) assert.ErrorContains(t, err, "expected type float, but value is true") From 63df93c1a1ab2aec3c0ecc464c1a12e557044c50 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 21 Jun 2023 18:40:15 +0200 Subject: [PATCH 35/60] Make instance path a required arg --- cmd/bundle/init.go | 8 +++----- internal/init_test.go | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index 40f9829b112..a1540369d1b 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -6,20 +6,18 @@ import ( ) var initCmd = &cobra.Command{ - Use: "init", + Use: "init TEMPLATE_PATH INSTANCE_PATH", Short: "Initialize Template", Long: `Initialize template`, - Args: cobra.ExactArgs(1), + Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - return template.Materialize(args[0], targetDir, configFile) + return template.Materialize(args[0], args[1], configFile) }, } -var targetDir string var configFile string func init() { - initCmd.Flags().StringVar(&targetDir, "target-dir", ".", "path to directory template will be initialized in") initCmd.Flags().StringVar(&configFile, "config-file", "", "path to config to use for template initialization") initCmd.MarkFlagRequired("config-file") AddCommand(initCmd) diff --git a/internal/init_test.go b/internal/init_test.go index 58c48a03cdd..6d37fa141cc 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -36,7 +36,7 @@ func TestTemplateInitializationForDevConfig(t *testing.T) { // materialize the template cmd := root.RootCmd - cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", tmp, "--config-file", filepath.Join(tmp, "config.json")}) + cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), tmp, "--config-file", filepath.Join(tmp, "config.json")}) err = cmd.Execute() require.NoError(t, err) @@ -78,7 +78,7 @@ func TestTemplateInitializationForProdConfig(t *testing.T) { cmd := root.RootCmd childCommands := cmd.Commands() fmt.Println(childCommands) - cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--target-dir", instanceDir, "--config-file", filepath.Join(configDir, "my_config.json")}) + cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), instanceDir, "--config-file", filepath.Join(configDir, "my_config.json")}) err = cmd.Execute() require.NoError(t, err) From ca9de54b8649f375ab8ec75129f503ad61284ec7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 21 Jun 2023 18:43:05 +0200 Subject: [PATCH 36/60] added todo --- libs/template/execute.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/template/execute.go b/libs/template/execute.go index 6096ccc094d..4801d8da553 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -54,6 +54,7 @@ func generateFile(config map[string]any, pathTemplate, contentTemplate string, p return os.WriteFile(path, []byte(fileContent), perm) } +// TODO: use local filer client for this function. https://github.com/databricks/cli/issues/511 func walkFileTree(config map[string]any, templateRoot, instanceRoot string) error { return filepath.WalkDir(templateRoot, func(path string, d fs.DirEntry, err error) error { if err != nil { From b66b69bc2712a0e60d2e49ef795aced140be762c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Jul 2023 15:50:07 +0200 Subject: [PATCH 37/60] - --- cmd/bundle/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index a1540369d1b..ccde90dd47c 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -18,7 +18,7 @@ var initCmd = &cobra.Command{ var configFile string func init() { - initCmd.Flags().StringVar(&configFile, "config-file", "", "path to config to use for template initialization") + initCmd.Flags().StringVar(&configFile, "config-file", "", "input parameters for template initialization") initCmd.MarkFlagRequired("config-file") AddCommand(initCmd) } From 3d06fb67e8e0d0a65a67b567b8b42fa45394e69c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Jul 2023 15:59:25 +0200 Subject: [PATCH 38/60] fix acc test prefix --- internal/init_test.go | 4 ++-- libs/template/helpers.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/init_test.go b/internal/init_test.go index 6d37fa141cc..6caeb167513 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -18,7 +18,7 @@ func assertFileContains(t *testing.T, path string, substr string) { assert.Contains(t, string(b), substr) } -func TestTemplateInitializationForDevConfig(t *testing.T) { +func TestAccTemplateInitializationForDevConfig(t *testing.T) { // create target directory with the input config tmp := t.TempDir() f, err := os.Create(filepath.Join(tmp, "config.json")) @@ -48,7 +48,7 @@ func TestTemplateInitializationForDevConfig(t *testing.T) { assertFileContains(t, filepath.Join(tmp, "development_project", ".github"), "This is a development project") } -func TestTemplateInitializationForProdConfig(t *testing.T) { +func TestAccTemplateInitializationForProdConfig(t *testing.T) { // create target directory with the input config tmp := t.TempDir() diff --git a/libs/template/helpers.go b/libs/template/helpers.go index 8e072bba135..27f829ee2f7 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -8,7 +8,8 @@ import ( var errSkipThisFile = errors.New("skip generating this file") var HelperFuncs = template.FuncMap{ - "skipThisFile": func() error { - panic(errSkipThisFile) + "skipThisFile": func() (any, error) { + return nil, errSkipThisFile + // panic(errSkipThisFile) }, } From 4a53e0ce5ab1cb6521692102c811b80baa081e52 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Jul 2023 16:01:47 +0200 Subject: [PATCH 39/60] remove panic from skipThisFile --- libs/template/helpers.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/template/helpers.go b/libs/template/helpers.go index 27f829ee2f7..ffca0828f7d 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -8,8 +8,9 @@ import ( var errSkipThisFile = errors.New("skip generating this file") var HelperFuncs = template.FuncMap{ + // Text template execution returns the error only if it's the second return + // value from a function: https://pkg.go.dev/text/template#hdr-Pipelines "skipThisFile": func() (any, error) { return nil, errSkipThisFile - // panic(errSkipThisFile) }, } From 95391f792c5f49914d7a7bb429315c46ea2a149d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Jul 2023 16:04:32 +0200 Subject: [PATCH 40/60] rename schema.json to databricks_template_schema.json --- .../{schema.json => databricks_template_schema.json} | 0 libs/template/materialize.go | 2 +- .../skip_dir/{schema.json => databricks_template_schema.json} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename internal/testdata/init/templateDefinition/{schema.json => databricks_template_schema.json} (100%) rename libs/template/testdata/skip_dir/{schema.json => databricks_template_schema.json} (100%) diff --git a/internal/testdata/init/templateDefinition/schema.json b/internal/testdata/init/templateDefinition/databricks_template_schema.json similarity index 100% rename from internal/testdata/init/templateDefinition/schema.json rename to internal/testdata/init/templateDefinition/databricks_template_schema.json diff --git a/libs/template/materialize.go b/libs/template/materialize.go index 7bfe9cb03ca..ecb75749cf1 100644 --- a/libs/template/materialize.go +++ b/libs/template/materialize.go @@ -4,7 +4,7 @@ import ( "path/filepath" ) -const schemaFileName = "schema.json" +const schemaFileName = "databricks_template_schema.json" const templateDirName = "template" func Materialize(templateRoot, instanceRoot, configPath string) error { diff --git a/libs/template/testdata/skip_dir/schema.json b/libs/template/testdata/skip_dir/databricks_template_schema.json similarity index 100% rename from libs/template/testdata/skip_dir/schema.json rename to libs/template/testdata/skip_dir/databricks_template_schema.json From 2486a96cd78e7033281b25ec92143e76a18870c6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Jul 2023 10:54:31 +0200 Subject: [PATCH 41/60] add support for library --- .../init/templateDefinition/library/foo | 6 ++ .../template/{{.project_name}}/azure_file | 3 + libs/template/execute.go | 51 +++++++++++----- libs/template/execute_test.go | 58 +++++++++++++------ libs/template/helpers.go | 4 ++ libs/template/materialize.go | 10 +++- 6 files changed, 100 insertions(+), 32 deletions(-) create mode 100644 internal/testdata/init/templateDefinition/library/foo diff --git a/internal/testdata/init/templateDefinition/library/foo b/internal/testdata/init/templateDefinition/library/foo new file mode 100644 index 00000000000..d783c898245 --- /dev/null +++ b/internal/testdata/init/templateDefinition/library/foo @@ -0,0 +1,6 @@ +{{define "email"}}shreyas.goenka@databricks.com{{end}} +{{define "get_host"}} +{{ with urlParse . }}{{- + print .Scheme `://` .Host +-}}{{end}} +{{end}} diff --git a/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file b/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file index e1be17cb50a..2fe97dbba12 100644 --- a/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file +++ b/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file @@ -2,3 +2,6 @@ This file should only be generated for Azure {{if ne .cloud_type "Azure"}} {{skipThisFile}} {{end}} + +{{ template "email" }} +{{ template "get_host" "https://adb-xxxx.xx.azuredatabricks.net/sql/queries" }} diff --git a/libs/template/execute.go b/libs/template/execute.go index 4801d8da553..8b234111194 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -2,6 +2,7 @@ package template import ( "errors" + "fmt" "io/fs" "os" "path/filepath" @@ -9,39 +10,63 @@ import ( "text/template" ) -// Executes the template by applying config on it. Returns the materialized config +type renderer struct { + config map[string]any + + baseTemplate *template.Template +} + +func newRenderer(config map[string]any, libraryRoot string) (*renderer, error) { + tmpl, err := template.New("").Funcs(HelperFuncs).ParseGlob(filepath.Join(libraryRoot, "*")) + if err != nil { + return nil, err + } + + return &renderer{ + config: config, + baseTemplate: tmpl, + }, nil +} + +// Executes the template by applying config on it. Returns the materialized template // as a string -func executeTemplate(config map[string]any, templateDefinition string) (string, error) { - // configure template with helper functions - tmpl, err := template.New("").Funcs(HelperFuncs).Parse(templateDefinition) +func (r *renderer) executeTemplate(templateDefinition string) (string, error) { + // Create copy of base template so as to not overwrite it + tmpl, err := r.baseTemplate.Clone() if err != nil { return "", err } - // execute template + // Parse the template text + tmpl, err = tmpl.Parse(templateDefinition) + if err != nil { + return "", err + } + + // Execute template and get result result := strings.Builder{} - err = tmpl.Execute(&result, config) + err = tmpl.Execute(&result, r.config) if err != nil { return "", err } return result.String(), nil } -func generateFile(config map[string]any, pathTemplate, contentTemplate string, perm fs.FileMode) error { +func (r *renderer) generateFile(pathTemplate, contentTemplate string, perm fs.FileMode) error { // compute file content - fileContent, err := executeTemplate(config, contentTemplate) + fileContent, err := r.executeTemplate(contentTemplate) if errors.Is(err, errSkipThisFile) { // skip this file return nil } if err != nil { - return err + return fmt.Errorf("failed to compute file content for %s. %w", pathTemplate, err) } // compute the path for this file - path, err := executeTemplate(config, pathTemplate) + path, err := r.executeTemplate(pathTemplate) if err != nil { - return err + return fmt.Errorf("failed to compute path for %s. %w", pathTemplate, err) } // create any intermediate directories required. Directories are lazily generated // only when they are required for a file. @@ -55,7 +80,7 @@ func generateFile(config map[string]any, pathTemplate, contentTemplate string, p } // TODO: use local filer client for this function. https://github.com/databricks/cli/issues/511 -func walkFileTree(config map[string]any, templateRoot, instanceRoot string) error { +func walkFileTree(r *renderer, templateRoot, instanceRoot string) error { return filepath.WalkDir(templateRoot, func(path string, d fs.DirEntry, err error) error { if err != nil { return err @@ -87,6 +112,6 @@ func walkFileTree(config map[string]any, templateRoot, instanceRoot string) erro return err } - return generateFile(config, filepath.Join(instanceRoot, relPathTemplate), contentTemplate, info.Mode().Perm()) + return r.generateFile(filepath.Join(instanceRoot, relPathTemplate), contentTemplate, info.Mode().Perm()) }) } diff --git a/libs/template/execute_test.go b/libs/template/execute_test.go index 55ac03a2aae..a9431536c96 100644 --- a/libs/template/execute_test.go +++ b/libs/template/execute_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "testing" + "text/template" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,22 +18,40 @@ Sheep wool is the best! {{else}} {{.Animal}} wool is not too bad... {{end}} +My email is {{template "email"}} ` - statement, err := executeTemplate(map[string]any{ - "Material": "wool", - "count": 1, - "Animal": "sheep", - }, templateText) + + r := renderer{ + config: map[string]any{ + "Material": "wool", + "count": 1, + "Animal": "sheep", + }, + baseTemplate: template.Must(template.New("base").Parse(`{{define "email"}}shreyas.goenka@databricks.com{{end}}`)), + } + + statement, err := r.executeTemplate(templateText) require.NoError(t, err) - assert.Equal(t, "\"1 items are made of wool\".\n\nSheep wool is the best!\n\n", statement) + assert.Contains(t, statement, `"1 items are made of wool"`) + assert.NotContains(t, statement, `cat wool is not too bad.."`) + assert.Contains(t, statement, "Sheep wool is the best!") + assert.Contains(t, statement, `My email is shreyas.goenka@databricks.com`) - statement, err = executeTemplate(map[string]any{ - "Material": "wool", - "count": 1, - "Animal": "cat", - }, templateText) + r = renderer{ + config: map[string]any{ + "Material": "wool", + "count": 1, + "Animal": "cat", + }, + baseTemplate: template.Must(template.New("base").Parse(`{{define "email"}}hrithik.roshan@databricks.com{{end}}`)), + } + + statement, err = r.executeTemplate(templateText) require.NoError(t, err) - assert.Equal(t, "\"1 items are made of wool\".\n\ncat wool is not too bad...\n\n", statement) + assert.Contains(t, statement, `"1 items are made of wool"`) + assert.Contains(t, statement, `cat wool is not too bad...`) + assert.NotContains(t, statement, "Sheep wool is the best!") + assert.Contains(t, statement, `My email is hrithik.roshan@databricks.com`) } func TestGenerateFile(t *testing.T) { @@ -46,11 +65,16 @@ func TestGenerateFile(t *testing.T) { {{.Animal}} wool is not too bad... {{end}} ` - err := generateFile(map[string]any{ - "Material": "wool", - "count": 1, - "Animal": "cat", - }, pathTemplate, contentTemplate, 0444) + + r := renderer{ + config: map[string]any{ + "Material": "wool", + "count": 1, + "Animal": "cat", + }, + baseTemplate: template.New("base"), + } + err := r.generateFile(pathTemplate, contentTemplate, 0444) require.NoError(t, err) // assert file exists diff --git a/libs/template/helpers.go b/libs/template/helpers.go index ffca0828f7d..ca255457e9a 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -2,6 +2,7 @@ package template import ( "errors" + "net/url" "text/template" ) @@ -13,4 +14,7 @@ var HelperFuncs = template.FuncMap{ "skipThisFile": func() (any, error) { return nil, errSkipThisFile }, + "urlParse": func(rawUrl string) (*url.URL, error) { + return url.Parse(rawUrl) + }, } diff --git a/libs/template/materialize.go b/libs/template/materialize.go index ecb75749cf1..03eac22fc3d 100644 --- a/libs/template/materialize.go +++ b/libs/template/materialize.go @@ -6,6 +6,7 @@ import ( const schemaFileName = "databricks_template_schema.json" const templateDirName = "template" +const libraryDirName = "library" func Materialize(templateRoot, instanceRoot, configPath string) error { // read the file containing schema for template input parameters @@ -14,12 +15,17 @@ func Materialize(templateRoot, instanceRoot, configPath string) error { return err } - // read user config to initalize the template with + // read user config to initialize the template with config, err := schema.ReadConfig(configPath) if err != nil { return err } + r, err := newRenderer(config, filepath.Join(templateRoot, libraryDirName)) + if err != nil { + return err + } + // materialize the template - return walkFileTree(config, filepath.Join(templateRoot, templateDirName), instanceRoot) + return walkFileTree(r, filepath.Join(templateRoot, templateDirName), instanceRoot) } From 02d53f90e1d9542e8fd038c5c049f309654d1fc3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Jul 2023 14:36:20 +0200 Subject: [PATCH 42/60] fix tests --- libs/template/execute.go | 18 +++++++++++++++--- libs/template/materialize_test.go | 8 ++++---- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/libs/template/execute.go b/libs/template/execute.go index 8b234111194..e341a620c7e 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -11,16 +11,28 @@ import ( ) type renderer struct { - config map[string]any - + config map[string]any baseTemplate *template.Template } func newRenderer(config map[string]any, libraryRoot string) (*renderer, error) { - tmpl, err := template.New("").Funcs(HelperFuncs).ParseGlob(filepath.Join(libraryRoot, "*")) + // All user defined functions will be available inside library root + libraryGlob := filepath.Join(libraryRoot, "*") + + // Initialize new template, with helper functions loaded + tmpl := template.New("").Funcs(HelperFuncs) + + // Load files in the library to the template + matches, err := filepath.Glob(libraryGlob) if err != nil { return nil, err } + if len(matches) != 0 { + tmpl, err = tmpl.ParseGlob(libraryGlob) + if err != nil { + return nil, err + } + } return &renderer{ config: config, diff --git a/libs/template/materialize_test.go b/libs/template/materialize_test.go index 6596d58d3c2..5678c9bb136 100644 --- a/libs/template/materialize_test.go +++ b/libs/template/materialize_test.go @@ -66,10 +66,10 @@ func TestMaterializeFilePermissionsAreCopiedForUnix(t *testing.T) { tmp := t.TempDir() - // create template schema in temp directory + // setup template in temp directory err := os.Mkdir(filepath.Join(tmp, "my_tmpl"), 0777) require.NoError(t, err) - err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "schema.json"), []byte(` + err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "databricks_template_schema.json"), []byte(` { "properties": { "a": { @@ -81,10 +81,10 @@ func TestMaterializeFilePermissionsAreCopiedForUnix(t *testing.T) { } }`), 0644) require.NoError(t, err) - - // A normal file with the executable bit not flipped err = os.Mkdir(filepath.Join(tmp, "my_tmpl", "template"), 0777) require.NoError(t, err) + + // A normal file with the executable bit not flipped err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "{{.a}}"), []byte("abc"), 0600) require.NoError(t, err) From a3a01b730256bdb86cb965eb97a4e89e5700a30e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Jul 2023 14:43:44 +0200 Subject: [PATCH 43/60] cleanup test --- internal/init_test.go | 15 ++++++++++++++- .../testdata/init/templateDefinition/library/foo | 6 +----- .../template/{{.project_name}}/azure_file | 5 +---- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/internal/init_test.go b/internal/init_test.go index 6caeb167513..c497107eda6 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -18,6 +18,12 @@ func assertFileContains(t *testing.T, path string, substr string) { assert.Contains(t, string(b), substr) } +func assertLocalFileContent(t *testing.T, path string, expected string) { + b, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, string(b), expected) +} + func TestAccTemplateInitializationForDevConfig(t *testing.T) { // create target directory with the input config tmp := t.TempDir() @@ -86,6 +92,13 @@ func TestAccTemplateInitializationForProdConfig(t *testing.T) { assert.FileExists(t, filepath.Join(instanceDir, "production_project", "azure_file")) assert.FileExists(t, filepath.Join(instanceDir, "production_project", ".azure_devops")) assert.NoFileExists(t, filepath.Join(instanceDir, "production_project", "aws_file")) - assertFileContains(t, filepath.Join(instanceDir, "production_project", "azure_file"), "This file should only be generated for Azure") + + assertLocalFileContent(t, filepath.Join(instanceDir, "production_project", "azure_file"), + ` +This file should only be generated for Azure +shreyas.goenka@databricks.com +https://adb-xxxx.xx.azuredatabricks.net +`) + assertFileContains(t, filepath.Join(instanceDir, "production_project", ".azure_devops"), "This is a production project") } diff --git a/internal/testdata/init/templateDefinition/library/foo b/internal/testdata/init/templateDefinition/library/foo index d783c898245..2b917a32294 100644 --- a/internal/testdata/init/templateDefinition/library/foo +++ b/internal/testdata/init/templateDefinition/library/foo @@ -1,6 +1,2 @@ {{define "email"}}shreyas.goenka@databricks.com{{end}} -{{define "get_host"}} -{{ with urlParse . }}{{- - print .Scheme `://` .Host --}}{{end}} -{{end}} +{{define "get_host"}}{{ with urlParse . }}{{print .Scheme `://` .Host}}{{end}}{{end}} diff --git a/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file b/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file index 2fe97dbba12..b011b3832b2 100644 --- a/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file +++ b/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file @@ -1,7 +1,4 @@ +{{if ne .cloud_type "Azure"}}{{skipThisFile}}{{end}} This file should only be generated for Azure -{{if ne .cloud_type "Azure"}} -{{skipThisFile}} -{{end}} - {{ template "email" }} {{ template "get_host" "https://adb-xxxx.xx.azuredatabricks.net/sql/queries" }} From 2b841f02abbb0d8c3a83269598b24f71dd07e31f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Jul 2023 14:55:13 +0200 Subject: [PATCH 44/60] fix test for windows --- internal/init_test.go | 21 ++++++++++----------- libs/template/materialize_test.go | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/internal/init_test.go b/internal/init_test.go index c497107eda6..df3febfce46 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -18,10 +18,10 @@ func assertFileContains(t *testing.T, path string, substr string) { assert.Contains(t, string(b), substr) } -func assertLocalFileContent(t *testing.T, path string, expected string) { +func assertFileNotContains(t *testing.T, path string, substr string) { b, err := os.ReadFile(path) require.NoError(t, err) - assert.Equal(t, string(b), expected) + assert.NotContains(t, string(b), substr) } func TestAccTemplateInitializationForDevConfig(t *testing.T) { @@ -88,17 +88,16 @@ func TestAccTemplateInitializationForProdConfig(t *testing.T) { err = cmd.Execute() require.NoError(t, err) - // assert on materialized template + // assert on materialized template files assert.FileExists(t, filepath.Join(instanceDir, "production_project", "azure_file")) assert.FileExists(t, filepath.Join(instanceDir, "production_project", ".azure_devops")) assert.NoFileExists(t, filepath.Join(instanceDir, "production_project", "aws_file")) - - assertLocalFileContent(t, filepath.Join(instanceDir, "production_project", "azure_file"), - ` -This file should only be generated for Azure -shreyas.goenka@databricks.com -https://adb-xxxx.xx.azuredatabricks.net -`) - assertFileContains(t, filepath.Join(instanceDir, "production_project", ".azure_devops"), "This is a production project") + + // assert azure_file is computed correctly + azureFilePath := filepath.Join(instanceDir, "production_project", "azure_file") + assertFileContains(t, azureFilePath, "This file should only be generated for Azure") + assertFileContains(t, azureFilePath, "shreyas.goenka@databricks.com") + assertFileContains(t, azureFilePath, "https://adb-xxxx.xx.azuredatabricks.net") + assertFileNotContains(t, azureFilePath, "https://adb-xxxx.xx.azuredatabricks.net/sql/queries") } diff --git a/libs/template/materialize_test.go b/libs/template/materialize_test.go index 5678c9bb136..d73b54e2b30 100644 --- a/libs/template/materialize_test.go +++ b/libs/template/materialize_test.go @@ -137,7 +137,7 @@ func TestMaterializeFilePermissionsAreCopiedForWindows(t *testing.T) { // create template in temp directory err := os.Mkdir(filepath.Join(tmp, "my_tmpl"), 0777) require.NoError(t, err) - err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "schema.json"), []byte(` + err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "databricks_template_schema.json"), []byte(` { "properties": { "a": { From 7ccd81c5434c4647e6c927ce076baa6ce95cbaab Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Jul 2023 15:00:03 +0200 Subject: [PATCH 45/60] added comments --- libs/template/execute.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/libs/template/execute.go b/libs/template/execute.go index e341a620c7e..3a99383b6b4 100644 --- a/libs/template/execute.go +++ b/libs/template/execute.go @@ -10,8 +10,16 @@ import ( "text/template" ) +// This structure renders any template files during project initialization type renderer struct { - config map[string]any + // A config that is the "dot" value available to any template being rendered. + // Refer to https://pkg.go.dev/text/template for how templates can use + // this "dot" value + config map[string]any + + // A base template with helper functions and user defined template in the + // library directory loaded. This is used as the base to compute any project + // templates during file tree walk baseTemplate *template.Template } From 4111b03a670efd12c6a06b0a3c222956041c8aec Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Jul 2023 15:12:30 +0200 Subject: [PATCH 46/60] renamed file to renderer --- libs/template/{execute.go => renderer.go} | 0 libs/template/{execute_test.go => renderer_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename libs/template/{execute.go => renderer.go} (100%) rename libs/template/{execute_test.go => renderer_test.go} (100%) diff --git a/libs/template/execute.go b/libs/template/renderer.go similarity index 100% rename from libs/template/execute.go rename to libs/template/renderer.go diff --git a/libs/template/execute_test.go b/libs/template/renderer_test.go similarity index 100% rename from libs/template/execute_test.go rename to libs/template/renderer_test.go From 1c63f69ed2c3bf37d35baf4e4b207dc3776fbed2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Jul 2023 10:48:45 +0200 Subject: [PATCH 47/60] make instance-path optional --- cmd/bundle/init.go | 8 +++++--- internal/init_test.go | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index ccde90dd47c..bd0ed5da52c 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -9,16 +9,18 @@ var initCmd = &cobra.Command{ Use: "init TEMPLATE_PATH INSTANCE_PATH", Short: "Initialize Template", Long: `Initialize template`, - Args: cobra.ExactArgs(2), + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return template.Materialize(args[0], args[1], configFile) + return template.Materialize(args[0], outputDir, configFile) }, } var configFile string +var outputDir string func init() { - initCmd.Flags().StringVar(&configFile, "config-file", "", "input parameters for template initialization") + initCmd.Flags().StringVar(&configFile, "config-file", "", "Input parameters for template initialization") + initCmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to output the generated project into") initCmd.MarkFlagRequired("config-file") AddCommand(initCmd) } diff --git a/internal/init_test.go b/internal/init_test.go index df3febfce46..e6ec1dff740 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -42,7 +42,7 @@ func TestAccTemplateInitializationForDevConfig(t *testing.T) { // materialize the template cmd := root.RootCmd - cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), tmp, "--config-file", filepath.Join(tmp, "config.json")}) + cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--output-dir", tmp, "--config-file", filepath.Join(tmp, "config.json")}) err = cmd.Execute() require.NoError(t, err) @@ -84,7 +84,7 @@ func TestAccTemplateInitializationForProdConfig(t *testing.T) { cmd := root.RootCmd childCommands := cmd.Commands() fmt.Println(childCommands) - cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), instanceDir, "--config-file", filepath.Join(configDir, "my_config.json")}) + cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--output-dir", instanceDir, "--config-file", filepath.Join(configDir, "my_config.json")}) err = cmd.Execute() require.NoError(t, err) From c433f96b2109301aa49b96285a0ecad93535cf35 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Jul 2023 11:08:56 +0200 Subject: [PATCH 48/60] added tests for variable read and function defination --- cmd/bundle/init.go | 5 ++++ libs/template/renderer_test.go | 30 +++++++++++++++++++ .../testdata/email/library/email.tmpl | 1 + .../template/testdata/email/template/my_email | 1 + .../get_host/library/get_host_func.tmpl | 1 + .../testdata/get_host/template/my_host | 1 + 6 files changed, 39 insertions(+) create mode 100644 libs/template/testdata/email/library/email.tmpl create mode 100644 libs/template/testdata/email/template/my_email create mode 100644 libs/template/testdata/get_host/library/get_host_func.tmpl create mode 100644 libs/template/testdata/get_host/template/my_host diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index bd0ed5da52c..9acc7199993 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -18,9 +18,14 @@ var initCmd = &cobra.Command{ var configFile string var outputDir string +// TODO: move integration tests to be unit tests OR increase coverage + func init() { initCmd.Flags().StringVar(&configFile, "config-file", "", "Input parameters for template initialization") initCmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to output the generated project into") initCmd.MarkFlagRequired("config-file") + // TODO: make this flag optional and initialize into current directory. + // Should we initialize into current directory? + initCmd.MarkFlagRequired("output-dir") AddCommand(initCmd) } diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index a9431536c96..de1ae7c2f69 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -10,6 +10,36 @@ import ( "github.com/stretchr/testify/require" ) +func TestRendererVariableRead(t *testing.T) { + r, err := newRenderer(nil, "./testdata/email/library") + require.NoError(t, err) + + tmpDir := t.TempDir() + + err = walkFileTree(r, "./testdata/email/template", tmpDir) + require.NoError(t, err) + + b, err := os.ReadFile(filepath.Join(tmpDir, "my_email")) + require.NoError(t, err) + + assert.Equal(t, "shreyas.goenka@databricks.com\n", string(b)) +} + +func TestUrlParseUsageInFunction(t *testing.T) { + r, err := newRenderer(nil, "./testdata/get_host/library") + require.NoError(t, err) + + tmpDir := t.TempDir() + + err = walkFileTree(r, "./testdata/get_host/template", tmpDir) + require.NoError(t, err) + + b, err := os.ReadFile(filepath.Join(tmpDir, "my_host")) + require.NoError(t, err) + + assert.Equal(t, "https://www.host.com\n", string(b)) +} + func TestExecuteTemplate(t *testing.T) { templateText := `"{{.count}} items are made of {{.Material}}". diff --git a/libs/template/testdata/email/library/email.tmpl b/libs/template/testdata/email/library/email.tmpl new file mode 100644 index 00000000000..1897d46b3ed --- /dev/null +++ b/libs/template/testdata/email/library/email.tmpl @@ -0,0 +1 @@ +{{define "email"}}shreyas.goenka@databricks.com{{end}} diff --git a/libs/template/testdata/email/template/my_email b/libs/template/testdata/email/template/my_email new file mode 100644 index 00000000000..0b74ef47cd4 --- /dev/null +++ b/libs/template/testdata/email/template/my_email @@ -0,0 +1 @@ +{{template "email"}} diff --git a/libs/template/testdata/get_host/library/get_host_func.tmpl b/libs/template/testdata/get_host/library/get_host_func.tmpl new file mode 100644 index 00000000000..e88cd23acf8 --- /dev/null +++ b/libs/template/testdata/get_host/library/get_host_func.tmpl @@ -0,0 +1 @@ +{{define "get_host"}}{{ with urlParse . }}{{print .Scheme `://` .Host}}{{end}}{{end}} diff --git a/libs/template/testdata/get_host/template/my_host b/libs/template/testdata/get_host/template/my_host new file mode 100644 index 00000000000..b7a738790da --- /dev/null +++ b/libs/template/testdata/get_host/template/my_host @@ -0,0 +1 @@ +{{template "get_host" "https://www.host.com/a/b/c/d/e?o=123#fragment"}} From 9805653331fe605160735fd3aaca829f71054e42 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Jul 2023 14:40:22 +0200 Subject: [PATCH 49/60] added fail method and checks that regex works properly --- libs/template/helpers.go | 17 ++++++++++++ libs/template/renderer.go | 4 +++ libs/template/renderer_test.go | 27 ++++++++++++++++++- .../is_https/library/is_https_check.tmpl | 6 +++++ .../is_https/template_is_https/my_check | 3 +++ .../is_https/template_not_https/my_check | 1 + 6 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 libs/template/testdata/is_https/library/is_https_check.tmpl create mode 100644 libs/template/testdata/is_https/template_is_https/my_check create mode 100644 libs/template/testdata/is_https/template_not_https/my_check diff --git a/libs/template/helpers.go b/libs/template/helpers.go index ca255457e9a..1bd88fcf425 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -2,12 +2,23 @@ package template import ( "errors" + "fmt" "net/url" + "regexp" "text/template" ) var errSkipThisFile = errors.New("skip generating this file") + +type ErrFail struct { + msg string +} + +func (err ErrFail) Error() string { + return err.msg +} + var HelperFuncs = template.FuncMap{ // Text template execution returns the error only if it's the second return // value from a function: https://pkg.go.dev/text/template#hdr-Pipelines @@ -17,4 +28,10 @@ var HelperFuncs = template.FuncMap{ "urlParse": func(rawUrl string) (*url.URL, error) { return url.Parse(rawUrl) }, + "regexpCompile": func(expr string) (*regexp.Regexp, error) { + return regexp.Compile(expr) + }, + "fail": func(format string, args ...any) (any, error) { + return nil, ErrFail{fmt.Sprintf(format, args...)} + }, } diff --git a/libs/template/renderer.go b/libs/template/renderer.go index 3a99383b6b4..9a8b337c40d 100644 --- a/libs/template/renderer.go +++ b/libs/template/renderer.go @@ -79,6 +79,10 @@ func (r *renderer) generateFile(pathTemplate, contentTemplate string, perm fs.Fi // skip this file return nil } + // Capture errors caused by the "fail" helper function + if target := (&ErrFail{}); errors.As(err, target) { + return target + } if err != nil { return fmt.Errorf("failed to compute file content for %s. %w", pathTemplate, err) } diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index de1ae7c2f69..4a0282cc8c8 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -25,7 +25,7 @@ func TestRendererVariableRead(t *testing.T) { assert.Equal(t, "shreyas.goenka@databricks.com\n", string(b)) } -func TestUrlParseUsageInFunction(t *testing.T) { +func TestRendererUrlParseUsageInFunction(t *testing.T) { r, err := newRenderer(nil, "./testdata/get_host/library") require.NoError(t, err) @@ -40,6 +40,31 @@ func TestUrlParseUsageInFunction(t *testing.T) { assert.Equal(t, "https://www.host.com\n", string(b)) } +func TestRendererRegexpCheckFailing(t *testing.T) { + r, err := newRenderer(nil, "./testdata/is_https/library") + require.NoError(t, err) + + tmpDir := t.TempDir() + + err = walkFileTree(r, "./testdata/is_https/template_not_https", tmpDir) + require.NoError(t, err) +} + +func TestRendererRegexpCheckPassing(t *testing.T) { + r, err := newRenderer(nil, "./testdata/is_https/library") + require.NoError(t, err) + + tmpDir := t.TempDir() + + err = walkFileTree(r, "./testdata/is_https/template_is_https", tmpDir) + require.NoError(t, err) + + b, err := os.ReadFile(filepath.Join(tmpDir, "my_check")) + require.NoError(t, err) + + assert.Equal(t, "this file is created if validation passes\n", string(b)) +} + func TestExecuteTemplate(t *testing.T) { templateText := `"{{.count}} items are made of {{.Material}}". diff --git a/libs/template/testdata/is_https/library/is_https_check.tmpl b/libs/template/testdata/is_https/library/is_https_check.tmpl new file mode 100644 index 00000000000..c43083eb113 --- /dev/null +++ b/libs/template/testdata/is_https/library/is_https_check.tmpl @@ -0,0 +1,6 @@ +{{define "is_https_check" -}} + {{- $regex := regexpCompile `^https` -}} + {{- if not ($regex.MatchString .) -}} + {{- fail "expected %s to start with https" . -}} + {{- end -}} +{{- end}} diff --git a/libs/template/testdata/is_https/template_is_https/my_check b/libs/template/testdata/is_https/template_is_https/my_check new file mode 100644 index 00000000000..58cd7b87fdc --- /dev/null +++ b/libs/template/testdata/is_https/template_is_https/my_check @@ -0,0 +1,3 @@ +{{- template "is_https_check" "https://www.databricks.com" -}} + +this file is created if validation passes diff --git a/libs/template/testdata/is_https/template_not_https/my_check b/libs/template/testdata/is_https/template_not_https/my_check new file mode 100644 index 00000000000..c3dc74c04c3 --- /dev/null +++ b/libs/template/testdata/is_https/template_not_https/my_check @@ -0,0 +1 @@ +{{template "is_https_check" "/not-a-url"}} From 223540e104f20417ade510fe5f7d1d515c758acf Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Jul 2023 15:17:15 +0200 Subject: [PATCH 50/60] wip support for skip, to undo --- cmd/bundle/init.go | 2 +- .../databricks_template_schema.json | 13 +- .../{{.directory_name}}/.{{.file_name}} | 8 + .../aws_file | 0 .../azure_file | 0 .../template/{{.project_name}}/.{{.ci_type}} | 5 - libs/template/helpers.go | 10 ++ libs/template/materialize_test.go | 6 + libs/template/renderer.go | 137 +++++++++++++--- libs/template/renderer_test.go | 154 +++++++++++++++++- .../is_https/template_is_https/my_check | 1 - 11 files changed, 289 insertions(+), 47 deletions(-) create mode 100644 internal/testdata/init/templateDefinition/template/{{.directory_name}}/.{{.file_name}} rename internal/testdata/init/templateDefinition/template/{{{.project_name}} => {{.directory_name}}}/aws_file (100%) rename internal/testdata/init/templateDefinition/template/{{{.project_name}} => {{.directory_name}}}/azure_file (100%) delete mode 100644 internal/testdata/init/templateDefinition/template/{{.project_name}}/.{{.ci_type}} diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index 9acc7199993..598e6492bd0 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -6,7 +6,7 @@ import ( ) var initCmd = &cobra.Command{ - Use: "init TEMPLATE_PATH INSTANCE_PATH", + Use: "init TEMPLATE_PATH", Short: "Initialize Template", Long: `Initialize template`, Args: cobra.ExactArgs(1), diff --git a/internal/testdata/init/templateDefinition/databricks_template_schema.json b/internal/testdata/init/templateDefinition/databricks_template_schema.json index 5e8b40e83e5..cd0046f4258 100644 --- a/internal/testdata/init/templateDefinition/databricks_template_schema.json +++ b/internal/testdata/init/templateDefinition/databricks_template_schema.json @@ -1,19 +1,18 @@ { "properties": { - "project_name": { - "description": "Name of the project", + "directory_name": { + "description": "Name of the directory", "type": "string" }, - "cloud_type": { - "description": "type of the cloud for the project", + "skip_extra_content_if_abc": { "type": "string" }, - "is_production": { + "skip_file_if_true": { "type": "boolean" }, - "ci_type": { + "file_name": { "type": "string", - "description": "type of the CI runner, eg: github, azure devops" + "description": "Name of the file" } } } diff --git a/internal/testdata/init/templateDefinition/template/{{.directory_name}}/.{{.file_name}} b/internal/testdata/init/templateDefinition/template/{{.directory_name}}/.{{.file_name}} new file mode 100644 index 00000000000..f71e78daadd --- /dev/null +++ b/internal/testdata/init/templateDefinition/template/{{.directory_name}}/.{{.file_name}} @@ -0,0 +1,8 @@ +{{if eq .skip_extra_content_if_abc "abc" -}} +this is extra content +{{- end -}} + +{{if eq .skip_file_if_true }} +this is extra content +{{- end -}} +{{- skipThisFile -}} diff --git a/internal/testdata/init/templateDefinition/template/{{.project_name}}/aws_file b/internal/testdata/init/templateDefinition/template/{{.directory_name}}/aws_file similarity index 100% rename from internal/testdata/init/templateDefinition/template/{{.project_name}}/aws_file rename to internal/testdata/init/templateDefinition/template/{{.directory_name}}/aws_file diff --git a/internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file b/internal/testdata/init/templateDefinition/template/{{.directory_name}}/azure_file similarity index 100% rename from internal/testdata/init/templateDefinition/template/{{.project_name}}/azure_file rename to internal/testdata/init/templateDefinition/template/{{.directory_name}}/azure_file diff --git a/internal/testdata/init/templateDefinition/template/{{.project_name}}/.{{.ci_type}} b/internal/testdata/init/templateDefinition/template/{{.project_name}}/.{{.ci_type}} deleted file mode 100644 index b5c2d444085..00000000000 --- a/internal/testdata/init/templateDefinition/template/{{.project_name}}/.{{.ci_type}} +++ /dev/null @@ -1,5 +0,0 @@ -{{if .is_production}} -This is a production project -{{else}} -This is a development project -{{end}} diff --git a/libs/template/helpers.go b/libs/template/helpers.go index 1bd88fcf425..610d061a861 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -6,10 +6,13 @@ import ( "net/url" "regexp" "text/template" + + "golang.org/x/exp/slices" ) var errSkipThisFile = errors.New("skip generating this file") +var skipPatterns = make([]string, 0) type ErrFail struct { msg string @@ -25,6 +28,13 @@ var HelperFuncs = template.FuncMap{ "skipThisFile": func() (any, error) { return nil, errSkipThisFile }, + // TODO: write an explanation for this function + "skip": func(pattern string) error { + if !slices.Contains(skipPatterns, pattern) { + skipPatterns = append(skipPatterns, pattern) + } + return nil + }, "urlParse": func(rawUrl string) (*url.URL, error) { return url.Parse(rawUrl) }, diff --git a/libs/template/materialize_test.go b/libs/template/materialize_test.go index d73b54e2b30..2bbc0295597 100644 --- a/libs/template/materialize_test.go +++ b/libs/template/materialize_test.go @@ -28,6 +28,12 @@ func assertFilePerm(t *testing.T, path string, perm fs.FileMode) { assert.Equal(t, perm, stat.Mode().Perm()) } +func assertFileContent(t *testing.T, path string, content string) { + b, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, content, string(b)) +} + func TestMaterializeEmptyDirsAreNotGenerated(t *testing.T) { tmp := setupConfig(t, ` { diff --git a/libs/template/renderer.go b/libs/template/renderer.go index 9a8b337c40d..80f36870d47 100644 --- a/libs/template/renderer.go +++ b/libs/template/renderer.go @@ -8,6 +8,8 @@ import ( "path/filepath" "strings" "text/template" + + "golang.org/x/exp/slices" ) // This structure renders any template files during project initialization @@ -72,55 +74,142 @@ func (r *renderer) executeTemplate(templateDefinition string) (string, error) { return result.String(), nil } -func (r *renderer) generateFile(pathTemplate, contentTemplate string, perm fs.FileMode) error { +func (r *renderer) generateFile(pathTemplate, contentTemplate string, perm fs.FileMode) (*inMemoryFile, error) { // compute file content fileContent, err := r.executeTemplate(contentTemplate) if errors.Is(err, errSkipThisFile) { // skip this file - return nil + return nil, nil } + // Capture errors caused by the "fail" helper function if target := (&ErrFail{}); errors.As(err, target) { - return target + return nil, target } if err != nil { - return fmt.Errorf("failed to compute file content for %s. %w", pathTemplate, err) + return nil, fmt.Errorf("failed to compute file content for %s. %w", pathTemplate, err) } // compute the path for this file path, err := r.executeTemplate(pathTemplate) if err != nil { - return fmt.Errorf("failed to compute path for %s. %w", pathTemplate, err) + return nil, fmt.Errorf("failed to compute path for %s. %w", pathTemplate, err) } - // create any intermediate directories required. Directories are lazily generated - // only when they are required for a file. - err = os.MkdirAll(filepath.Dir(path), 0755) + + return &inMemoryFile{ + path: path, + content: fileContent, + perm: perm, + }, nil +} + +type inMemoryFile struct { + path string + // TODO: use bytes in to serialize for binary files, Can we just use string here, is it the same + content string + perm fs.FileMode +} + +func (r *renderer) generateDir(templateDir, instanceDir string) (map[*inMemoryFile]any, error) { + entries, err := os.ReadDir(templateDir) if err != nil { - return err + return nil, err + } + + // Args from any calls to the {{skip}} helper function will be appended to this list. + skipPatterns := make([]string, 0) + // Add skip functional closure which uses the newly initialized slice to store patterns + r.baseTemplate.Funcs(template.FuncMap{ + "skip": func(pattern string) error { + if !slices.Contains(skipPatterns, pattern) { + skipPatterns = append(skipPatterns, pattern) + } + return nil + }, + }) + + // Initialize set to contain in memory files + files := make(map[*inMemoryFile]any, 0) + + // TODO: use slice and pure functions for skip computation + // TODO: Have a global list of skip patterns accumulated for the entire file tree + // TODO: Write file tree to disk all at once + // TODO: optimization skip subdirectoies if we already already can + + // Compute the in memory file representation for the executed template + for _, entry := range entries { + if entry.IsDir() { + continue + } + + // read template file to get the templatized content for the file + b, err := os.ReadFile(filepath.Join(templateDir, entry.Name())) + if err != nil { + return nil, err + } + contentTemplate := string(b) + + // Generate an in memory representation of the file, by executing the template + f, err := r.generateFile(filepath.Join(instanceDir, entry.Name()), contentTemplate, entry.Type().Perm()) + if err != nil { + return nil, err + } + files[f] = nil } - // write content to file - return os.WriteFile(path, []byte(fileContent), perm) + // Match glob patterns stored, and delete matching in memory files + return nil, deleteSkippedFiles(files, skipPatterns) } -// TODO: use local filer client for this function. https://github.com/databricks/cli/issues/511 -func walkFileTree(r *renderer, templateRoot, instanceRoot string) error { - return filepath.WalkDir(templateRoot, func(path string, d fs.DirEntry, err error) error { +// deletes any files in [files] whose name matches a glob pattern in [skipPatterns] +func deleteSkippedFiles(files map[*inMemoryFile]any, skipPatterns []string) error { + for f := range files { + isSkipped := false + for _, pattern := range skipPatterns { + matched, err := filepath.Match(pattern, filepath.Base(f.path)) + if err != nil { + return fmt.Errorf("error while trying to match file %s against glob pattern %s: %w", filepath.Base(f.path), pattern, err) + } + if matched { + isSkipped = true + break + } + } + if isSkipped { + delete(files, f) + } + } + return nil +} + +func materializeFiles(files map[*inMemoryFile]any) error { + for f := range files { + // create any intermediate directories required. Directories are lazily generated + // only when they are required for a file. + err := os.MkdirAll(filepath.Dir(f.path), 0755) if err != nil { return err } - // skip if current entry is a directory - if d.IsDir() { - return nil + // write content to file + err = os.WriteFile(f.path, []byte(f.content), f.perm) + if err != nil { + return err } + } + return nil +} - // read template file to get the templatized content for the file - b, err := os.ReadFile(path) +func walkFileTree(r *renderer, templateRoot, instanceRoot string) error { + return filepath.WalkDir(templateRoot, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } - contentTemplate := string(b) + + // skip if current entry is not a directory + if !d.IsDir() { + return nil + } // get relative path to the template file, This forms the template for the // path to the file @@ -129,13 +218,13 @@ func walkFileTree(r *renderer, templateRoot, instanceRoot string) error { return err } - // Get info about the template file. Used to ensure instance path also - // has the same permission bits - info, err := d.Info() + // Compute in memory representation for files in the current directory + files, err := r.generateDir(path, filepath.Join(instanceRoot, relPathTemplate)) if err != nil { return err } - return r.generateFile(filepath.Join(instanceRoot, relPathTemplate), contentTemplate, info.Mode().Perm()) + // Materialize these files onto the disk + return materializeFiles(files) }) } diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index 4a0282cc8c8..036ad96b43c 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -1,6 +1,7 @@ package template import ( + "io/fs" "os" "path/filepath" "testing" @@ -129,19 +130,154 @@ func TestGenerateFile(t *testing.T) { }, baseTemplate: template.New("base"), } - err := r.generateFile(pathTemplate, contentTemplate, 0444) + f, err := r.generateFile(pathTemplate, contentTemplate, 0444) require.NoError(t, err) - // assert file exists - assert.FileExists(t, filepath.Join(tmp, "cat", "wool", "foo", "1.txt")) + // assert file content + assert.Equal(t, "\"1 items are made of wool\".\n\t\n\tcat wool is not too bad...\n\t\n\t", f.content) - // assert file content is created correctly - b, err := os.ReadFile(filepath.Join(tmp, "cat", "wool", "foo", "1.txt")) + // assert file permissions are correctly assigned + assert.Equal(t, fs.FileMode(0444), f.perm) + + // assert file path + assert.Equal(t, filepath.Join(tmp, "cat", "wool", "foo", "1.txt"), f.path) +} + +func TestDeleteSkippedFiles(t *testing.T) { + tmpDir := t.TempDir() + inputFiles := map[*inMemoryFile]any{ + { + path: filepath.Join(tmpDir, "aaa"), + content: "one", + perm: 0444, + }: nil, + { + path: filepath.Join(tmpDir, "abb"), + content: "two", + perm: 0444, + }: nil, + { + path: filepath.Join(tmpDir, "bbb"), + content: "three", + perm: 0666, + }: nil, + } + + err := deleteSkippedFiles(inputFiles, []string{"aaa", "abb"}) require.NoError(t, err) - assert.Equal(t, "\"1 items are made of wool\".\n\t\n\tcat wool is not too bad...\n\t\n\t", string(b)) - // assert file permissions are correctly assigned - stat, err := os.Stat(filepath.Join(tmp, "cat", "wool", "foo", "1.txt")) + assert.Len(t, inputFiles, 1) + for v := range inputFiles { + assert.Equal(t, inMemoryFile{ + path: filepath.Join(tmpDir, "bbb"), + content: "three", + perm: 0666, + }, *v) + } +} + +func TestDeleteSkippedFilesWithGlobPatterns(t *testing.T) { + tmpDir := t.TempDir() + inputFiles := map[*inMemoryFile]any{ + { + path: filepath.Join(tmpDir, "aaa"), + content: "one", + perm: 0444, + }: nil, + { + path: filepath.Join(tmpDir, "abb"), + content: "two", + perm: 0444, + }: nil, + { + path: filepath.Join(tmpDir, "bbb"), + content: "three", + perm: 0666, + }: nil, + { + path: filepath.Join(tmpDir, "ddd"), + content: "four", + perm: 0666, + }: nil, + } + + err := deleteSkippedFiles(inputFiles, []string{"a*"}) + require.NoError(t, err) + + files := make([]inMemoryFile, 0) + for v := range inputFiles { + files = append(files, *v) + } + assert.Len(t, files, 2) + assert.Contains(t, files, inMemoryFile{ + path: filepath.Join(tmpDir, "bbb"), + content: "three", + perm: 0666, + }) + assert.Contains(t, files, inMemoryFile{ + path: filepath.Join(tmpDir, "ddd"), + content: "four", + perm: 0666, + }) +} + +func TestSkipAllFiles(t *testing.T) { + tmpDir := t.TempDir() + inputFiles := map[*inMemoryFile]any{ + { + path: filepath.Join(tmpDir, "aaa"), + content: "one", + perm: 0444, + }: nil, + { + path: filepath.Join(tmpDir, "abb"), + content: "two", + perm: 0444, + }: nil, + { + path: filepath.Join(tmpDir, "bbb"), + content: "three", + perm: 0666, + }: nil, + } + + err := deleteSkippedFiles(inputFiles, []string{"*"}) require.NoError(t, err) - assert.Equal(t, uint(0444), uint(stat.Mode().Perm())) + assert.Len(t, inputFiles, 0) +} + +func TestTemplateMaterializeFiles(t *testing.T) { + tmpDir := t.TempDir() + inputFiles := map[*inMemoryFile]any{ + { + path: filepath.Join(tmpDir, "aaa"), + content: "one", + perm: 0444, + }: nil, + { + path: filepath.Join(tmpDir, "abb"), + content: "two", + perm: 0444, + }: nil, + { + path: filepath.Join(tmpDir, "bbb"), + content: "three", + perm: 0666, + }: nil, + } + + err := materializeFiles(inputFiles) + assert.NoError(t, err) + + path := filepath.Join(tmpDir, "aaa") + assertFilePerm(t, path, 0444) + assertFileContent(t, path, "one") + + path = filepath.Join(tmpDir, "abb") + assertFilePerm(t, path, 0444) + assertFileContent(t, path, "two") + + path = filepath.Join(tmpDir, "bbb") + assertFilePerm(t, path, 0666) + assertFileContent(t, path, "three") } diff --git a/libs/template/testdata/is_https/template_is_https/my_check b/libs/template/testdata/is_https/template_is_https/my_check index 58cd7b87fdc..a9298c3b014 100644 --- a/libs/template/testdata/is_https/template_is_https/my_check +++ b/libs/template/testdata/is_https/template_is_https/my_check @@ -1,3 +1,2 @@ {{- template "is_https_check" "https://www.databricks.com" -}} - this file is created if validation passes From 0a7e92cd1ef5d6fa3ccaf8fd46d06300a65fe0ce Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Jul 2023 13:48:41 +0200 Subject: [PATCH 51/60] wip, sending for style feedback --- libs/template/helpers.go | 19 -- libs/template/materialize.go | 5 +- libs/template/renderer.go | 234 +++++++++------- libs/template/renderer_test.go | 488 +++++++++++++++++---------------- 4 files changed, 378 insertions(+), 368 deletions(-) diff --git a/libs/template/helpers.go b/libs/template/helpers.go index 610d061a861..2f206d2b5d5 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -1,19 +1,12 @@ package template import ( - "errors" "fmt" "net/url" "regexp" "text/template" - - "golang.org/x/exp/slices" ) -var errSkipThisFile = errors.New("skip generating this file") - -var skipPatterns = make([]string, 0) - type ErrFail struct { msg string } @@ -23,18 +16,6 @@ func (err ErrFail) Error() string { } var HelperFuncs = template.FuncMap{ - // Text template execution returns the error only if it's the second return - // value from a function: https://pkg.go.dev/text/template#hdr-Pipelines - "skipThisFile": func() (any, error) { - return nil, errSkipThisFile - }, - // TODO: write an explanation for this function - "skip": func(pattern string) error { - if !slices.Contains(skipPatterns, pattern) { - skipPatterns = append(skipPatterns, pattern) - } - return nil - }, "urlParse": func(rawUrl string) (*url.URL, error) { return url.Parse(rawUrl) }, diff --git a/libs/template/materialize.go b/libs/template/materialize.go index 03eac22fc3d..b4b1b0e5128 100644 --- a/libs/template/materialize.go +++ b/libs/template/materialize.go @@ -1,6 +1,7 @@ package template import ( + "context" "path/filepath" ) @@ -21,11 +22,11 @@ func Materialize(templateRoot, instanceRoot, configPath string) error { return err } - r, err := newRenderer(config, filepath.Join(templateRoot, libraryDirName)) + r, err := newRenderer(context.TODO(), config, filepath.Join(templateRoot, libraryDirName), instanceRoot, filepath.Join(templateRoot, templateDirName)) if err != nil { return err } // materialize the template - return walkFileTree(r, filepath.Join(templateRoot, templateDirName), instanceRoot) + return walk(r, ".") } diff --git a/libs/template/renderer.go b/libs/template/renderer.go index 80f36870d47..0febc005e9b 100644 --- a/libs/template/renderer.go +++ b/libs/template/renderer.go @@ -1,19 +1,32 @@ package template import ( + "context" "errors" "fmt" + "io" "io/fs" - "os" "path/filepath" "strings" "text/template" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/logger" "golang.org/x/exp/slices" ) +type inMemoryFile struct { + path string + // TODO: use bytes in to serialize for binary files, Can we just use string here, is it the same + content string + perm fs.FileMode +} + // This structure renders any template files during project initialization type renderer struct { + ctx context.Context + // A config that is the "dot" value available to any template being rendered. // Refer to https://pkg.go.dev/text/template for how templates can use // this "dot" value @@ -23,9 +36,15 @@ type renderer struct { // library directory loaded. This is used as the base to compute any project // templates during file tree walk baseTemplate *template.Template + + files []*inMemoryFile + skipPatterns []string + + templateFiler filer.Filer + instanceFiler filer.Filer } -func newRenderer(config map[string]any, libraryRoot string) (*renderer, error) { +func newRenderer(ctx context.Context, config map[string]any, libraryRoot, instanceRoot, templateRoot string) (*renderer, error) { // All user defined functions will be available inside library root libraryGlob := filepath.Join(libraryRoot, "*") @@ -44,9 +63,27 @@ func newRenderer(config map[string]any, libraryRoot string) (*renderer, error) { } } + // create template filer + templateFiler, err := filer.NewLocalClient(templateRoot) + if err != nil { + return nil, err + } + + instanceFiler, err := filer.NewLocalClient(instanceRoot) + if err != nil { + return nil, err + } + + ctx = log.NewContext(ctx, log.GetLogger(ctx).With("action", "initialize-template")) + return &renderer{ - config: config, - baseTemplate: tmpl, + config: config, + baseTemplate: tmpl, + files: make([]*inMemoryFile, 0), + templateFiler: templateFiler, + ctx: ctx, + skipPatterns: make([]string, 0), + instanceFiler: instanceFiler, }, nil } @@ -74,125 +111,110 @@ func (r *renderer) executeTemplate(templateDefinition string) (string, error) { return result.String(), nil } -func (r *renderer) generateFile(pathTemplate, contentTemplate string, perm fs.FileMode) (*inMemoryFile, error) { - // compute file content - fileContent, err := r.executeTemplate(contentTemplate) - if errors.Is(err, errSkipThisFile) { - // skip this file - return nil, nil +func (r *renderer) computeFile(relPathTemplate string) (*inMemoryFile, error) { + // read template file contents + templateReader, err := r.templateFiler.Read(r.ctx, relPathTemplate) + if err != nil { + return nil, err + } + contentTemplate, err := io.ReadAll(templateReader) + if err != nil { + return nil, err } + // execute the contents of the file as a template + content, err := r.executeTemplate(string(contentTemplate)) // Capture errors caused by the "fail" helper function if target := (&ErrFail{}); errors.As(err, target) { return nil, target } if err != nil { - return nil, fmt.Errorf("failed to compute file content for %s. %w", pathTemplate, err) + return nil, fmt.Errorf("failed to compute file content for %s. %w", relPathTemplate, err) } - // compute the path for this file - path, err := r.executeTemplate(pathTemplate) + // Execute relative path template to get materialized path for the file + relPath, err := r.executeTemplate(relPathTemplate) if err != nil { - return nil, fmt.Errorf("failed to compute path for %s. %w", pathTemplate, err) + return nil, err } + // Read permissions for the file + info, err := r.templateFiler.Stat(r.ctx, relPathTemplate) + if err != nil { + return nil, err + } + perm := info.Mode().Perm() + return &inMemoryFile{ - path: path, - content: fileContent, + path: relPath, + content: content, perm: perm, }, nil } -type inMemoryFile struct { - path string - // TODO: use bytes in to serialize for binary files, Can we just use string here, is it the same - content string - perm fs.FileMode -} +func walk(r *renderer, dirPathTemplate string) error { + entries, err := r.templateFiler.ReadDir(r.ctx, dirPathTemplate) + if err != nil { + return err + } + + // Separate files and directories from entries. We would like to process + // all the files first to capture all skip glob patterns. + files := make([]fs.DirEntry, 0) + directories := make([]fs.DirEntry, 0) + for _, entry := range entries { + if entry.IsDir() { + directories = append(directories, entry) + } else { + files = append(files, entry) + } + } -func (r *renderer) generateDir(templateDir, instanceDir string) (map[*inMemoryFile]any, error) { - entries, err := os.ReadDir(templateDir) + dirPath, err := r.executeTemplate(dirPathTemplate) if err != nil { - return nil, err + return err } - // Args from any calls to the {{skip}} helper function will be appended to this list. - skipPatterns := make([]string, 0) - // Add skip functional closure which uses the newly initialized slice to store patterns + // Add skip functional closure r.baseTemplate.Funcs(template.FuncMap{ - "skip": func(pattern string) error { - if !slices.Contains(skipPatterns, pattern) { - skipPatterns = append(skipPatterns, pattern) + "skip": func(relPattern string) error { + // patterns are specified relative to current directory of the file + // {{skip}} function is called from + pattern := filepath.Join(dirPath, relPattern) + if !slices.Contains(r.skipPatterns, pattern) { + logger.Infof(r.ctx, "adding skip pattern: %s", pattern) + r.skipPatterns = append(r.skipPatterns, pattern) } return nil }, }) - // Initialize set to contain in memory files - files := make(map[*inMemoryFile]any, 0) - - // TODO: use slice and pure functions for skip computation - // TODO: Have a global list of skip patterns accumulated for the entire file tree - // TODO: Write file tree to disk all at once - // TODO: optimization skip subdirectoies if we already already can - - // Compute the in memory file representation for the executed template - for _, entry := range entries { - if entry.IsDir() { - continue - } - - // read template file to get the templatized content for the file - b, err := os.ReadFile(filepath.Join(templateDir, entry.Name())) + // Compute files in current directory, and add them to file tree + for _, f := range files { + instanceFile, err := r.computeFile(filepath.Join(dirPathTemplate, f.Name())) if err != nil { - return nil, err - } - contentTemplate := string(b) - - // Generate an in memory representation of the file, by executing the template - f, err := r.generateFile(filepath.Join(instanceDir, entry.Name()), contentTemplate, entry.Type().Perm()) - if err != nil { - return nil, err + return err } - files[f] = nil + logger.Infof(r.ctx, "added file to in memory file tree: %s", instanceFile) + r.files = append(r.files, instanceFile) } - // Match glob patterns stored, and delete matching in memory files - return nil, deleteSkippedFiles(files, skipPatterns) -} - -// deletes any files in [files] whose name matches a glob pattern in [skipPatterns] -func deleteSkippedFiles(files map[*inMemoryFile]any, skipPatterns []string) error { - for f := range files { - isSkipped := false - for _, pattern := range skipPatterns { - matched, err := filepath.Match(pattern, filepath.Base(f.path)) - if err != nil { - return fmt.Errorf("error while trying to match file %s against glob pattern %s: %w", filepath.Base(f.path), pattern, err) - } - if matched { - isSkipped = true - break - } - } - if isSkipped { - delete(files, f) + // Recursively walk subdirectories, skipping any that match any of the currently + // accumulated skip patterns + for _, d := range directories { + path, err := r.executeTemplate(filepath.Join(dirPath, d.Name())) + if err != nil { + return err } - } - return nil -} - -func materializeFiles(files map[*inMemoryFile]any) error { - for f := range files { - // create any intermediate directories required. Directories are lazily generated - // only when they are required for a file. - err := os.MkdirAll(filepath.Dir(f.path), 0755) + isSkipped, err := r.isSkipped(path) if err != nil { return err } - - // write content to file - err = os.WriteFile(f.path, []byte(f.content), f.perm) + if isSkipped { + logger.Infof(r.ctx, "skipping walking directory: %s", path) + continue + } + err = walk(r, filepath.Join(dirPathTemplate, d.Name())) if err != nil { return err } @@ -200,31 +222,33 @@ func materializeFiles(files map[*inMemoryFile]any) error { return nil } -func walkFileTree(r *renderer, templateRoot, instanceRoot string) error { - return filepath.WalkDir(templateRoot, func(path string, d fs.DirEntry, err error) error { +func (r *renderer) persistToDisk() error { + for _, file := range r.files { + isSkipped, err := r.isSkipped(file.path) if err != nil { return err } - - // skip if current entry is not a directory - if !d.IsDir() { - return nil + if isSkipped { + log.Infof(r.ctx, "skipping file: %s", file.path) + continue } - - // get relative path to the template file, This forms the template for the - // path to the file - relPathTemplate, err := filepath.Rel(templateRoot, path) + err = r.instanceFiler.Write(r.ctx, file.path, strings.NewReader(file.content), filer.CreateParentDirectories) if err != nil { return err } + } + return nil +} - // Compute in memory representation for files in the current directory - files, err := r.generateDir(path, filepath.Join(instanceRoot, relPathTemplate)) +func (r *renderer) isSkipped(path string) (bool, error) { + for _, pattern := range r.skipPatterns { + isMatch, err := filepath.Match(pattern, path) if err != nil { - return err + return false, err } - - // Materialize these files onto the disk - return materializeFiles(files) - }) + if isMatch { + return true, nil + } + } + return false, nil } diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index 036ad96b43c..08ca222717b 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -1,283 +1,287 @@ package template import ( - "io/fs" + "context" "os" "path/filepath" "testing" - "text/template" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestRendererVariableRead(t *testing.T) { - r, err := newRenderer(nil, "./testdata/email/library") - require.NoError(t, err) - tmpDir := t.TempDir() - err = walkFileTree(r, "./testdata/email/template", tmpDir) - require.NoError(t, err) - - b, err := os.ReadFile(filepath.Join(tmpDir, "my_email")) + r, err := newRenderer(context.Background(), nil, "./testdata/email/library", tmpDir, "./testdata/email/template") require.NoError(t, err) - assert.Equal(t, "shreyas.goenka@databricks.com\n", string(b)) -} - -func TestRendererUrlParseUsageInFunction(t *testing.T) { - r, err := newRenderer(nil, "./testdata/get_host/library") + err = walk(r, ".") require.NoError(t, err) - tmpDir := t.TempDir() - - err = walkFileTree(r, "./testdata/get_host/template", tmpDir) + err = r.persistToDisk() require.NoError(t, err) - b, err := os.ReadFile(filepath.Join(tmpDir, "my_host")) - require.NoError(t, err) - - assert.Equal(t, "https://www.host.com\n", string(b)) -} - -func TestRendererRegexpCheckFailing(t *testing.T) { - r, err := newRenderer(nil, "./testdata/is_https/library") - require.NoError(t, err) - - tmpDir := t.TempDir() - - err = walkFileTree(r, "./testdata/is_https/template_not_https", tmpDir) + b, err := os.ReadFile(filepath.Join(tmpDir, "my_email")) require.NoError(t, err) + assert.Equal(t, "shreyas.goenka@databricks.com\n", string(b)) } -func TestRendererRegexpCheckPassing(t *testing.T) { - r, err := newRenderer(nil, "./testdata/is_https/library") - require.NoError(t, err) - +func TestRendererUrlParseUsageInFunction(t *testing.T) { tmpDir := t.TempDir() - err = walkFileTree(r, "./testdata/is_https/template_is_https", tmpDir) + r, err := newRenderer(context.Background(), nil, "./testdata/get_host/library", tmpDir, "./testdata/get_host/template") require.NoError(t, err) - b, err := os.ReadFile(filepath.Join(tmpDir, "my_check")) + err = walk(r, ".") require.NoError(t, err) - assert.Equal(t, "this file is created if validation passes\n", string(b)) -} - -func TestExecuteTemplate(t *testing.T) { - templateText := - `"{{.count}} items are made of {{.Material}}". -{{if eq .Animal "sheep" }} -Sheep wool is the best! -{{else}} -{{.Animal}} wool is not too bad... -{{end}} -My email is {{template "email"}} -` - - r := renderer{ - config: map[string]any{ - "Material": "wool", - "count": 1, - "Animal": "sheep", - }, - baseTemplate: template.Must(template.New("base").Parse(`{{define "email"}}shreyas.goenka@databricks.com{{end}}`)), - } - - statement, err := r.executeTemplate(templateText) + err = r.persistToDisk() require.NoError(t, err) - assert.Contains(t, statement, `"1 items are made of wool"`) - assert.NotContains(t, statement, `cat wool is not too bad.."`) - assert.Contains(t, statement, "Sheep wool is the best!") - assert.Contains(t, statement, `My email is shreyas.goenka@databricks.com`) - - r = renderer{ - config: map[string]any{ - "Material": "wool", - "count": 1, - "Animal": "cat", - }, - baseTemplate: template.Must(template.New("base").Parse(`{{define "email"}}hrithik.roshan@databricks.com{{end}}`)), - } - - statement, err = r.executeTemplate(templateText) - require.NoError(t, err) - assert.Contains(t, statement, `"1 items are made of wool"`) - assert.Contains(t, statement, `cat wool is not too bad...`) - assert.NotContains(t, statement, "Sheep wool is the best!") - assert.Contains(t, statement, `My email is hrithik.roshan@databricks.com`) -} -func TestGenerateFile(t *testing.T) { - tmp := t.TempDir() - - pathTemplate := filepath.Join(tmp, "{{.Animal}}", "{{.Material}}", "foo", "{{.count}}.txt") - contentTemplate := `"{{.count}} items are made of {{.Material}}". - {{if eq .Animal "sheep" }} - Sheep wool is the best! - {{else}} - {{.Animal}} wool is not too bad... - {{end}} - ` - - r := renderer{ - config: map[string]any{ - "Material": "wool", - "count": 1, - "Animal": "cat", - }, - baseTemplate: template.New("base"), - } - f, err := r.generateFile(pathTemplate, contentTemplate, 0444) + b, err := os.ReadFile(filepath.Join(tmpDir, "my_host")) require.NoError(t, err) - // assert file content - assert.Equal(t, "\"1 items are made of wool\".\n\t\n\tcat wool is not too bad...\n\t\n\t", f.content) - - // assert file permissions are correctly assigned - assert.Equal(t, fs.FileMode(0444), f.perm) - - // assert file path - assert.Equal(t, filepath.Join(tmp, "cat", "wool", "foo", "1.txt"), f.path) + assert.Equal(t, "https://www.host.com\n", string(b)) } -func TestDeleteSkippedFiles(t *testing.T) { +func TestRendererRegexpCheckFailing(t *testing.T) { tmpDir := t.TempDir() - inputFiles := map[*inMemoryFile]any{ - { - path: filepath.Join(tmpDir, "aaa"), - content: "one", - perm: 0444, - }: nil, - { - path: filepath.Join(tmpDir, "abb"), - content: "two", - perm: 0444, - }: nil, - { - path: filepath.Join(tmpDir, "bbb"), - content: "three", - perm: 0666, - }: nil, - } - - err := deleteSkippedFiles(inputFiles, []string{"aaa", "abb"}) - require.NoError(t, err) - - assert.Len(t, inputFiles, 1) - for v := range inputFiles { - assert.Equal(t, inMemoryFile{ - path: filepath.Join(tmpDir, "bbb"), - content: "three", - perm: 0666, - }, *v) - } -} -func TestDeleteSkippedFilesWithGlobPatterns(t *testing.T) { - tmpDir := t.TempDir() - inputFiles := map[*inMemoryFile]any{ - { - path: filepath.Join(tmpDir, "aaa"), - content: "one", - perm: 0444, - }: nil, - { - path: filepath.Join(tmpDir, "abb"), - content: "two", - perm: 0444, - }: nil, - { - path: filepath.Join(tmpDir, "bbb"), - content: "three", - perm: 0666, - }: nil, - { - path: filepath.Join(tmpDir, "ddd"), - content: "four", - perm: 0666, - }: nil, - } - - err := deleteSkippedFiles(inputFiles, []string{"a*"}) + r, err := newRenderer(context.Background(), nil, "./testdata/is_https/library", tmpDir, "./testdata/is_https/template_not_https") require.NoError(t, err) - files := make([]inMemoryFile, 0) - for v := range inputFiles { - files = append(files, *v) - } - assert.Len(t, files, 2) - assert.Contains(t, files, inMemoryFile{ - path: filepath.Join(tmpDir, "bbb"), - content: "three", - perm: 0666, - }) - assert.Contains(t, files, inMemoryFile{ - path: filepath.Join(tmpDir, "ddd"), - content: "four", - perm: 0666, - }) + err = walk(r, ".") + assert.Equal(t, "expected /not-a-url to start with https", err.Error()) } -func TestSkipAllFiles(t *testing.T) { - tmpDir := t.TempDir() - inputFiles := map[*inMemoryFile]any{ - { - path: filepath.Join(tmpDir, "aaa"), - content: "one", - perm: 0444, - }: nil, - { - path: filepath.Join(tmpDir, "abb"), - content: "two", - perm: 0444, - }: nil, - { - path: filepath.Join(tmpDir, "bbb"), - content: "three", - perm: 0666, - }: nil, - } - - err := deleteSkippedFiles(inputFiles, []string{"*"}) - require.NoError(t, err) - assert.Len(t, inputFiles, 0) -} - -func TestTemplateMaterializeFiles(t *testing.T) { - tmpDir := t.TempDir() - inputFiles := map[*inMemoryFile]any{ - { - path: filepath.Join(tmpDir, "aaa"), - content: "one", - perm: 0444, - }: nil, - { - path: filepath.Join(tmpDir, "abb"), - content: "two", - perm: 0444, - }: nil, - { - path: filepath.Join(tmpDir, "bbb"), - content: "three", - perm: 0666, - }: nil, - } - - err := materializeFiles(inputFiles) - assert.NoError(t, err) - - path := filepath.Join(tmpDir, "aaa") - assertFilePerm(t, path, 0444) - assertFileContent(t, path, "one") - - path = filepath.Join(tmpDir, "abb") - assertFilePerm(t, path, 0444) - assertFileContent(t, path, "two") - - path = filepath.Join(tmpDir, "bbb") - assertFilePerm(t, path, 0666) - assertFileContent(t, path, "three") -} +// func TestRendererRegexpCheckPassing(t *testing.T) { +// r, err := newRenderer(nil, "./testdata/is_https/library") +// require.NoError(t, err) + +// tmpDir := t.TempDir() + +// err = walkFileTree(r, "./testdata/is_https/template_is_https", tmpDir) +// require.NoError(t, err) + +// b, err := os.ReadFile(filepath.Join(tmpDir, "my_check")) +// require.NoError(t, err) + +// assert.Equal(t, "this file is created if validation passes\n", string(b)) +// } + +// func TestExecuteTemplate(t *testing.T) { +// templateText := +// `"{{.count}} items are made of {{.Material}}". +// {{if eq .Animal "sheep" }} +// Sheep wool is the best! +// {{else}} +// {{.Animal}} wool is not too bad... +// {{end}} +// My email is {{template "email"}} +// ` + +// r := renderer{ +// config: map[string]any{ +// "Material": "wool", +// "count": 1, +// "Animal": "sheep", +// }, +// baseTemplate: template.Must(template.New("base").Parse(`{{define "email"}}shreyas.goenka@databricks.com{{end}}`)), +// } + +// statement, err := r.executeTemplate(templateText) +// require.NoError(t, err) +// assert.Contains(t, statement, `"1 items are made of wool"`) +// assert.NotContains(t, statement, `cat wool is not too bad.."`) +// assert.Contains(t, statement, "Sheep wool is the best!") +// assert.Contains(t, statement, `My email is shreyas.goenka@databricks.com`) + +// r = renderer{ +// config: map[string]any{ +// "Material": "wool", +// "count": 1, +// "Animal": "cat", +// }, +// baseTemplate: template.Must(template.New("base").Parse(`{{define "email"}}hrithik.roshan@databricks.com{{end}}`)), +// } + +// statement, err = r.executeTemplate(templateText) +// require.NoError(t, err) +// assert.Contains(t, statement, `"1 items are made of wool"`) +// assert.Contains(t, statement, `cat wool is not too bad...`) +// assert.NotContains(t, statement, "Sheep wool is the best!") +// assert.Contains(t, statement, `My email is hrithik.roshan@databricks.com`) +// } + +// func TestGenerateFile(t *testing.T) { +// tmp := t.TempDir() + +// pathTemplate := filepath.Join(tmp, "{{.Animal}}", "{{.Material}}", "foo", "{{.count}}.txt") +// contentTemplate := `"{{.count}} items are made of {{.Material}}". +// {{if eq .Animal "sheep" }} +// Sheep wool is the best! +// {{else}} +// {{.Animal}} wool is not too bad... +// {{end}} +// ` + +// r := renderer{ +// config: map[string]any{ +// "Material": "wool", +// "count": 1, +// "Animal": "cat", +// }, +// baseTemplate: template.New("base"), +// } +// f, err := r.generateFile(pathTemplate, contentTemplate, 0444) +// require.NoError(t, err) + +// // assert file content +// assert.Equal(t, "\"1 items are made of wool\".\n\t\n\tcat wool is not too bad...\n\t\n\t", f.content) + +// // assert file permissions are correctly assigned +// assert.Equal(t, fs.FileMode(0444), f.perm) + +// // assert file path +// assert.Equal(t, filepath.Join(tmp, "cat", "wool", "foo", "1.txt"), f.relPath) +// } + +// func TestDeleteSkippedFiles(t *testing.T) { +// tmpDir := t.TempDir() +// inputFiles := map[*inMemoryFile]any{ +// { +// relPath: filepath.Join(tmpDir, "aaa"), +// content: "one", +// perm: 0444, +// }: nil, +// { +// relPath: filepath.Join(tmpDir, "abb"), +// content: "two", +// perm: 0444, +// }: nil, +// { +// relPath: filepath.Join(tmpDir, "bbb"), +// content: "three", +// perm: 0666, +// }: nil, +// } + +// err := deleteSkippedFiles(inputFiles, []string{"aaa", "abb"}) +// require.NoError(t, err) + +// assert.Len(t, inputFiles, 1) +// for v := range inputFiles { +// assert.Equal(t, inMemoryFile{ +// relPath: filepath.Join(tmpDir, "bbb"), +// content: "three", +// perm: 0666, +// }, *v) +// } +// } + +// func TestDeleteSkippedFilesWithGlobPatterns(t *testing.T) { +// tmpDir := t.TempDir() +// inputFiles := map[*inMemoryFile]any{ +// { +// relPath: filepath.Join(tmpDir, "aaa"), +// content: "one", +// perm: 0444, +// }: nil, +// { +// relPath: filepath.Join(tmpDir, "abb"), +// content: "two", +// perm: 0444, +// }: nil, +// { +// relPath: filepath.Join(tmpDir, "bbb"), +// content: "three", +// perm: 0666, +// }: nil, +// { +// relPath: filepath.Join(tmpDir, "ddd"), +// content: "four", +// perm: 0666, +// }: nil, +// } + +// err := deleteSkippedFiles(inputFiles, []string{"a*"}) +// require.NoError(t, err) + +// files := make([]inMemoryFile, 0) +// for v := range inputFiles { +// files = append(files, *v) +// } +// assert.Len(t, files, 2) +// assert.Contains(t, files, inMemoryFile{ +// relPath: filepath.Join(tmpDir, "bbb"), +// content: "three", +// perm: 0666, +// }) +// assert.Contains(t, files, inMemoryFile{ +// relPath: filepath.Join(tmpDir, "ddd"), +// content: "four", +// perm: 0666, +// }) +// } + +// func TestSkipAllFiles(t *testing.T) { +// tmpDir := t.TempDir() +// inputFiles := map[*inMemoryFile]any{ +// { +// relPath: filepath.Join(tmpDir, "aaa"), +// content: "one", +// perm: 0444, +// }: nil, +// { +// relPath: filepath.Join(tmpDir, "abb"), +// content: "two", +// perm: 0444, +// }: nil, +// { +// relPath: filepath.Join(tmpDir, "bbb"), +// content: "three", +// perm: 0666, +// }: nil, +// } + +// err := deleteSkippedFiles(inputFiles, []string{"*"}) +// require.NoError(t, err) +// assert.Len(t, inputFiles, 0) +// } + +// func TestTemplateMaterializeFiles(t *testing.T) { +// tmpDir := t.TempDir() +// inputFiles := map[*inMemoryFile]any{ +// { +// relPath: filepath.Join(tmpDir, "aaa"), +// content: "one", +// perm: 0444, +// }: nil, +// { +// relPath: filepath.Join(tmpDir, "abb"), +// content: "two", +// perm: 0444, +// }: nil, +// { +// relPath: filepath.Join(tmpDir, "bbb"), +// content: "three", +// perm: 0666, +// }: nil, +// } + +// err := materializeFiles(inputFiles) +// assert.NoError(t, err) + +// path := filepath.Join(tmpDir, "aaa") +// assertFilePerm(t, path, 0444) +// assertFileContent(t, path, "one") + +// path = filepath.Join(tmpDir, "abb") +// assertFilePerm(t, path, 0444) +// assertFileContent(t, path, "two") + +// path = filepath.Join(tmpDir, "bbb") +// assertFilePerm(t, path, 0666) +// assertFileContent(t, path, "three") +// } From 1d46eefacb17097893ba61c2edc514047e845834 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 23 Jul 2023 23:42:48 +0200 Subject: [PATCH 52/60] delete unnecessary files --- internal/init_test.go | 103 ------- libs/template/helpers.go | 28 -- libs/template/materialize.go | 32 -- libs/template/materialize_test.go | 192 ------------ libs/template/renderer.go | 254 ---------------- libs/template/renderer_test.go | 287 ------------------ .../testdata/email/library/email.tmpl | 1 - .../template/testdata/email/template/my_email | 1 - .../get_host/library/get_host_func.tmpl | 1 - .../testdata/get_host/template/my_host | 1 - .../is_https/library/is_https_check.tmpl | 6 - .../is_https/template_is_https/my_check | 2 - .../is_https/template_not_https/my_check | 1 - .../skip_dir/databricks_template_schema.json | 16 - .../skip_dir/template/{{.a}}/.gitkeep | 0 .../testdata/skip_dir/template/{{.c}}/abc | 4 - 16 files changed, 929 deletions(-) delete mode 100644 internal/init_test.go delete mode 100644 libs/template/helpers.go delete mode 100644 libs/template/materialize.go delete mode 100644 libs/template/materialize_test.go delete mode 100644 libs/template/renderer.go delete mode 100644 libs/template/renderer_test.go delete mode 100644 libs/template/testdata/email/library/email.tmpl delete mode 100644 libs/template/testdata/email/template/my_email delete mode 100644 libs/template/testdata/get_host/library/get_host_func.tmpl delete mode 100644 libs/template/testdata/get_host/template/my_host delete mode 100644 libs/template/testdata/is_https/library/is_https_check.tmpl delete mode 100644 libs/template/testdata/is_https/template_is_https/my_check delete mode 100644 libs/template/testdata/is_https/template_not_https/my_check delete mode 100644 libs/template/testdata/skip_dir/databricks_template_schema.json delete mode 100644 libs/template/testdata/skip_dir/template/{{.a}}/.gitkeep delete mode 100644 libs/template/testdata/skip_dir/template/{{.c}}/abc diff --git a/internal/init_test.go b/internal/init_test.go deleted file mode 100644 index e6ec1dff740..00000000000 --- a/internal/init_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package internal - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - _ "github.com/databricks/cli/cmd/bundle" - "github.com/databricks/cli/cmd/root" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func assertFileContains(t *testing.T, path string, substr string) { - b, err := os.ReadFile(path) - require.NoError(t, err) - assert.Contains(t, string(b), substr) -} - -func assertFileNotContains(t *testing.T, path string, substr string) { - b, err := os.ReadFile(path) - require.NoError(t, err) - assert.NotContains(t, string(b), substr) -} - -func TestAccTemplateInitializationForDevConfig(t *testing.T) { - // create target directory with the input config - tmp := t.TempDir() - f, err := os.Create(filepath.Join(tmp, "config.json")) - require.NoError(t, err) - _, err = f.WriteString(` - { - "project_name": "development_project", - "cloud_type": "AWS", - "ci_type": "github", - "is_production": false - } - `) - f.Close() - require.NoError(t, err) - - // materialize the template - cmd := root.RootCmd - cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--output-dir", tmp, "--config-file", filepath.Join(tmp, "config.json")}) - err = cmd.Execute() - require.NoError(t, err) - - // assert on materialized template - assert.FileExists(t, filepath.Join(tmp, "development_project", "aws_file")) - assert.FileExists(t, filepath.Join(tmp, "development_project", ".github")) - assert.NoFileExists(t, filepath.Join(tmp, "development_project", "azure_file")) - assertFileContains(t, filepath.Join(tmp, "development_project", "aws_file"), "This file should only be generated for AWS") - assertFileContains(t, filepath.Join(tmp, "development_project", ".github"), "This is a development project") -} - -func TestAccTemplateInitializationForProdConfig(t *testing.T) { - // create target directory with the input config - tmp := t.TempDir() - - // create target directory to with the input config - configDir := filepath.Join(tmp, "dir-with-config") - err := os.Mkdir(configDir, os.ModePerm) - require.NoError(t, err) - f, err := os.Create(filepath.Join(configDir, "my_config.json")) - require.NoError(t, err) - _, err = f.WriteString(` - { - "project_name": "production_project", - "cloud_type": "Azure", - "ci_type": "azure_devops", - "is_production": true - } - `) - f.Close() - require.NoError(t, err) - - // create directory to initialize the template instance within - instanceDir := filepath.Join(tmp, "dir-with-instance") - err = os.Mkdir(instanceDir, os.ModePerm) - require.NoError(t, err) - - // materialize the template - cmd := root.RootCmd - childCommands := cmd.Commands() - fmt.Println(childCommands) - cmd.SetArgs([]string{"bundle", "init", filepath.FromSlash("testdata/init/templateDefinition"), "--output-dir", instanceDir, "--config-file", filepath.Join(configDir, "my_config.json")}) - err = cmd.Execute() - require.NoError(t, err) - - // assert on materialized template files - assert.FileExists(t, filepath.Join(instanceDir, "production_project", "azure_file")) - assert.FileExists(t, filepath.Join(instanceDir, "production_project", ".azure_devops")) - assert.NoFileExists(t, filepath.Join(instanceDir, "production_project", "aws_file")) - assertFileContains(t, filepath.Join(instanceDir, "production_project", ".azure_devops"), "This is a production project") - - // assert azure_file is computed correctly - azureFilePath := filepath.Join(instanceDir, "production_project", "azure_file") - assertFileContains(t, azureFilePath, "This file should only be generated for Azure") - assertFileContains(t, azureFilePath, "shreyas.goenka@databricks.com") - assertFileContains(t, azureFilePath, "https://adb-xxxx.xx.azuredatabricks.net") - assertFileNotContains(t, azureFilePath, "https://adb-xxxx.xx.azuredatabricks.net/sql/queries") -} diff --git a/libs/template/helpers.go b/libs/template/helpers.go deleted file mode 100644 index 2f206d2b5d5..00000000000 --- a/libs/template/helpers.go +++ /dev/null @@ -1,28 +0,0 @@ -package template - -import ( - "fmt" - "net/url" - "regexp" - "text/template" -) - -type ErrFail struct { - msg string -} - -func (err ErrFail) Error() string { - return err.msg -} - -var HelperFuncs = template.FuncMap{ - "urlParse": func(rawUrl string) (*url.URL, error) { - return url.Parse(rawUrl) - }, - "regexpCompile": func(expr string) (*regexp.Regexp, error) { - return regexp.Compile(expr) - }, - "fail": func(format string, args ...any) (any, error) { - return nil, ErrFail{fmt.Sprintf(format, args...)} - }, -} diff --git a/libs/template/materialize.go b/libs/template/materialize.go deleted file mode 100644 index b4b1b0e5128..00000000000 --- a/libs/template/materialize.go +++ /dev/null @@ -1,32 +0,0 @@ -package template - -import ( - "context" - "path/filepath" -) - -const schemaFileName = "databricks_template_schema.json" -const templateDirName = "template" -const libraryDirName = "library" - -func Materialize(templateRoot, instanceRoot, configPath string) error { - // read the file containing schema for template input parameters - schema, err := ReadSchema(filepath.Join(templateRoot, schemaFileName)) - if err != nil { - return err - } - - // read user config to initialize the template with - config, err := schema.ReadConfig(configPath) - if err != nil { - return err - } - - r, err := newRenderer(context.TODO(), config, filepath.Join(templateRoot, libraryDirName), instanceRoot, filepath.Join(templateRoot, templateDirName)) - if err != nil { - return err - } - - // materialize the template - return walk(r, ".") -} diff --git a/libs/template/materialize_test.go b/libs/template/materialize_test.go deleted file mode 100644 index 2bbc0295597..00000000000 --- a/libs/template/materialize_test.go +++ /dev/null @@ -1,192 +0,0 @@ -package template - -import ( - "io/fs" - "os" - "path/filepath" - "runtime" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func setupConfig(t *testing.T, config string) string { - // create target directory with the input config - tmp := t.TempDir() - f, err := os.Create(filepath.Join(tmp, "config.json")) - require.NoError(t, err) - _, err = f.WriteString(config) - f.Close() - require.NoError(t, err) - return tmp -} - -func assertFilePerm(t *testing.T, path string, perm fs.FileMode) { - stat, err := os.Stat(path) - require.NoError(t, err) - assert.Equal(t, perm, stat.Mode().Perm()) -} - -func assertFileContent(t *testing.T, path string, content string) { - b, err := os.ReadFile(path) - require.NoError(t, err) - assert.Equal(t, content, string(b)) -} - -func TestMaterializeEmptyDirsAreNotGenerated(t *testing.T) { - tmp := setupConfig(t, ` - { - "a": "this directory is created because it contains a file", - "b": "this variable is not used anywhere", - "c": "this directory will be skipped if d=foo", - "d": "foo" - }`) - err := Materialize("./testdata/skip_dir", tmp, filepath.Join(tmp, "config.json")) - require.NoError(t, err) - - assert.DirExists(t, filepath.Join(tmp, "this directory is created because it contains a file")) - assert.FileExists(t, filepath.Join(tmp, "this directory is created because it contains a file/.gitkeep")) - assert.NoDirExists(t, filepath.Join(tmp, "this directory will be skipped if d=foo")) - - tmp2 := setupConfig(t, ` - { - "a": "this directory is created because it contains a file", - "b": "this variable is not used anywhere", - "c": "this directory will be skipped if d=foo", - "d": "bar" - }`) - err = Materialize("./testdata/skip_dir", tmp2, filepath.Join(tmp2, "config.json")) - require.NoError(t, err) - - assert.DirExists(t, filepath.Join(tmp2, "this directory is created because it contains a file")) - assert.FileExists(t, filepath.Join(tmp2, "this directory is created because it contains a file/.gitkeep")) - assert.DirExists(t, filepath.Join(tmp2, "this directory will be skipped if d=foo")) - assert.FileExists(t, filepath.Join(tmp2, "this directory will be skipped if d=foo/abc")) -} - -func TestMaterializeFilePermissionsAreCopiedForUnix(t *testing.T) { - if runtime.GOOS == "windows" { - t.SkipNow() - } - - tmp := t.TempDir() - - // setup template in temp directory - err := os.Mkdir(filepath.Join(tmp, "my_tmpl"), 0777) - require.NoError(t, err) - err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "databricks_template_schema.json"), []byte(` - { - "properties": { - "a": { - "type": "string" - }, - "b": { - "type": "string" - } - } - }`), 0644) - require.NoError(t, err) - err = os.Mkdir(filepath.Join(tmp, "my_tmpl", "template"), 0777) - require.NoError(t, err) - - // A normal file with the executable bit not flipped - err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "{{.a}}"), []byte("abc"), 0600) - require.NoError(t, err) - - // A read only file - err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "{{.b}}"), []byte("def"), 0400) - require.NoError(t, err) - - // A read only executable file - err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "foo"), []byte("ghi"), 0500) - require.NoError(t, err) - - // An executable script, accessable by non user access classes - err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "bar"), []byte("ghi"), 0755) - require.NoError(t, err) - - // create config.json file - err = os.Mkdir(filepath.Join(tmp, "config"), 0777) - require.NoError(t, err) - configPath := filepath.Join(tmp, "config", "config.json") - err = os.WriteFile(configPath, []byte(` - { - "a": "Amsterdam", - "b": "Hague" - }`), 0644) - require.NoError(t, err) - - // create directory to initialize the template in - instanceRoot := filepath.Join(tmp, "instance") - err = os.Mkdir(instanceRoot, 0777) - require.NoError(t, err) - - // materialize the template - err = Materialize(filepath.Join(tmp, "my_tmpl"), instanceRoot, configPath) - require.NoError(t, err) - - // assert template files have the correct permission bits set - assertFilePerm(t, filepath.Join(instanceRoot, "Amsterdam"), 0600) - assertFilePerm(t, filepath.Join(instanceRoot, "Hague"), 0400) - assertFilePerm(t, filepath.Join(instanceRoot, "foo"), 0500) - assertFilePerm(t, filepath.Join(instanceRoot, "bar"), 0755) -} - -func TestMaterializeFilePermissionsAreCopiedForWindows(t *testing.T) { - if runtime.GOOS != "windows" { - t.SkipNow() - } - - tmp := t.TempDir() - - // create template in temp directory - err := os.Mkdir(filepath.Join(tmp, "my_tmpl"), 0777) - require.NoError(t, err) - err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "databricks_template_schema.json"), []byte(` - { - "properties": { - "a": { - "type": "string" - }, - "b": { - "type": "string" - } - } - }`), 0644) - require.NoError(t, err) - - // A normal file with the executable bit not flipped - err = os.Mkdir(filepath.Join(tmp, "my_tmpl", "template"), 0777) - require.NoError(t, err) - err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "{{.a}}"), []byte("abc"), 0666) - require.NoError(t, err) - - // A read only file - err = os.WriteFile(filepath.Join(tmp, "my_tmpl", "template", "{{.b}}"), []byte("def"), 0444) - require.NoError(t, err) - - // create config.json file - err = os.Mkdir(filepath.Join(tmp, "config"), 0777) - require.NoError(t, err) - configPath := filepath.Join(tmp, "config", "config.json") - err = os.WriteFile(configPath, []byte(` - { - "a": "Amsterdam", - "b": "Hague" - }`), 0644) - require.NoError(t, err) - - // create directory to initialize the template in - instanceRoot := filepath.Join(tmp, "instance") - err = os.Mkdir(instanceRoot, 0777) - require.NoError(t, err) - - // materialize the template - err = Materialize(filepath.Join(tmp, "my_tmpl"), instanceRoot, configPath) - require.NoError(t, err) - - // assert template files have the correct permission bits set - assertFilePerm(t, filepath.Join(instanceRoot, "Amsterdam"), 0666) - assertFilePerm(t, filepath.Join(instanceRoot, "Hague"), 0444) -} diff --git a/libs/template/renderer.go b/libs/template/renderer.go deleted file mode 100644 index 0febc005e9b..00000000000 --- a/libs/template/renderer.go +++ /dev/null @@ -1,254 +0,0 @@ -package template - -import ( - "context" - "errors" - "fmt" - "io" - "io/fs" - "path/filepath" - "strings" - "text/template" - - "github.com/databricks/cli/libs/filer" - "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go/logger" - "golang.org/x/exp/slices" -) - -type inMemoryFile struct { - path string - // TODO: use bytes in to serialize for binary files, Can we just use string here, is it the same - content string - perm fs.FileMode -} - -// This structure renders any template files during project initialization -type renderer struct { - ctx context.Context - - // A config that is the "dot" value available to any template being rendered. - // Refer to https://pkg.go.dev/text/template for how templates can use - // this "dot" value - config map[string]any - - // A base template with helper functions and user defined template in the - // library directory loaded. This is used as the base to compute any project - // templates during file tree walk - baseTemplate *template.Template - - files []*inMemoryFile - skipPatterns []string - - templateFiler filer.Filer - instanceFiler filer.Filer -} - -func newRenderer(ctx context.Context, config map[string]any, libraryRoot, instanceRoot, templateRoot string) (*renderer, error) { - // All user defined functions will be available inside library root - libraryGlob := filepath.Join(libraryRoot, "*") - - // Initialize new template, with helper functions loaded - tmpl := template.New("").Funcs(HelperFuncs) - - // Load files in the library to the template - matches, err := filepath.Glob(libraryGlob) - if err != nil { - return nil, err - } - if len(matches) != 0 { - tmpl, err = tmpl.ParseGlob(libraryGlob) - if err != nil { - return nil, err - } - } - - // create template filer - templateFiler, err := filer.NewLocalClient(templateRoot) - if err != nil { - return nil, err - } - - instanceFiler, err := filer.NewLocalClient(instanceRoot) - if err != nil { - return nil, err - } - - ctx = log.NewContext(ctx, log.GetLogger(ctx).With("action", "initialize-template")) - - return &renderer{ - config: config, - baseTemplate: tmpl, - files: make([]*inMemoryFile, 0), - templateFiler: templateFiler, - ctx: ctx, - skipPatterns: make([]string, 0), - instanceFiler: instanceFiler, - }, nil -} - -// Executes the template by applying config on it. Returns the materialized template -// as a string -func (r *renderer) executeTemplate(templateDefinition string) (string, error) { - // Create copy of base template so as to not overwrite it - tmpl, err := r.baseTemplate.Clone() - if err != nil { - return "", err - } - - // Parse the template text - tmpl, err = tmpl.Parse(templateDefinition) - if err != nil { - return "", err - } - - // Execute template and get result - result := strings.Builder{} - err = tmpl.Execute(&result, r.config) - if err != nil { - return "", err - } - return result.String(), nil -} - -func (r *renderer) computeFile(relPathTemplate string) (*inMemoryFile, error) { - // read template file contents - templateReader, err := r.templateFiler.Read(r.ctx, relPathTemplate) - if err != nil { - return nil, err - } - contentTemplate, err := io.ReadAll(templateReader) - if err != nil { - return nil, err - } - - // execute the contents of the file as a template - content, err := r.executeTemplate(string(contentTemplate)) - // Capture errors caused by the "fail" helper function - if target := (&ErrFail{}); errors.As(err, target) { - return nil, target - } - if err != nil { - return nil, fmt.Errorf("failed to compute file content for %s. %w", relPathTemplate, err) - } - - // Execute relative path template to get materialized path for the file - relPath, err := r.executeTemplate(relPathTemplate) - if err != nil { - return nil, err - } - - // Read permissions for the file - info, err := r.templateFiler.Stat(r.ctx, relPathTemplate) - if err != nil { - return nil, err - } - perm := info.Mode().Perm() - - return &inMemoryFile{ - path: relPath, - content: content, - perm: perm, - }, nil -} - -func walk(r *renderer, dirPathTemplate string) error { - entries, err := r.templateFiler.ReadDir(r.ctx, dirPathTemplate) - if err != nil { - return err - } - - // Separate files and directories from entries. We would like to process - // all the files first to capture all skip glob patterns. - files := make([]fs.DirEntry, 0) - directories := make([]fs.DirEntry, 0) - for _, entry := range entries { - if entry.IsDir() { - directories = append(directories, entry) - } else { - files = append(files, entry) - } - } - - dirPath, err := r.executeTemplate(dirPathTemplate) - if err != nil { - return err - } - - // Add skip functional closure - r.baseTemplate.Funcs(template.FuncMap{ - "skip": func(relPattern string) error { - // patterns are specified relative to current directory of the file - // {{skip}} function is called from - pattern := filepath.Join(dirPath, relPattern) - if !slices.Contains(r.skipPatterns, pattern) { - logger.Infof(r.ctx, "adding skip pattern: %s", pattern) - r.skipPatterns = append(r.skipPatterns, pattern) - } - return nil - }, - }) - - // Compute files in current directory, and add them to file tree - for _, f := range files { - instanceFile, err := r.computeFile(filepath.Join(dirPathTemplate, f.Name())) - if err != nil { - return err - } - logger.Infof(r.ctx, "added file to in memory file tree: %s", instanceFile) - r.files = append(r.files, instanceFile) - } - - // Recursively walk subdirectories, skipping any that match any of the currently - // accumulated skip patterns - for _, d := range directories { - path, err := r.executeTemplate(filepath.Join(dirPath, d.Name())) - if err != nil { - return err - } - isSkipped, err := r.isSkipped(path) - if err != nil { - return err - } - if isSkipped { - logger.Infof(r.ctx, "skipping walking directory: %s", path) - continue - } - err = walk(r, filepath.Join(dirPathTemplate, d.Name())) - if err != nil { - return err - } - } - return nil -} - -func (r *renderer) persistToDisk() error { - for _, file := range r.files { - isSkipped, err := r.isSkipped(file.path) - if err != nil { - return err - } - if isSkipped { - log.Infof(r.ctx, "skipping file: %s", file.path) - continue - } - err = r.instanceFiler.Write(r.ctx, file.path, strings.NewReader(file.content), filer.CreateParentDirectories) - if err != nil { - return err - } - } - return nil -} - -func (r *renderer) isSkipped(path string) (bool, error) { - for _, pattern := range r.skipPatterns { - isMatch, err := filepath.Match(pattern, path) - if err != nil { - return false, err - } - if isMatch { - return true, nil - } - } - return false, nil -} diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go deleted file mode 100644 index 08ca222717b..00000000000 --- a/libs/template/renderer_test.go +++ /dev/null @@ -1,287 +0,0 @@ -package template - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRendererVariableRead(t *testing.T) { - tmpDir := t.TempDir() - - r, err := newRenderer(context.Background(), nil, "./testdata/email/library", tmpDir, "./testdata/email/template") - require.NoError(t, err) - - err = walk(r, ".") - require.NoError(t, err) - - err = r.persistToDisk() - require.NoError(t, err) - - b, err := os.ReadFile(filepath.Join(tmpDir, "my_email")) - require.NoError(t, err) - assert.Equal(t, "shreyas.goenka@databricks.com\n", string(b)) -} - -func TestRendererUrlParseUsageInFunction(t *testing.T) { - tmpDir := t.TempDir() - - r, err := newRenderer(context.Background(), nil, "./testdata/get_host/library", tmpDir, "./testdata/get_host/template") - require.NoError(t, err) - - err = walk(r, ".") - require.NoError(t, err) - - err = r.persistToDisk() - require.NoError(t, err) - - b, err := os.ReadFile(filepath.Join(tmpDir, "my_host")) - require.NoError(t, err) - - assert.Equal(t, "https://www.host.com\n", string(b)) -} - -func TestRendererRegexpCheckFailing(t *testing.T) { - tmpDir := t.TempDir() - - r, err := newRenderer(context.Background(), nil, "./testdata/is_https/library", tmpDir, "./testdata/is_https/template_not_https") - require.NoError(t, err) - - err = walk(r, ".") - assert.Equal(t, "expected /not-a-url to start with https", err.Error()) -} - -// func TestRendererRegexpCheckPassing(t *testing.T) { -// r, err := newRenderer(nil, "./testdata/is_https/library") -// require.NoError(t, err) - -// tmpDir := t.TempDir() - -// err = walkFileTree(r, "./testdata/is_https/template_is_https", tmpDir) -// require.NoError(t, err) - -// b, err := os.ReadFile(filepath.Join(tmpDir, "my_check")) -// require.NoError(t, err) - -// assert.Equal(t, "this file is created if validation passes\n", string(b)) -// } - -// func TestExecuteTemplate(t *testing.T) { -// templateText := -// `"{{.count}} items are made of {{.Material}}". -// {{if eq .Animal "sheep" }} -// Sheep wool is the best! -// {{else}} -// {{.Animal}} wool is not too bad... -// {{end}} -// My email is {{template "email"}} -// ` - -// r := renderer{ -// config: map[string]any{ -// "Material": "wool", -// "count": 1, -// "Animal": "sheep", -// }, -// baseTemplate: template.Must(template.New("base").Parse(`{{define "email"}}shreyas.goenka@databricks.com{{end}}`)), -// } - -// statement, err := r.executeTemplate(templateText) -// require.NoError(t, err) -// assert.Contains(t, statement, `"1 items are made of wool"`) -// assert.NotContains(t, statement, `cat wool is not too bad.."`) -// assert.Contains(t, statement, "Sheep wool is the best!") -// assert.Contains(t, statement, `My email is shreyas.goenka@databricks.com`) - -// r = renderer{ -// config: map[string]any{ -// "Material": "wool", -// "count": 1, -// "Animal": "cat", -// }, -// baseTemplate: template.Must(template.New("base").Parse(`{{define "email"}}hrithik.roshan@databricks.com{{end}}`)), -// } - -// statement, err = r.executeTemplate(templateText) -// require.NoError(t, err) -// assert.Contains(t, statement, `"1 items are made of wool"`) -// assert.Contains(t, statement, `cat wool is not too bad...`) -// assert.NotContains(t, statement, "Sheep wool is the best!") -// assert.Contains(t, statement, `My email is hrithik.roshan@databricks.com`) -// } - -// func TestGenerateFile(t *testing.T) { -// tmp := t.TempDir() - -// pathTemplate := filepath.Join(tmp, "{{.Animal}}", "{{.Material}}", "foo", "{{.count}}.txt") -// contentTemplate := `"{{.count}} items are made of {{.Material}}". -// {{if eq .Animal "sheep" }} -// Sheep wool is the best! -// {{else}} -// {{.Animal}} wool is not too bad... -// {{end}} -// ` - -// r := renderer{ -// config: map[string]any{ -// "Material": "wool", -// "count": 1, -// "Animal": "cat", -// }, -// baseTemplate: template.New("base"), -// } -// f, err := r.generateFile(pathTemplate, contentTemplate, 0444) -// require.NoError(t, err) - -// // assert file content -// assert.Equal(t, "\"1 items are made of wool\".\n\t\n\tcat wool is not too bad...\n\t\n\t", f.content) - -// // assert file permissions are correctly assigned -// assert.Equal(t, fs.FileMode(0444), f.perm) - -// // assert file path -// assert.Equal(t, filepath.Join(tmp, "cat", "wool", "foo", "1.txt"), f.relPath) -// } - -// func TestDeleteSkippedFiles(t *testing.T) { -// tmpDir := t.TempDir() -// inputFiles := map[*inMemoryFile]any{ -// { -// relPath: filepath.Join(tmpDir, "aaa"), -// content: "one", -// perm: 0444, -// }: nil, -// { -// relPath: filepath.Join(tmpDir, "abb"), -// content: "two", -// perm: 0444, -// }: nil, -// { -// relPath: filepath.Join(tmpDir, "bbb"), -// content: "three", -// perm: 0666, -// }: nil, -// } - -// err := deleteSkippedFiles(inputFiles, []string{"aaa", "abb"}) -// require.NoError(t, err) - -// assert.Len(t, inputFiles, 1) -// for v := range inputFiles { -// assert.Equal(t, inMemoryFile{ -// relPath: filepath.Join(tmpDir, "bbb"), -// content: "three", -// perm: 0666, -// }, *v) -// } -// } - -// func TestDeleteSkippedFilesWithGlobPatterns(t *testing.T) { -// tmpDir := t.TempDir() -// inputFiles := map[*inMemoryFile]any{ -// { -// relPath: filepath.Join(tmpDir, "aaa"), -// content: "one", -// perm: 0444, -// }: nil, -// { -// relPath: filepath.Join(tmpDir, "abb"), -// content: "two", -// perm: 0444, -// }: nil, -// { -// relPath: filepath.Join(tmpDir, "bbb"), -// content: "three", -// perm: 0666, -// }: nil, -// { -// relPath: filepath.Join(tmpDir, "ddd"), -// content: "four", -// perm: 0666, -// }: nil, -// } - -// err := deleteSkippedFiles(inputFiles, []string{"a*"}) -// require.NoError(t, err) - -// files := make([]inMemoryFile, 0) -// for v := range inputFiles { -// files = append(files, *v) -// } -// assert.Len(t, files, 2) -// assert.Contains(t, files, inMemoryFile{ -// relPath: filepath.Join(tmpDir, "bbb"), -// content: "three", -// perm: 0666, -// }) -// assert.Contains(t, files, inMemoryFile{ -// relPath: filepath.Join(tmpDir, "ddd"), -// content: "four", -// perm: 0666, -// }) -// } - -// func TestSkipAllFiles(t *testing.T) { -// tmpDir := t.TempDir() -// inputFiles := map[*inMemoryFile]any{ -// { -// relPath: filepath.Join(tmpDir, "aaa"), -// content: "one", -// perm: 0444, -// }: nil, -// { -// relPath: filepath.Join(tmpDir, "abb"), -// content: "two", -// perm: 0444, -// }: nil, -// { -// relPath: filepath.Join(tmpDir, "bbb"), -// content: "three", -// perm: 0666, -// }: nil, -// } - -// err := deleteSkippedFiles(inputFiles, []string{"*"}) -// require.NoError(t, err) -// assert.Len(t, inputFiles, 0) -// } - -// func TestTemplateMaterializeFiles(t *testing.T) { -// tmpDir := t.TempDir() -// inputFiles := map[*inMemoryFile]any{ -// { -// relPath: filepath.Join(tmpDir, "aaa"), -// content: "one", -// perm: 0444, -// }: nil, -// { -// relPath: filepath.Join(tmpDir, "abb"), -// content: "two", -// perm: 0444, -// }: nil, -// { -// relPath: filepath.Join(tmpDir, "bbb"), -// content: "three", -// perm: 0666, -// }: nil, -// } - -// err := materializeFiles(inputFiles) -// assert.NoError(t, err) - -// path := filepath.Join(tmpDir, "aaa") -// assertFilePerm(t, path, 0444) -// assertFileContent(t, path, "one") - -// path = filepath.Join(tmpDir, "abb") -// assertFilePerm(t, path, 0444) -// assertFileContent(t, path, "two") - -// path = filepath.Join(tmpDir, "bbb") -// assertFilePerm(t, path, 0666) -// assertFileContent(t, path, "three") -// } diff --git a/libs/template/testdata/email/library/email.tmpl b/libs/template/testdata/email/library/email.tmpl deleted file mode 100644 index 1897d46b3ed..00000000000 --- a/libs/template/testdata/email/library/email.tmpl +++ /dev/null @@ -1 +0,0 @@ -{{define "email"}}shreyas.goenka@databricks.com{{end}} diff --git a/libs/template/testdata/email/template/my_email b/libs/template/testdata/email/template/my_email deleted file mode 100644 index 0b74ef47cd4..00000000000 --- a/libs/template/testdata/email/template/my_email +++ /dev/null @@ -1 +0,0 @@ -{{template "email"}} diff --git a/libs/template/testdata/get_host/library/get_host_func.tmpl b/libs/template/testdata/get_host/library/get_host_func.tmpl deleted file mode 100644 index e88cd23acf8..00000000000 --- a/libs/template/testdata/get_host/library/get_host_func.tmpl +++ /dev/null @@ -1 +0,0 @@ -{{define "get_host"}}{{ with urlParse . }}{{print .Scheme `://` .Host}}{{end}}{{end}} diff --git a/libs/template/testdata/get_host/template/my_host b/libs/template/testdata/get_host/template/my_host deleted file mode 100644 index b7a738790da..00000000000 --- a/libs/template/testdata/get_host/template/my_host +++ /dev/null @@ -1 +0,0 @@ -{{template "get_host" "https://www.host.com/a/b/c/d/e?o=123#fragment"}} diff --git a/libs/template/testdata/is_https/library/is_https_check.tmpl b/libs/template/testdata/is_https/library/is_https_check.tmpl deleted file mode 100644 index c43083eb113..00000000000 --- a/libs/template/testdata/is_https/library/is_https_check.tmpl +++ /dev/null @@ -1,6 +0,0 @@ -{{define "is_https_check" -}} - {{- $regex := regexpCompile `^https` -}} - {{- if not ($regex.MatchString .) -}} - {{- fail "expected %s to start with https" . -}} - {{- end -}} -{{- end}} diff --git a/libs/template/testdata/is_https/template_is_https/my_check b/libs/template/testdata/is_https/template_is_https/my_check deleted file mode 100644 index a9298c3b014..00000000000 --- a/libs/template/testdata/is_https/template_is_https/my_check +++ /dev/null @@ -1,2 +0,0 @@ -{{- template "is_https_check" "https://www.databricks.com" -}} -this file is created if validation passes diff --git a/libs/template/testdata/is_https/template_not_https/my_check b/libs/template/testdata/is_https/template_not_https/my_check deleted file mode 100644 index c3dc74c04c3..00000000000 --- a/libs/template/testdata/is_https/template_not_https/my_check +++ /dev/null @@ -1 +0,0 @@ -{{template "is_https_check" "/not-a-url"}} diff --git a/libs/template/testdata/skip_dir/databricks_template_schema.json b/libs/template/testdata/skip_dir/databricks_template_schema.json deleted file mode 100644 index e3d5253b26a..00000000000 --- a/libs/template/testdata/skip_dir/databricks_template_schema.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "properties": { - "a": { - "type": "string" - }, - "b": { - "type": "string" - }, - "c": { - "type": "string" - }, - "d": { - "type": "string" - } - } -} diff --git a/libs/template/testdata/skip_dir/template/{{.a}}/.gitkeep b/libs/template/testdata/skip_dir/template/{{.a}}/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/libs/template/testdata/skip_dir/template/{{.c}}/abc b/libs/template/testdata/skip_dir/template/{{.c}}/abc deleted file mode 100644 index 6af8c5850c9..00000000000 --- a/libs/template/testdata/skip_dir/template/{{.c}}/abc +++ /dev/null @@ -1,4 +0,0 @@ -{{if eq .d "foo"}} -{{skipThisFile}} -{{end}} -Hello, World From 11bac281e4b91e025b33d2ebf9474795fb9c7bee Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 23 Jul 2023 23:43:14 +0200 Subject: [PATCH 53/60] - --- cmd/bundle/init.go | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 cmd/bundle/init.go diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go deleted file mode 100644 index 598e6492bd0..00000000000 --- a/cmd/bundle/init.go +++ /dev/null @@ -1,31 +0,0 @@ -package bundle - -import ( - "github.com/databricks/cli/libs/template" - "github.com/spf13/cobra" -) - -var initCmd = &cobra.Command{ - Use: "init TEMPLATE_PATH", - Short: "Initialize Template", - Long: `Initialize template`, - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - return template.Materialize(args[0], outputDir, configFile) - }, -} - -var configFile string -var outputDir string - -// TODO: move integration tests to be unit tests OR increase coverage - -func init() { - initCmd.Flags().StringVar(&configFile, "config-file", "", "Input parameters for template initialization") - initCmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to output the generated project into") - initCmd.MarkFlagRequired("config-file") - // TODO: make this flag optional and initialize into current directory. - // Should we initialize into current directory? - initCmd.MarkFlagRequired("output-dir") - AddCommand(initCmd) -} From a622df2046b0b35dedf59a0943668eb9f0ca32d4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 24 Jul 2023 00:29:29 +0200 Subject: [PATCH 54/60] some refactoring and now supporting default values --- libs/template/schema.go | 40 ++++++++++++++++++++---------------- libs/template/schema_test.go | 38 ++++++++++++++++++++++++++-------- libs/template/validators.go | 12 +++++++++-- 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/libs/template/schema.go b/libs/template/schema.go index 4edcc0d4b32..c19bd8421cd 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -7,8 +7,6 @@ import ( "reflect" ) -const LatestSchemaVersion = 0 - // This is a JSON Schema compliant struct that we use to do validation checks on // the provided configuration type Schema struct { @@ -28,6 +26,7 @@ const ( type Property struct { Type PropertyType `json:"type"` Description string `json:"description"` + Default any `json:"default"` } // function to check whether a float value represents an integer @@ -38,8 +37,8 @@ func isIntegerValue(v float64) bool { // cast value to integer for config values that are floats but are supposed to be // integers according to the schema // -// Needed because the default json unmarshaller for maps converts all numbers to floats -func castFloatToInt(config map[string]any, schema *Schema) error { +// Needed because the default json unmarshaler for maps converts all numbers to floats +func castFloatConfigValuesToInt(config map[string]any, schema *Schema) error { for k, v := range config { // error because all config keys should be defined in schema too if _, ok := schema.Properties[k]; !ok { @@ -72,15 +71,19 @@ func castFloatToInt(config map[string]any, schema *Schema) error { return nil } -func validateType(v any, fieldType PropertyType) error { - validateFunc, ok := validators[fieldType] - if !ok { - return nil +func assignDefaultConfigValues(config map[string]any, schema *Schema) error { + for k, v := range schema.Properties { + if _, ok := config[k]; !ok { + if v.Default == nil { + return fmt.Errorf("input parameter %s is not defined in config", k) + } + config[k] = v.Default + } } - return validateFunc(v) + return nil } -func (schema *Schema) ValidateConfig(config map[string]any) error { +func validateConfigValueTypes(config map[string]any, schema *Schema) error { // validate types defined in config for k, v := range config { fieldMetadata, ok := schema.Properties[k] @@ -92,12 +95,6 @@ func (schema *Schema) ValidateConfig(config map[string]any) error { return fmt.Errorf("incorrect type for %s. %w", k, err) } } - // assert all fields are defined in - for k := range schema.Properties { - if _, ok := config[k]; !ok { - return fmt.Errorf("input parameter %s is not defined in config", k) - } - } return nil } @@ -115,6 +112,7 @@ func ReadSchema(path string) (*Schema, error) { } func (schema *Schema) ReadConfig(path string) (map[string]any, error) { + // Read config file var config map[string]any b, err := os.ReadFile(path) if err != nil { @@ -125,15 +123,21 @@ func (schema *Schema) ReadConfig(path string) (map[string]any, error) { return nil, err } + // Assign default value to any fields that do not have a value yet + err = assignDefaultConfigValues(config, schema) + if err != nil { + return nil, err + } + // cast any fields that are supposed to be integers. The json unmarshalling // for a generic map converts all numbers to floating point - err = castFloatToInt(config, schema) + err = castFloatConfigValuesToInt(config, schema) if err != nil { return nil, err } // validate config according to schema - err = schema.ValidateConfig(config) + err = validateConfigValueTypes(config, schema) if err != nil { return nil, err } diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go index f3b69a0e677..6ccea3a9037 100644 --- a/libs/template/schema_test.go +++ b/libs/template/schema_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestTemplateSchematIsInterger(t *testing.T) { +func TestTemplateSchemaIsInteger(t *testing.T) { assert.False(t, isIntegerValue(1.1)) assert.False(t, isIntegerValue(0.1)) assert.False(t, isIntegerValue(-0.1)) @@ -58,7 +58,7 @@ func TestTemplateSchemaCastFloatToInt(t *testing.T) { assert.IsType(t, true, config["bool_val"]) assert.IsType(t, "abc", config["string_val"]) - err = castFloatToInt(config, &schema) + err = castFloatConfigValuesToInt(config, &schema) require.NoError(t, err) // assert type after casting, that the float value was converted to an integer @@ -90,7 +90,7 @@ func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { err = json.Unmarshal([]byte(configJson), &config) require.NoError(t, err) - err = castFloatToInt(config, &schema) + err = castFloatConfigValuesToInt(config, &schema) assert.ErrorContains(t, err, "bar is not defined as an input parameter for the template") } @@ -115,7 +115,7 @@ func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { err = json.Unmarshal([]byte(configJson), &config) require.NoError(t, err) - err = castFloatToInt(config, &schema) + err = castFloatConfigValuesToInt(config, &schema) assert.ErrorContains(t, err, "expected foo to have integer value but it is 1.1") } @@ -202,7 +202,7 @@ func TestTemplateSchemaValidateConfig(t *testing.T) { "string_val": "abc", } - err = schema.ValidateConfig(config) + err = validateConfigValueTypes(config, &schema) assert.NoError(t, err) } @@ -236,7 +236,7 @@ func TestTemplateSchemaValidateConfigFailsForUnknownField(t *testing.T) { "string_val": "abc", } - err = schema.ValidateConfig(config) + err = validateConfigValueTypes(config, &schema) assert.ErrorContains(t, err, "foo is not defined as an input parameter for the template") } @@ -270,7 +270,7 @@ func TestTemplateSchemaValidateConfigFailsForWhenIncorrectTypes(t *testing.T) { "string_val": "abc", } - err = schema.ValidateConfig(config) + err = validateConfigValueTypes(config, &schema) assert.ErrorContains(t, err, "incorrect type for bool_val. expected type boolean, but value is \"true\"") } @@ -295,6 +295,28 @@ func TestTemplateSchemaValidateConfigFailsForWhenMissingInputParams(t *testing.T "int_val": 1, } - err = schema.ValidateConfig(config) + err = assignDefaultConfigValues(config, &schema) assert.ErrorContains(t, err, "input parameter string_val is not defined in config") } + +func TestTemplateDefaultAssignment(t *testing.T) { + // define schema for config + schemaJson := `{ + "properties": { + "foo": { + "type": "integer", + "default": 1 + } + } + }` + var schema Schema + err := json.Unmarshal([]byte(schemaJson), &schema) + require.NoError(t, err) + + // define the config + config := map[string]any{} + + err = assignDefaultConfigValues(config, &schema) + assert.NoError(t, err) + assert.Equal(t, 1.0, config["foo"]) +} diff --git a/libs/template/validators.go b/libs/template/validators.go index 96995560355..fc6e758b6ec 100644 --- a/libs/template/validators.go +++ b/libs/template/validators.go @@ -7,7 +7,15 @@ import ( "golang.org/x/exp/slices" ) -type Validator func(v any) error +type validator func(v any) error + +func validateType(v any, fieldType PropertyType) error { + validateFunc, ok := validators[fieldType] + if !ok { + return nil + } + return validateFunc(v) +} func validateString(v any) error { if _, ok := v.(string); !ok { @@ -39,7 +47,7 @@ func validateInteger(v any) error { return nil } -var validators map[PropertyType]Validator = map[PropertyType]Validator{ +var validators map[PropertyType]validator = map[PropertyType]validator{ PropertyTypeString: validateString, PropertyTypeBoolean: validateBoolean, PropertyTypeInt: validateInteger, From a2465a86b0fc94d4f6a0f2b7878db07cdfca4a60 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 24 Jul 2023 00:30:31 +0200 Subject: [PATCH 55/60] - --- .../databricks_template_schema.json | 18 ------------------ .../init/templateDefinition/library/foo | 2 -- .../{{.directory_name}}/.{{.file_name}} | 8 -------- .../template/{{.directory_name}}/aws_file | 4 ---- .../template/{{.directory_name}}/azure_file | 4 ---- 5 files changed, 36 deletions(-) delete mode 100644 internal/testdata/init/templateDefinition/databricks_template_schema.json delete mode 100644 internal/testdata/init/templateDefinition/library/foo delete mode 100644 internal/testdata/init/templateDefinition/template/{{.directory_name}}/.{{.file_name}} delete mode 100644 internal/testdata/init/templateDefinition/template/{{.directory_name}}/aws_file delete mode 100644 internal/testdata/init/templateDefinition/template/{{.directory_name}}/azure_file diff --git a/internal/testdata/init/templateDefinition/databricks_template_schema.json b/internal/testdata/init/templateDefinition/databricks_template_schema.json deleted file mode 100644 index cd0046f4258..00000000000 --- a/internal/testdata/init/templateDefinition/databricks_template_schema.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "properties": { - "directory_name": { - "description": "Name of the directory", - "type": "string" - }, - "skip_extra_content_if_abc": { - "type": "string" - }, - "skip_file_if_true": { - "type": "boolean" - }, - "file_name": { - "type": "string", - "description": "Name of the file" - } - } -} diff --git a/internal/testdata/init/templateDefinition/library/foo b/internal/testdata/init/templateDefinition/library/foo deleted file mode 100644 index 2b917a32294..00000000000 --- a/internal/testdata/init/templateDefinition/library/foo +++ /dev/null @@ -1,2 +0,0 @@ -{{define "email"}}shreyas.goenka@databricks.com{{end}} -{{define "get_host"}}{{ with urlParse . }}{{print .Scheme `://` .Host}}{{end}}{{end}} diff --git a/internal/testdata/init/templateDefinition/template/{{.directory_name}}/.{{.file_name}} b/internal/testdata/init/templateDefinition/template/{{.directory_name}}/.{{.file_name}} deleted file mode 100644 index f71e78daadd..00000000000 --- a/internal/testdata/init/templateDefinition/template/{{.directory_name}}/.{{.file_name}} +++ /dev/null @@ -1,8 +0,0 @@ -{{if eq .skip_extra_content_if_abc "abc" -}} -this is extra content -{{- end -}} - -{{if eq .skip_file_if_true }} -this is extra content -{{- end -}} -{{- skipThisFile -}} diff --git a/internal/testdata/init/templateDefinition/template/{{.directory_name}}/aws_file b/internal/testdata/init/templateDefinition/template/{{.directory_name}}/aws_file deleted file mode 100644 index a5f33d20e4f..00000000000 --- a/internal/testdata/init/templateDefinition/template/{{.directory_name}}/aws_file +++ /dev/null @@ -1,4 +0,0 @@ -This file should only be generated for AWS -{{if ne .cloud_type "AWS"}} -{{skipThisFile}} -{{end}} diff --git a/internal/testdata/init/templateDefinition/template/{{.directory_name}}/azure_file b/internal/testdata/init/templateDefinition/template/{{.directory_name}}/azure_file deleted file mode 100644 index b011b3832b2..00000000000 --- a/internal/testdata/init/templateDefinition/template/{{.directory_name}}/azure_file +++ /dev/null @@ -1,4 +0,0 @@ -{{if ne .cloud_type "Azure"}}{{skipThisFile}}{{end}} -This file should only be generated for Azure -{{ template "email" }} -{{ template "get_host" "https://adb-xxxx.xx.azuredatabricks.net/sql/queries" }} From f4356d48daf677dcb2243ae31a04675c838ea792 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 25 Jul 2023 13:15:02 +0200 Subject: [PATCH 56/60] address comments --- bundle/schema/docs.go | 17 ++-- bundle/schema/docs_test.go | 11 ++- bundle/schema/openapi.go | 23 ++--- bundle/schema/openapi_test.go | 17 ++-- bundle/schema/schema.go | 94 ++++++------------- bundle/schema/schema_test.go | 2 +- libs/schema/schema.go | 49 ++++++++++ libs/template/schema.go | 62 +++++-------- libs/template/schema_test.go | 166 ++++++++++++---------------------- libs/template/validators.go | 22 +++-- 10 files changed, 206 insertions(+), 257 deletions(-) create mode 100644 libs/schema/schema.go diff --git a/bundle/schema/docs.go b/bundle/schema/docs.go index 13a4549d0ec..00237c299e9 100644 --- a/bundle/schema/docs.go +++ b/bundle/schema/docs.go @@ -8,6 +8,7 @@ import ( "reflect" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/schema" "github.com/databricks/databricks-sdk-go/openapi" ) @@ -39,7 +40,7 @@ func BundleDocs(openapiSpecPath string) (*Docs, error) { } openapiReader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*Schema), + Memo: make(map[string]*schema.Schema), } resourcesDocs, err := openapiReader.ResourcesDocs() if err != nil { @@ -88,22 +89,22 @@ func initializeBundleDocs() (*Docs, error) { } // *Docs are a subset of *Schema, this function selects that subset -func schemaToDocs(schema *Schema) *Docs { +func schemaToDocs(jsonSchema *schema.Schema) *Docs { // terminate recursion if schema is nil - if schema == nil { + if jsonSchema == nil { return nil } docs := &Docs{ - Description: schema.Description, + Description: jsonSchema.Description, } - if len(schema.Properties) > 0 { + if len(jsonSchema.Properties) > 0 { docs.Properties = make(map[string]*Docs) } - for k, v := range schema.Properties { + for k, v := range jsonSchema.Properties { docs.Properties[k] = schemaToDocs(v) } - docs.Items = schemaToDocs(schema.Items) - if additionalProperties, ok := schema.AdditionalProperties.(*Schema); ok { + docs.Items = schemaToDocs(jsonSchema.Items) + if additionalProperties, ok := jsonSchema.AdditionalProperties.(*schema.Schema); ok { docs.AdditionalProperties = schemaToDocs(additionalProperties) } return docs diff --git a/bundle/schema/docs_test.go b/bundle/schema/docs_test.go index 84d804b07bf..9b2c3f4ed94 100644 --- a/bundle/schema/docs_test.go +++ b/bundle/schema/docs_test.go @@ -4,30 +4,31 @@ import ( "encoding/json" "testing" + "github.com/databricks/cli/libs/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestSchemaToDocs(t *testing.T) { - schema := &Schema{ + jsonSchema := &schema.Schema{ Type: "object", Description: "root doc", - Properties: map[string]*Schema{ + Properties: map[string]*schema.Schema{ "foo": {Type: "number", Description: "foo doc"}, "bar": {Type: "string"}, "octave": { Type: "object", - AdditionalProperties: &Schema{Type: "number"}, + AdditionalProperties: &schema.Schema{Type: "number"}, Description: "octave docs", }, "scales": { Type: "object", Description: "scale docs", - Items: &Schema{Type: "string"}, + Items: &schema.Schema{Type: "string"}, }, }, } - docs := schemaToDocs(schema) + docs := schemaToDocs(jsonSchema) docsJson, err := json.MarshalIndent(docs, " ", " ") require.NoError(t, err) diff --git a/bundle/schema/openapi.go b/bundle/schema/openapi.go index 9b4b27dd94e..1b92599c8d4 100644 --- a/bundle/schema/openapi.go +++ b/bundle/schema/openapi.go @@ -5,17 +5,18 @@ import ( "fmt" "strings" + "github.com/databricks/cli/libs/schema" "github.com/databricks/databricks-sdk-go/openapi" ) type OpenapiReader struct { OpenapiSpec *openapi.Specification - Memo map[string]*Schema + Memo map[string]*schema.Schema } const SchemaPathPrefix = "#/components/schemas/" -func (reader *OpenapiReader) readOpenapiSchema(path string) (*Schema, error) { +func (reader *OpenapiReader) readOpenapiSchema(path string) (*schema.Schema, error) { schemaKey := strings.TrimPrefix(path, SchemaPathPrefix) // return early if we already have a computed schema @@ -35,7 +36,7 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*Schema, error) { if err != nil { return nil, err } - jsonSchema := &Schema{} + jsonSchema := &schema.Schema{} err = json.Unmarshal(bytes, jsonSchema) if err != nil { return nil, err @@ -50,7 +51,7 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*Schema, error) { if err != nil { return nil, err } - additionalProperties := &Schema{} + additionalProperties := &schema.Schema{} err = json.Unmarshal(b, additionalProperties) if err != nil { return nil, err @@ -65,7 +66,7 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*Schema, error) { } // safe againt loops in refs -func (reader *OpenapiReader) safeResolveRefs(root *Schema, tracker *tracker) (*Schema, error) { +func (reader *OpenapiReader) safeResolveRefs(root *schema.Schema, tracker *tracker) (*schema.Schema, error) { if root.Reference == nil { return reader.traverseSchema(root, tracker) } @@ -100,9 +101,9 @@ func (reader *OpenapiReader) safeResolveRefs(root *Schema, tracker *tracker) (*S return root, err } -func (reader *OpenapiReader) traverseSchema(root *Schema, tracker *tracker) (*Schema, error) { +func (reader *OpenapiReader) traverseSchema(root *schema.Schema, tracker *tracker) (*schema.Schema, error) { // case primitive (or invalid) - if root.Type != Object && root.Type != Array { + if root.Type != schema.ObjectType && root.Type != schema.ArrayType { return root, nil } // only root references are resolved @@ -128,9 +129,9 @@ func (reader *OpenapiReader) traverseSchema(root *Schema, tracker *tracker) (*Sc root.Items = itemsSchema } // case map - additionionalProperties, ok := root.AdditionalProperties.(*Schema) - if ok && additionionalProperties != nil { - valueSchema, err := reader.safeResolveRefs(additionionalProperties, tracker) + additionalProperties, ok := root.AdditionalProperties.(*schema.Schema) + if ok && additionalProperties != nil { + valueSchema, err := reader.safeResolveRefs(additionalProperties, tracker) if err != nil { return nil, err } @@ -139,7 +140,7 @@ func (reader *OpenapiReader) traverseSchema(root *Schema, tracker *tracker) (*Sc return root, nil } -func (reader *OpenapiReader) readResolvedSchema(path string) (*Schema, error) { +func (reader *OpenapiReader) readResolvedSchema(path string) (*schema.Schema, error) { root, err := reader.readOpenapiSchema(path) if err != nil { return nil, err diff --git a/bundle/schema/openapi_test.go b/bundle/schema/openapi_test.go index 282fac8dfd3..a23605a047b 100644 --- a/bundle/schema/openapi_test.go +++ b/bundle/schema/openapi_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "testing" + "github.com/databricks/cli/libs/schema" "github.com/databricks/databricks-sdk-go/openapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -47,7 +48,7 @@ func TestReadSchemaForObject(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*Schema), + Memo: make(map[string]*schema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -105,7 +106,7 @@ func TestReadSchemaForArray(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*Schema), + Memo: make(map[string]*schema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -151,7 +152,7 @@ func TestReadSchemaForMap(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*Schema), + Memo: make(map[string]*schema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -200,7 +201,7 @@ func TestRootReferenceIsResolved(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*Schema), + Memo: make(map[string]*schema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -250,7 +251,7 @@ func TestSelfReferenceLoopErrors(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*Schema), + Memo: make(map[string]*schema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -284,7 +285,7 @@ func TestCrossReferenceLoopErrors(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*Schema), + Memo: make(map[string]*schema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -329,7 +330,7 @@ func TestReferenceResolutionForMapInObject(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*Schema), + Memo: make(map[string]*schema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -399,7 +400,7 @@ func TestReferenceResolutionForArrayInObject(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*Schema), + Memo: make(map[string]*schema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go index 7a55cbd2baf..3783c2b0b92 100644 --- a/bundle/schema/schema.go +++ b/bundle/schema/schema.go @@ -5,40 +5,9 @@ import ( "fmt" "reflect" "strings" -) - -// defines schema for a json object -type Schema struct { - // Type of the object - Type JavascriptType `json:"type,omitempty"` - - // Description of the object. This is rendered as inline documentation in the - // IDE. This is manually injected here using schema.Docs - Description string `json:"description,omitempty"` - - // Schemas for the fields of an struct. The keys are the first json tag. - // The values are the schema for the type of the field - Properties map[string]*Schema `json:"properties,omitempty"` - - // The schema for all values of an array - Items *Schema `json:"items,omitempty"` - - // The schema for any properties not mentioned in the Schema.Properties field. - // this validates maps[string]any in bundle configuration - // OR - // A boolean type with value false. Setting false here validates that all - // properties in the config have been defined in the json schema as properties - // - // Its type during runtime will either be *Schema or bool - AdditionalProperties any `json:"additionalProperties,omitempty"` - // Required properties for the object. Any fields missing the "omitempty" - // json tag will be included - Required []string `json:"required,omitempty"` - - // URI to a json schema - Reference *string `json:"$ref,omitempty"` -} + "github.com/databricks/cli/libs/schema" +) // This function translates golang types into json schema. Here is the mapping // between json schema types and golang types @@ -61,7 +30,7 @@ type Schema struct { // // - []MyStruct -> {type: object, properties: {}, additionalProperties: false} // for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties -func New(golangType reflect.Type, docs *Docs) (*Schema, error) { +func New(golangType reflect.Type, docs *Docs) (*schema.Schema, error) { tracker := newTracker() schema, err := safeToSchema(golangType, docs, "", tracker) if err != nil { @@ -70,39 +39,28 @@ func New(golangType reflect.Type, docs *Docs) (*Schema, error) { return schema, nil } -type JavascriptType string - -const ( - Invalid JavascriptType = "invalid" - Boolean JavascriptType = "boolean" - String JavascriptType = "string" - Number JavascriptType = "number" - Object JavascriptType = "object" - Array JavascriptType = "array" -) - -func javascriptType(golangType reflect.Type) (JavascriptType, error) { +func jsonSchemaType(golangType reflect.Type) (schema.Type, error) { switch golangType.Kind() { case reflect.Bool: - return Boolean, nil + return schema.BooleanType, nil case reflect.String: - return String, nil + return schema.StringType, nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64: - return Number, nil + return schema.NumberType, nil case reflect.Struct: - return Object, nil + return schema.ObjectType, nil case reflect.Map: if golangType.Key().Kind() != reflect.String { - return Invalid, fmt.Errorf("only strings map keys are valid. key type: %v", golangType.Key().Kind()) + return schema.InvalidType, fmt.Errorf("only strings map keys are valid. key type: %v", golangType.Key().Kind()) } - return Object, nil + return schema.ObjectType, nil case reflect.Array, reflect.Slice: - return Array, nil + return schema.ArrayType, nil default: - return Invalid, fmt.Errorf("unhandled golang type: %s", golangType) + return schema.InvalidType, fmt.Errorf("unhandled golang type: %s", golangType) } } @@ -121,7 +79,7 @@ func javascriptType(golangType reflect.Type) (JavascriptType, error) { // like array, map or no json tags // // - tracker: Keeps track of types / traceIds seen during recursive traversal -func safeToSchema(golangType reflect.Type, docs *Docs, traceId string, tracker *tracker) (*Schema, error) { +func safeToSchema(golangType reflect.Type, docs *Docs, traceId string, tracker *tracker) (*schema.Schema, error) { // WE ERROR OUT IF THERE ARE CYCLES IN THE JSON SCHEMA // There are mechanisms to deal with cycles though recursive identifiers in json // schema. However if we use them, we would need to make sure we are able to detect @@ -174,29 +132,29 @@ func getStructFields(golangType reflect.Type) []reflect.StructField { return fields } -func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, error) { +func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*schema.Schema, error) { // *Struct and Struct generate identical json schemas if golangType.Kind() == reflect.Pointer { return safeToSchema(golangType.Elem(), docs, "", tracker) } if golangType.Kind() == reflect.Interface { - return &Schema{}, nil + return &schema.Schema{}, nil } - rootJavascriptType, err := javascriptType(golangType) + rootJavascriptType, err := jsonSchemaType(golangType) if err != nil { return nil, err } - schema := &Schema{Type: rootJavascriptType} + jsonSchema := &schema.Schema{Type: rootJavascriptType} if docs != nil { - schema.Description = docs.Description + jsonSchema.Description = docs.Description } // case array/slice if golangType.Kind() == reflect.Array || golangType.Kind() == reflect.Slice { elemGolangType := golangType.Elem() - elemJavascriptType, err := javascriptType(elemGolangType) + elemJavascriptType, err := jsonSchemaType(elemGolangType) if err != nil { return nil, err } @@ -208,7 +166,7 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e if err != nil { return nil, err } - schema.Items = &Schema{ + jsonSchema.Items = &schema.Schema{ Type: elemJavascriptType, Properties: elemProps.Properties, AdditionalProperties: elemProps.AdditionalProperties, @@ -226,7 +184,7 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e if docs != nil { childDocs = docs.AdditionalProperties } - schema.AdditionalProperties, err = safeToSchema(golangType.Elem(), childDocs, "", tracker) + jsonSchema.AdditionalProperties, err = safeToSchema(golangType.Elem(), childDocs, "", tracker) if err != nil { return nil, err } @@ -235,7 +193,7 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e // case struct if golangType.Kind() == reflect.Struct { children := getStructFields(golangType) - properties := map[string]*Schema{} + properties := map[string]*schema.Schema{} required := []string{} for _, child := range children { bundleTag := child.Tag.Get("bundle") @@ -281,10 +239,10 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e properties[childName] = fieldProps } - schema.AdditionalProperties = false - schema.Properties = properties - schema.Required = required + jsonSchema.AdditionalProperties = false + jsonSchema.Properties = properties + jsonSchema.Required = required } - return schema, nil + return jsonSchema, nil } diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index 66baf8736a6..034919816bf 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -281,7 +281,7 @@ func TestStructOfMapsSchema(t *testing.T) { func TestStructOfSliceSchema(t *testing.T) { type Bar struct { MySlice []string `json:"my_slice"` - } + }, type Foo struct { Bar Bar `json:"bar"` diff --git a/libs/schema/schema.go b/libs/schema/schema.go new file mode 100644 index 00000000000..5785a1862bd --- /dev/null +++ b/libs/schema/schema.go @@ -0,0 +1,49 @@ +package schema + +// defines schema for a json object +type Schema struct { + // Type of the object + Type Type `json:"type,omitempty"` + + // Description of the object. This is rendered as inline documentation in the + // IDE. This is manually injected here using schema.Docs + Description string `json:"description,omitempty"` + + // Schemas for the fields of an struct. The keys are the first json tag. + // The values are the schema for the type of the field + Properties map[string]*Schema `json:"properties,omitempty"` + + // The schema for all values of an array + Items *Schema `json:"items,omitempty"` + + // The schema for any properties not mentioned in the Schema.Properties field. + // this validates maps[string]any in bundle configuration + // OR + // A boolean type with value false. Setting false here validates that all + // properties in the config have been defined in the json schema as properties + // + // Its type during runtime will either be *Schema or bool + AdditionalProperties any `json:"additionalProperties,omitempty"` + + // Required properties for the object. Any fields missing the "omitempty" + // json tag will be included + Required []string `json:"required,omitempty"` + + // URI to a json schema + Reference *string `json:"$ref,omitempty"` + + // Default value for the property / object + Default any `json:"default,omitempty"` +} + +type Type string + +const ( + InvalidType Type = "invalid" + BooleanType Type = "boolean" + StringType Type = "string" + NumberType Type = "number" + ObjectType Type = "object" + ArrayType Type = "array" + IntegerType Type = "integer" +) diff --git a/libs/template/schema.go b/libs/template/schema.go index c19bd8421cd..1b92e31c81a 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -5,30 +5,10 @@ import ( "fmt" "os" "reflect" -) - -// This is a JSON Schema compliant struct that we use to do validation checks on -// the provided configuration -type Schema struct { - // A list of properties that can be used in the config - Properties map[string]Property `json:"properties"` -} -type PropertyType string - -const ( - PropertyTypeString = PropertyType("string") - PropertyTypeInt = PropertyType("integer") - PropertyTypeNumber = PropertyType("number") - PropertyTypeBoolean = PropertyType("boolean") + "github.com/databricks/cli/libs/schema" ) -type Property struct { - Type PropertyType `json:"type"` - Description string `json:"description"` - Default any `json:"default"` -} - // function to check whether a float value represents an integer func isIntegerValue(v float64) bool { return v == float64(int(v)) @@ -38,16 +18,15 @@ func isIntegerValue(v float64) bool { // integers according to the schema // // Needed because the default json unmarshaler for maps converts all numbers to floats -func castFloatConfigValuesToInt(config map[string]any, schema *Schema) error { +func castFloatConfigValuesToInt(config map[string]any, jsonSchema *schema.Schema) error { for k, v := range config { // error because all config keys should be defined in schema too - if _, ok := schema.Properties[k]; !ok { + fieldInfo, ok := jsonSchema.Properties[k] + if !ok { return fmt.Errorf("%s is not defined as an input parameter for the template", k) } - // skip non integer fields - fieldInfo := schema.Properties[k] - if fieldInfo.Type != PropertyTypeInt { + if fieldInfo.Type != schema.IntegerType { continue } @@ -71,26 +50,27 @@ func castFloatConfigValuesToInt(config map[string]any, schema *Schema) error { return nil } -func assignDefaultConfigValues(config map[string]any, schema *Schema) error { +func assignDefaultConfigValues(config map[string]any, schema *schema.Schema) error { for k, v := range schema.Properties { - if _, ok := config[k]; !ok { - if v.Default == nil { - return fmt.Errorf("input parameter %s is not defined in config", k) - } - config[k] = v.Default + if _, ok := config[k]; ok { + continue + } + if v.Default == nil { + return fmt.Errorf("input parameter %s is not defined in config", k) } + config[k] = v.Default } return nil } -func validateConfigValueTypes(config map[string]any, schema *Schema) error { +func validateConfigValueTypes(config map[string]any, schema *schema.Schema) error { // validate types defined in config for k, v := range config { - fieldMetadata, ok := schema.Properties[k] + fieldInfo, ok := schema.Properties[k] if !ok { return fmt.Errorf("%s is not defined as an input parameter for the template", k) } - err := validateType(v, fieldMetadata.Type) + err := validateType(v, fieldInfo.Type) if err != nil { return fmt.Errorf("incorrect type for %s. %w", k, err) } @@ -98,12 +78,12 @@ func validateConfigValueTypes(config map[string]any, schema *Schema) error { return nil } -func ReadSchema(path string) (*Schema, error) { +func ReadSchema(path string) (*schema.Schema, error) { schemaBytes, err := os.ReadFile(path) if err != nil { return nil, err } - schema := &Schema{} + schema := &schema.Schema{} err = json.Unmarshal(schemaBytes, schema) if err != nil { return nil, err @@ -111,7 +91,7 @@ func ReadSchema(path string) (*Schema, error) { return schema, nil } -func (schema *Schema) ReadConfig(path string) (map[string]any, error) { +func ReadConfig(path string, jsonSchema *schema.Schema) (map[string]any, error) { // Read config file var config map[string]any b, err := os.ReadFile(path) @@ -124,20 +104,20 @@ func (schema *Schema) ReadConfig(path string) (map[string]any, error) { } // Assign default value to any fields that do not have a value yet - err = assignDefaultConfigValues(config, schema) + err = assignDefaultConfigValues(config, jsonSchema) if err != nil { return nil, err } // cast any fields that are supposed to be integers. The json unmarshalling // for a generic map converts all numbers to floating point - err = castFloatConfigValuesToInt(config, schema) + err = castFloatConfigValuesToInt(config, jsonSchema) if err != nil { return nil, err } // validate config according to schema - err = validateConfigValueTypes(config, schema) + err = validateConfigValueTypes(config, jsonSchema) if err != nil { return nil, err } diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go index 6ccea3a9037..ea39fd895fe 100644 --- a/libs/template/schema_test.go +++ b/libs/template/schema_test.go @@ -4,22 +4,12 @@ import ( "encoding/json" "testing" + "github.com/databricks/cli/libs/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestTemplateSchemaIsInteger(t *testing.T) { - assert.False(t, isIntegerValue(1.1)) - assert.False(t, isIntegerValue(0.1)) - assert.False(t, isIntegerValue(-0.1)) - - assert.True(t, isIntegerValue(-1.0)) - assert.True(t, isIntegerValue(0.0)) - assert.True(t, isIntegerValue(2.0)) -} - -func TestTemplateSchemaCastFloatToInt(t *testing.T) { - // define schema for config +func testSchema(t *testing.T) *schema.Schema { schemaJson := `{ "properties": { "int_val": { @@ -36,9 +26,25 @@ func TestTemplateSchemaCastFloatToInt(t *testing.T) { } } }` - var schema Schema - err := json.Unmarshal([]byte(schemaJson), &schema) + var jsonSchema schema.Schema + err := json.Unmarshal([]byte(schemaJson), &jsonSchema) require.NoError(t, err) + return &jsonSchema +} + +func TestTemplateSchemaIsInteger(t *testing.T) { + assert.False(t, isIntegerValue(1.1)) + assert.False(t, isIntegerValue(0.1)) + assert.False(t, isIntegerValue(-0.1)) + + assert.True(t, isIntegerValue(-1.0)) + assert.True(t, isIntegerValue(0.0)) + assert.True(t, isIntegerValue(2.0)) +} + +func TestTemplateSchemaCastFloatToInt(t *testing.T) { + // define schema for config + jsonSchema := testSchema(t) // define the config configJson := `{ @@ -48,7 +54,7 @@ func TestTemplateSchemaCastFloatToInt(t *testing.T) { "string_val": "main hoon na" }` var config map[string]any - err = json.Unmarshal([]byte(configJson), &config) + err := json.Unmarshal([]byte(configJson), &config) require.NoError(t, err) // assert types before casting, checking that the integer was indeed loaded @@ -58,7 +64,7 @@ func TestTemplateSchemaCastFloatToInt(t *testing.T) { assert.IsType(t, true, config["bool_val"]) assert.IsType(t, "abc", config["string_val"]) - err = castFloatConfigValuesToInt(config, &schema) + err = castFloatConfigValuesToInt(config, jsonSchema) require.NoError(t, err) // assert type after casting, that the float value was converted to an integer @@ -78,8 +84,8 @@ func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { } } }` - var schema Schema - err := json.Unmarshal([]byte(schemaJson), &schema) + var jsonSchema schema.Schema + err := json.Unmarshal([]byte(schemaJson), &jsonSchema) require.NoError(t, err) // define the config @@ -90,7 +96,7 @@ func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { err = json.Unmarshal([]byte(configJson), &config) require.NoError(t, err) - err = castFloatConfigValuesToInt(config, &schema) + err = castFloatConfigValuesToInt(config, &jsonSchema) assert.ErrorContains(t, err, "bar is not defined as an input parameter for the template") } @@ -103,8 +109,8 @@ func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { } } }` - var schema Schema - err := json.Unmarshal([]byte(schemaJson), &schema) + var jsonSchema schema.Schema + err := json.Unmarshal([]byte(schemaJson), &jsonSchema) require.NoError(t, err) // define the config @@ -115,84 +121,66 @@ func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { err = json.Unmarshal([]byte(configJson), &config) require.NoError(t, err) - err = castFloatConfigValuesToInt(config, &schema) + err = castFloatConfigValuesToInt(config, &jsonSchema) assert.ErrorContains(t, err, "expected foo to have integer value but it is 1.1") } func TestTemplateSchemaValidateType(t *testing.T) { // assert validation passing - err := validateType(int(0), PropertyTypeInt) + err := validateType(int(0), schema.IntegerType) assert.NoError(t, err) - err = validateType(int32(1), PropertyTypeInt) + err = validateType(int32(1), schema.IntegerType) assert.NoError(t, err) - err = validateType(int64(1), PropertyTypeInt) + err = validateType(int64(1), schema.IntegerType) assert.NoError(t, err) - err = validateType(float32(1.1), PropertyTypeNumber) + err = validateType(float32(1.1), schema.NumberType) assert.NoError(t, err) - err = validateType(float64(1.2), PropertyTypeNumber) + err = validateType(float64(1.2), schema.NumberType) assert.NoError(t, err) - err = validateType(int(1), PropertyTypeNumber) + err = validateType(int(1), schema.NumberType) assert.NoError(t, err) - err = validateType(false, PropertyTypeBoolean) + err = validateType(false, schema.BooleanType) assert.NoError(t, err) - err = validateType("abc", PropertyTypeString) + err = validateType("abc", schema.StringType) assert.NoError(t, err) // assert validation failing for integers - err = validateType(float64(1.2), PropertyTypeInt) + err = validateType(float64(1.2), schema.IntegerType) assert.ErrorContains(t, err, "expected type integer, but value is 1.2") - err = validateType(true, PropertyTypeInt) + err = validateType(true, schema.IntegerType) assert.ErrorContains(t, err, "expected type integer, but value is true") - err = validateType("abc", PropertyTypeInt) + err = validateType("abc", schema.IntegerType) assert.ErrorContains(t, err, "expected type integer, but value is \"abc\"") // assert validation failing for floats - err = validateType(true, PropertyTypeNumber) + err = validateType(true, schema.NumberType) assert.ErrorContains(t, err, "expected type float, but value is true") - err = validateType("abc", PropertyTypeNumber) + err = validateType("abc", schema.NumberType) assert.ErrorContains(t, err, "expected type float, but value is \"abc\"") // assert validation failing for boolean - err = validateType(int(1), PropertyTypeBoolean) + err = validateType(int(1), schema.BooleanType) assert.ErrorContains(t, err, "expected type boolean, but value is 1") - err = validateType(float64(1), PropertyTypeBoolean) + err = validateType(float64(1), schema.BooleanType) assert.ErrorContains(t, err, "expected type boolean, but value is 1") - err = validateType("abc", PropertyTypeBoolean) + err = validateType("abc", schema.BooleanType) assert.ErrorContains(t, err, "expected type boolean, but value is \"abc\"") // assert validation failing for string - err = validateType(int(1), PropertyTypeString) + err = validateType(int(1), schema.StringType) assert.ErrorContains(t, err, "expected type string, but value is 1") - err = validateType(float64(1), PropertyTypeString) + err = validateType(float64(1), schema.StringType) assert.ErrorContains(t, err, "expected type string, but value is 1") - err = validateType(false, PropertyTypeString) + err = validateType(false, schema.StringType) assert.ErrorContains(t, err, "expected type string, but value is false") } func TestTemplateSchemaValidateConfig(t *testing.T) { // define schema for config - schemaJson := `{ - "properties": { - "int_val": { - "type": "integer" - }, - "float_val": { - "type": "number" - }, - "bool_val": { - "type": "boolean" - }, - "string_val": { - "type": "string" - } - } - }` - var schema Schema - err := json.Unmarshal([]byte(schemaJson), &schema) - require.NoError(t, err) + jsonSchema := testSchema(t) // define the config config := map[string]any{ @@ -202,31 +190,13 @@ func TestTemplateSchemaValidateConfig(t *testing.T) { "string_val": "abc", } - err = validateConfigValueTypes(config, &schema) + err := validateConfigValueTypes(config, jsonSchema) assert.NoError(t, err) } func TestTemplateSchemaValidateConfigFailsForUnknownField(t *testing.T) { // define schema for config - schemaJson := `{ - "properties": { - "int_val": { - "type": "integer" - }, - "float_val": { - "type": "number" - }, - "bool_val": { - "type": "boolean" - }, - "string_val": { - "type": "string" - } - } - }` - var schema Schema - err := json.Unmarshal([]byte(schemaJson), &schema) - require.NoError(t, err) + jsonSchema := testSchema(t) // define the config config := map[string]any{ @@ -236,31 +206,13 @@ func TestTemplateSchemaValidateConfigFailsForUnknownField(t *testing.T) { "string_val": "abc", } - err = validateConfigValueTypes(config, &schema) + err := validateConfigValueTypes(config, jsonSchema) assert.ErrorContains(t, err, "foo is not defined as an input parameter for the template") } func TestTemplateSchemaValidateConfigFailsForWhenIncorrectTypes(t *testing.T) { // define schema for config - schemaJson := `{ - "properties": { - "int_val": { - "type": "integer" - }, - "float_val": { - "type": "number" - }, - "bool_val": { - "type": "boolean" - }, - "string_val": { - "type": "string" - } - } - }` - var schema Schema - err := json.Unmarshal([]byte(schemaJson), &schema) - require.NoError(t, err) + jsonSchema := testSchema(t) // define the config config := map[string]any{ @@ -270,7 +222,7 @@ func TestTemplateSchemaValidateConfigFailsForWhenIncorrectTypes(t *testing.T) { "string_val": "abc", } - err = validateConfigValueTypes(config, &schema) + err := validateConfigValueTypes(config, jsonSchema) assert.ErrorContains(t, err, "incorrect type for bool_val. expected type boolean, but value is \"true\"") } @@ -286,8 +238,8 @@ func TestTemplateSchemaValidateConfigFailsForWhenMissingInputParams(t *testing.T } } }` - var schema Schema - err := json.Unmarshal([]byte(schemaJson), &schema) + var jsonSchema schema.Schema + err := json.Unmarshal([]byte(schemaJson), &jsonSchema) require.NoError(t, err) // define the config @@ -295,7 +247,7 @@ func TestTemplateSchemaValidateConfigFailsForWhenMissingInputParams(t *testing.T "int_val": 1, } - err = assignDefaultConfigValues(config, &schema) + err = assignDefaultConfigValues(config, &jsonSchema) assert.ErrorContains(t, err, "input parameter string_val is not defined in config") } @@ -309,14 +261,14 @@ func TestTemplateDefaultAssignment(t *testing.T) { } } }` - var schema Schema - err := json.Unmarshal([]byte(schemaJson), &schema) + var jsonSchema schema.Schema + err := json.Unmarshal([]byte(schemaJson), &jsonSchema) require.NoError(t, err) // define the config config := map[string]any{} - err = assignDefaultConfigValues(config, &schema) + err = assignDefaultConfigValues(config, &jsonSchema) assert.NoError(t, err) assert.Equal(t, 1.0, config["foo"]) } diff --git a/libs/template/validators.go b/libs/template/validators.go index fc6e758b6ec..6f816c0cec7 100644 --- a/libs/template/validators.go +++ b/libs/template/validators.go @@ -4,12 +4,14 @@ import ( "fmt" "reflect" + "github.com/databricks/cli/libs/schema" "golang.org/x/exp/slices" ) type validator func(v any) error -func validateType(v any, fieldType PropertyType) error { +// Maybe this can be a part of the schema library itself? +func validateType(v any, fieldType schema.Type) error { validateFunc, ok := validators[fieldType] if !ok { return nil @@ -32,7 +34,9 @@ func validateBoolean(v any) error { } func validateNumber(v any) error { - if !slices.Contains([]reflect.Kind{reflect.Float32, reflect.Float64, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64}, + if !slices.Contains([]reflect.Kind{reflect.Float32, reflect.Float64, reflect.Int, + reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, + reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64}, reflect.TypeOf(v).Kind()) { return fmt.Errorf("expected type float, but value is %#v", v) } @@ -40,16 +44,18 @@ func validateNumber(v any) error { } func validateInteger(v any) error { - if !slices.Contains([]reflect.Kind{reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64}, + if !slices.Contains([]reflect.Kind{reflect.Int, reflect.Int8, reflect.Int16, + reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, + reflect.Uint32, reflect.Uint64}, reflect.TypeOf(v).Kind()) { return fmt.Errorf("expected type integer, but value is %#v", v) } return nil } -var validators map[PropertyType]validator = map[PropertyType]validator{ - PropertyTypeString: validateString, - PropertyTypeBoolean: validateBoolean, - PropertyTypeInt: validateInteger, - PropertyTypeNumber: validateNumber, +var validators map[schema.Type]validator = map[schema.Type]validator{ + schema.StringType: validateString, + schema.BooleanType: validateBoolean, + schema.IntegerType: validateInteger, + schema.NumberType: validateNumber, } From 093c8d9d29596aeb31460656052aa92395e8f540 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 25 Jul 2023 13:18:18 +0200 Subject: [PATCH 57/60] lint --- bundle/schema/schema_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index 034919816bf..66baf8736a6 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -281,7 +281,7 @@ func TestStructOfMapsSchema(t *testing.T) { func TestStructOfSliceSchema(t *testing.T) { type Bar struct { MySlice []string `json:"my_slice"` - }, + } type Foo struct { Bar Bar `json:"bar"` From 8b54564143701e9aca7ec73f4b2303c1f999f8c0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 25 Jul 2023 13:20:54 +0200 Subject: [PATCH 58/60] - --- libs/template/validators.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/template/validators.go b/libs/template/validators.go index 6f816c0cec7..68b733a27d8 100644 --- a/libs/template/validators.go +++ b/libs/template/validators.go @@ -10,7 +10,6 @@ import ( type validator func(v any) error -// Maybe this can be a part of the schema library itself? func validateType(v any, fieldType schema.Type) error { validateFunc, ok := validators[fieldType] if !ok { From 896fecc77ab1cc8844d7dead6bad0981e45f548a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 25 Jul 2023 14:57:05 +0200 Subject: [PATCH 59/60] rename package schema -> jsonschema --- bundle/schema/docs.go | 8 ++--- bundle/schema/docs_test.go | 10 +++--- bundle/schema/openapi.go | 20 +++++------ bundle/schema/openapi_test.go | 18 +++++----- bundle/schema/schema.go | 34 +++++++++--------- libs/{schema => jsonschema}/schema.go | 2 +- libs/template/schema.go | 16 ++++----- libs/template/schema_test.go | 52 +++++++++++++-------------- libs/template/validators.go | 14 ++++---- 9 files changed, 87 insertions(+), 87 deletions(-) rename libs/{schema => jsonschema}/schema.go (98%) diff --git a/bundle/schema/docs.go b/bundle/schema/docs.go index 00237c299e9..5fcef4eddf2 100644 --- a/bundle/schema/docs.go +++ b/bundle/schema/docs.go @@ -8,7 +8,7 @@ import ( "reflect" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/schema" + "github.com/databricks/cli/libs/jsonschema" "github.com/databricks/databricks-sdk-go/openapi" ) @@ -40,7 +40,7 @@ func BundleDocs(openapiSpecPath string) (*Docs, error) { } openapiReader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*schema.Schema), + Memo: make(map[string]*jsonschema.Schema), } resourcesDocs, err := openapiReader.ResourcesDocs() if err != nil { @@ -89,7 +89,7 @@ func initializeBundleDocs() (*Docs, error) { } // *Docs are a subset of *Schema, this function selects that subset -func schemaToDocs(jsonSchema *schema.Schema) *Docs { +func schemaToDocs(jsonSchema *jsonschema.Schema) *Docs { // terminate recursion if schema is nil if jsonSchema == nil { return nil @@ -104,7 +104,7 @@ func schemaToDocs(jsonSchema *schema.Schema) *Docs { docs.Properties[k] = schemaToDocs(v) } docs.Items = schemaToDocs(jsonSchema.Items) - if additionalProperties, ok := jsonSchema.AdditionalProperties.(*schema.Schema); ok { + if additionalProperties, ok := jsonSchema.AdditionalProperties.(*jsonschema.Schema); ok { docs.AdditionalProperties = schemaToDocs(additionalProperties) } return docs diff --git a/bundle/schema/docs_test.go b/bundle/schema/docs_test.go index 9b2c3f4ed94..83ee681b036 100644 --- a/bundle/schema/docs_test.go +++ b/bundle/schema/docs_test.go @@ -4,27 +4,27 @@ import ( "encoding/json" "testing" - "github.com/databricks/cli/libs/schema" + "github.com/databricks/cli/libs/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestSchemaToDocs(t *testing.T) { - jsonSchema := &schema.Schema{ + jsonSchema := &jsonschema.Schema{ Type: "object", Description: "root doc", - Properties: map[string]*schema.Schema{ + Properties: map[string]*jsonschema.Schema{ "foo": {Type: "number", Description: "foo doc"}, "bar": {Type: "string"}, "octave": { Type: "object", - AdditionalProperties: &schema.Schema{Type: "number"}, + AdditionalProperties: &jsonschema.Schema{Type: "number"}, Description: "octave docs", }, "scales": { Type: "object", Description: "scale docs", - Items: &schema.Schema{Type: "string"}, + Items: &jsonschema.Schema{Type: "string"}, }, }, } diff --git a/bundle/schema/openapi.go b/bundle/schema/openapi.go index 1b92599c8d4..b0d676576a2 100644 --- a/bundle/schema/openapi.go +++ b/bundle/schema/openapi.go @@ -5,18 +5,18 @@ import ( "fmt" "strings" - "github.com/databricks/cli/libs/schema" + "github.com/databricks/cli/libs/jsonschema" "github.com/databricks/databricks-sdk-go/openapi" ) type OpenapiReader struct { OpenapiSpec *openapi.Specification - Memo map[string]*schema.Schema + Memo map[string]*jsonschema.Schema } const SchemaPathPrefix = "#/components/schemas/" -func (reader *OpenapiReader) readOpenapiSchema(path string) (*schema.Schema, error) { +func (reader *OpenapiReader) readOpenapiSchema(path string) (*jsonschema.Schema, error) { schemaKey := strings.TrimPrefix(path, SchemaPathPrefix) // return early if we already have a computed schema @@ -36,7 +36,7 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*schema.Schema, err if err != nil { return nil, err } - jsonSchema := &schema.Schema{} + jsonSchema := &jsonschema.Schema{} err = json.Unmarshal(bytes, jsonSchema) if err != nil { return nil, err @@ -51,7 +51,7 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*schema.Schema, err if err != nil { return nil, err } - additionalProperties := &schema.Schema{} + additionalProperties := &jsonschema.Schema{} err = json.Unmarshal(b, additionalProperties) if err != nil { return nil, err @@ -66,7 +66,7 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*schema.Schema, err } // safe againt loops in refs -func (reader *OpenapiReader) safeResolveRefs(root *schema.Schema, tracker *tracker) (*schema.Schema, error) { +func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) { if root.Reference == nil { return reader.traverseSchema(root, tracker) } @@ -101,9 +101,9 @@ func (reader *OpenapiReader) safeResolveRefs(root *schema.Schema, tracker *track return root, err } -func (reader *OpenapiReader) traverseSchema(root *schema.Schema, tracker *tracker) (*schema.Schema, error) { +func (reader *OpenapiReader) traverseSchema(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) { // case primitive (or invalid) - if root.Type != schema.ObjectType && root.Type != schema.ArrayType { + if root.Type != jsonschema.ObjectType && root.Type != jsonschema.ArrayType { return root, nil } // only root references are resolved @@ -129,7 +129,7 @@ func (reader *OpenapiReader) traverseSchema(root *schema.Schema, tracker *tracke root.Items = itemsSchema } // case map - additionalProperties, ok := root.AdditionalProperties.(*schema.Schema) + additionalProperties, ok := root.AdditionalProperties.(*jsonschema.Schema) if ok && additionalProperties != nil { valueSchema, err := reader.safeResolveRefs(additionalProperties, tracker) if err != nil { @@ -140,7 +140,7 @@ func (reader *OpenapiReader) traverseSchema(root *schema.Schema, tracker *tracke return root, nil } -func (reader *OpenapiReader) readResolvedSchema(path string) (*schema.Schema, error) { +func (reader *OpenapiReader) readResolvedSchema(path string) (*jsonschema.Schema, error) { root, err := reader.readOpenapiSchema(path) if err != nil { return nil, err diff --git a/bundle/schema/openapi_test.go b/bundle/schema/openapi_test.go index a23605a047b..0d71fa440b0 100644 --- a/bundle/schema/openapi_test.go +++ b/bundle/schema/openapi_test.go @@ -4,7 +4,7 @@ import ( "encoding/json" "testing" - "github.com/databricks/cli/libs/schema" + "github.com/databricks/cli/libs/jsonschema" "github.com/databricks/databricks-sdk-go/openapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -48,7 +48,7 @@ func TestReadSchemaForObject(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*schema.Schema), + Memo: make(map[string]*jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -106,7 +106,7 @@ func TestReadSchemaForArray(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*schema.Schema), + Memo: make(map[string]*jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -152,7 +152,7 @@ func TestReadSchemaForMap(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*schema.Schema), + Memo: make(map[string]*jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -201,7 +201,7 @@ func TestRootReferenceIsResolved(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*schema.Schema), + Memo: make(map[string]*jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -251,7 +251,7 @@ func TestSelfReferenceLoopErrors(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*schema.Schema), + Memo: make(map[string]*jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -285,7 +285,7 @@ func TestCrossReferenceLoopErrors(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*schema.Schema), + Memo: make(map[string]*jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -330,7 +330,7 @@ func TestReferenceResolutionForMapInObject(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*schema.Schema), + Memo: make(map[string]*jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -400,7 +400,7 @@ func TestReferenceResolutionForArrayInObject(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*schema.Schema), + Memo: make(map[string]*jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go index 3783c2b0b92..fee9b676a4c 100644 --- a/bundle/schema/schema.go +++ b/bundle/schema/schema.go @@ -6,7 +6,7 @@ import ( "reflect" "strings" - "github.com/databricks/cli/libs/schema" + "github.com/databricks/cli/libs/jsonschema" ) // This function translates golang types into json schema. Here is the mapping @@ -30,7 +30,7 @@ import ( // // - []MyStruct -> {type: object, properties: {}, additionalProperties: false} // for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties -func New(golangType reflect.Type, docs *Docs) (*schema.Schema, error) { +func New(golangType reflect.Type, docs *Docs) (*jsonschema.Schema, error) { tracker := newTracker() schema, err := safeToSchema(golangType, docs, "", tracker) if err != nil { @@ -39,28 +39,28 @@ func New(golangType reflect.Type, docs *Docs) (*schema.Schema, error) { return schema, nil } -func jsonSchemaType(golangType reflect.Type) (schema.Type, error) { +func jsonSchemaType(golangType reflect.Type) (jsonschema.Type, error) { switch golangType.Kind() { case reflect.Bool: - return schema.BooleanType, nil + return jsonschema.BooleanType, nil case reflect.String: - return schema.StringType, nil + return jsonschema.StringType, nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64: - return schema.NumberType, nil + return jsonschema.NumberType, nil case reflect.Struct: - return schema.ObjectType, nil + return jsonschema.ObjectType, nil case reflect.Map: if golangType.Key().Kind() != reflect.String { - return schema.InvalidType, fmt.Errorf("only strings map keys are valid. key type: %v", golangType.Key().Kind()) + return jsonschema.InvalidType, fmt.Errorf("only strings map keys are valid. key type: %v", golangType.Key().Kind()) } - return schema.ObjectType, nil + return jsonschema.ObjectType, nil case reflect.Array, reflect.Slice: - return schema.ArrayType, nil + return jsonschema.ArrayType, nil default: - return schema.InvalidType, fmt.Errorf("unhandled golang type: %s", golangType) + return jsonschema.InvalidType, fmt.Errorf("unhandled golang type: %s", golangType) } } @@ -79,7 +79,7 @@ func jsonSchemaType(golangType reflect.Type) (schema.Type, error) { // like array, map or no json tags // // - tracker: Keeps track of types / traceIds seen during recursive traversal -func safeToSchema(golangType reflect.Type, docs *Docs, traceId string, tracker *tracker) (*schema.Schema, error) { +func safeToSchema(golangType reflect.Type, docs *Docs, traceId string, tracker *tracker) (*jsonschema.Schema, error) { // WE ERROR OUT IF THERE ARE CYCLES IN THE JSON SCHEMA // There are mechanisms to deal with cycles though recursive identifiers in json // schema. However if we use them, we would need to make sure we are able to detect @@ -132,20 +132,20 @@ func getStructFields(golangType reflect.Type) []reflect.StructField { return fields } -func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*schema.Schema, error) { +func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*jsonschema.Schema, error) { // *Struct and Struct generate identical json schemas if golangType.Kind() == reflect.Pointer { return safeToSchema(golangType.Elem(), docs, "", tracker) } if golangType.Kind() == reflect.Interface { - return &schema.Schema{}, nil + return &jsonschema.Schema{}, nil } rootJavascriptType, err := jsonSchemaType(golangType) if err != nil { return nil, err } - jsonSchema := &schema.Schema{Type: rootJavascriptType} + jsonSchema := &jsonschema.Schema{Type: rootJavascriptType} if docs != nil { jsonSchema.Description = docs.Description @@ -166,7 +166,7 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*schema.Sc if err != nil { return nil, err } - jsonSchema.Items = &schema.Schema{ + jsonSchema.Items = &jsonschema.Schema{ Type: elemJavascriptType, Properties: elemProps.Properties, AdditionalProperties: elemProps.AdditionalProperties, @@ -193,7 +193,7 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*schema.Sc // case struct if golangType.Kind() == reflect.Struct { children := getStructFields(golangType) - properties := map[string]*schema.Schema{} + properties := map[string]*jsonschema.Schema{} required := []string{} for _, child := range children { bundleTag := child.Tag.Get("bundle") diff --git a/libs/schema/schema.go b/libs/jsonschema/schema.go similarity index 98% rename from libs/schema/schema.go rename to libs/jsonschema/schema.go index 5785a1862bd..49e31bb7434 100644 --- a/libs/schema/schema.go +++ b/libs/jsonschema/schema.go @@ -1,4 +1,4 @@ -package schema +package jsonschema // defines schema for a json object type Schema struct { diff --git a/libs/template/schema.go b/libs/template/schema.go index 1b92e31c81a..5edeec9e169 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -6,7 +6,7 @@ import ( "os" "reflect" - "github.com/databricks/cli/libs/schema" + "github.com/databricks/cli/libs/jsonschema" ) // function to check whether a float value represents an integer @@ -18,7 +18,7 @@ func isIntegerValue(v float64) bool { // integers according to the schema // // Needed because the default json unmarshaler for maps converts all numbers to floats -func castFloatConfigValuesToInt(config map[string]any, jsonSchema *schema.Schema) error { +func castFloatConfigValuesToInt(config map[string]any, jsonSchema *jsonschema.Schema) error { for k, v := range config { // error because all config keys should be defined in schema too fieldInfo, ok := jsonSchema.Properties[k] @@ -26,7 +26,7 @@ func castFloatConfigValuesToInt(config map[string]any, jsonSchema *schema.Schema return fmt.Errorf("%s is not defined as an input parameter for the template", k) } // skip non integer fields - if fieldInfo.Type != schema.IntegerType { + if fieldInfo.Type != jsonschema.IntegerType { continue } @@ -50,7 +50,7 @@ func castFloatConfigValuesToInt(config map[string]any, jsonSchema *schema.Schema return nil } -func assignDefaultConfigValues(config map[string]any, schema *schema.Schema) error { +func assignDefaultConfigValues(config map[string]any, schema *jsonschema.Schema) error { for k, v := range schema.Properties { if _, ok := config[k]; ok { continue @@ -63,7 +63,7 @@ func assignDefaultConfigValues(config map[string]any, schema *schema.Schema) err return nil } -func validateConfigValueTypes(config map[string]any, schema *schema.Schema) error { +func validateConfigValueTypes(config map[string]any, schema *jsonschema.Schema) error { // validate types defined in config for k, v := range config { fieldInfo, ok := schema.Properties[k] @@ -78,12 +78,12 @@ func validateConfigValueTypes(config map[string]any, schema *schema.Schema) erro return nil } -func ReadSchema(path string) (*schema.Schema, error) { +func ReadSchema(path string) (*jsonschema.Schema, error) { schemaBytes, err := os.ReadFile(path) if err != nil { return nil, err } - schema := &schema.Schema{} + schema := &jsonschema.Schema{} err = json.Unmarshal(schemaBytes, schema) if err != nil { return nil, err @@ -91,7 +91,7 @@ func ReadSchema(path string) (*schema.Schema, error) { return schema, nil } -func ReadConfig(path string, jsonSchema *schema.Schema) (map[string]any, error) { +func ReadConfig(path string, jsonSchema *jsonschema.Schema) (map[string]any, error) { // Read config file var config map[string]any b, err := os.ReadFile(path) diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go index ea39fd895fe..ba30f81a9a8 100644 --- a/libs/template/schema_test.go +++ b/libs/template/schema_test.go @@ -4,12 +4,12 @@ import ( "encoding/json" "testing" - "github.com/databricks/cli/libs/schema" + "github.com/databricks/cli/libs/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func testSchema(t *testing.T) *schema.Schema { +func testSchema(t *testing.T) *jsonschema.Schema { schemaJson := `{ "properties": { "int_val": { @@ -26,7 +26,7 @@ func testSchema(t *testing.T) *schema.Schema { } } }` - var jsonSchema schema.Schema + var jsonSchema jsonschema.Schema err := json.Unmarshal([]byte(schemaJson), &jsonSchema) require.NoError(t, err) return &jsonSchema @@ -84,7 +84,7 @@ func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { } } }` - var jsonSchema schema.Schema + var jsonSchema jsonschema.Schema err := json.Unmarshal([]byte(schemaJson), &jsonSchema) require.NoError(t, err) @@ -109,7 +109,7 @@ func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { } } }` - var jsonSchema schema.Schema + var jsonSchema jsonschema.Schema err := json.Unmarshal([]byte(schemaJson), &jsonSchema) require.NoError(t, err) @@ -127,54 +127,54 @@ func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { func TestTemplateSchemaValidateType(t *testing.T) { // assert validation passing - err := validateType(int(0), schema.IntegerType) + err := validateType(int(0), jsonschema.IntegerType) assert.NoError(t, err) - err = validateType(int32(1), schema.IntegerType) + err = validateType(int32(1), jsonschema.IntegerType) assert.NoError(t, err) - err = validateType(int64(1), schema.IntegerType) + err = validateType(int64(1), jsonschema.IntegerType) assert.NoError(t, err) - err = validateType(float32(1.1), schema.NumberType) + err = validateType(float32(1.1), jsonschema.NumberType) assert.NoError(t, err) - err = validateType(float64(1.2), schema.NumberType) + err = validateType(float64(1.2), jsonschema.NumberType) assert.NoError(t, err) - err = validateType(int(1), schema.NumberType) + err = validateType(int(1), jsonschema.NumberType) assert.NoError(t, err) - err = validateType(false, schema.BooleanType) + err = validateType(false, jsonschema.BooleanType) assert.NoError(t, err) - err = validateType("abc", schema.StringType) + err = validateType("abc", jsonschema.StringType) assert.NoError(t, err) // assert validation failing for integers - err = validateType(float64(1.2), schema.IntegerType) + err = validateType(float64(1.2), jsonschema.IntegerType) assert.ErrorContains(t, err, "expected type integer, but value is 1.2") - err = validateType(true, schema.IntegerType) + err = validateType(true, jsonschema.IntegerType) assert.ErrorContains(t, err, "expected type integer, but value is true") - err = validateType("abc", schema.IntegerType) + err = validateType("abc", jsonschema.IntegerType) assert.ErrorContains(t, err, "expected type integer, but value is \"abc\"") // assert validation failing for floats - err = validateType(true, schema.NumberType) + err = validateType(true, jsonschema.NumberType) assert.ErrorContains(t, err, "expected type float, but value is true") - err = validateType("abc", schema.NumberType) + err = validateType("abc", jsonschema.NumberType) assert.ErrorContains(t, err, "expected type float, but value is \"abc\"") // assert validation failing for boolean - err = validateType(int(1), schema.BooleanType) + err = validateType(int(1), jsonschema.BooleanType) assert.ErrorContains(t, err, "expected type boolean, but value is 1") - err = validateType(float64(1), schema.BooleanType) + err = validateType(float64(1), jsonschema.BooleanType) assert.ErrorContains(t, err, "expected type boolean, but value is 1") - err = validateType("abc", schema.BooleanType) + err = validateType("abc", jsonschema.BooleanType) assert.ErrorContains(t, err, "expected type boolean, but value is \"abc\"") // assert validation failing for string - err = validateType(int(1), schema.StringType) + err = validateType(int(1), jsonschema.StringType) assert.ErrorContains(t, err, "expected type string, but value is 1") - err = validateType(float64(1), schema.StringType) + err = validateType(float64(1), jsonschema.StringType) assert.ErrorContains(t, err, "expected type string, but value is 1") - err = validateType(false, schema.StringType) + err = validateType(false, jsonschema.StringType) assert.ErrorContains(t, err, "expected type string, but value is false") } @@ -238,7 +238,7 @@ func TestTemplateSchemaValidateConfigFailsForWhenMissingInputParams(t *testing.T } } }` - var jsonSchema schema.Schema + var jsonSchema jsonschema.Schema err := json.Unmarshal([]byte(schemaJson), &jsonSchema) require.NoError(t, err) @@ -261,7 +261,7 @@ func TestTemplateDefaultAssignment(t *testing.T) { } } }` - var jsonSchema schema.Schema + var jsonSchema jsonschema.Schema err := json.Unmarshal([]byte(schemaJson), &jsonSchema) require.NoError(t, err) diff --git a/libs/template/validators.go b/libs/template/validators.go index 68b733a27d8..0ae41e46128 100644 --- a/libs/template/validators.go +++ b/libs/template/validators.go @@ -4,13 +4,13 @@ import ( "fmt" "reflect" - "github.com/databricks/cli/libs/schema" + "github.com/databricks/cli/libs/jsonschema" "golang.org/x/exp/slices" ) type validator func(v any) error -func validateType(v any, fieldType schema.Type) error { +func validateType(v any, fieldType jsonschema.Type) error { validateFunc, ok := validators[fieldType] if !ok { return nil @@ -52,9 +52,9 @@ func validateInteger(v any) error { return nil } -var validators map[schema.Type]validator = map[schema.Type]validator{ - schema.StringType: validateString, - schema.BooleanType: validateBoolean, - schema.IntegerType: validateInteger, - schema.NumberType: validateNumber, +var validators map[jsonschema.Type]validator = map[jsonschema.Type]validator{ + jsonschema.StringType: validateString, + jsonschema.BooleanType: validateBoolean, + jsonschema.IntegerType: validateInteger, + jsonschema.NumberType: validateNumber, } From 55f363b889308412f77efbee2f349993acb5711a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 25 Jul 2023 16:43:22 +0200 Subject: [PATCH 60/60] use type switch --- libs/template/schema.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/libs/template/schema.go b/libs/template/schema.go index 5edeec9e169..957cd66c769 100644 --- a/libs/template/schema.go +++ b/libs/template/schema.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "os" - "reflect" "github.com/databricks/cli/libs/jsonschema" ) @@ -31,16 +30,13 @@ func castFloatConfigValuesToInt(config map[string]any, jsonSchema *jsonschema.Sc } // convert floating point type values to integer - valueType := reflect.TypeOf(v) - switch valueType.Kind() { - case reflect.Float32: - floatVal := v.(float32) + switch floatVal := v.(type) { + case float32: if !isIntegerValue(float64(floatVal)) { return fmt.Errorf("expected %s to have integer value but it is %v", k, v) } config[k] = int(floatVal) - case reflect.Float64: - floatVal := v.(float64) + case float64: if !isIntegerValue(floatVal) { return fmt.Errorf("expected %s to have integer value but it is %v", k, v) }