diff --git a/libs/jsonschema/instance.go b/libs/jsonschema/instance.go new file mode 100644 index 00000000000..02ab9f281f9 --- /dev/null +++ b/libs/jsonschema/instance.go @@ -0,0 +1,91 @@ +package jsonschema + +import ( + "encoding/json" + "fmt" + "os" +) + +// Load a JSON document and validate it against the JSON schema. Instance here +// refers to a JSON document. see: https://json-schema.org/draft/2020-12/json-schema-core.html#name-instance +func (s *Schema) LoadInstance(path string) (map[string]any, error) { + instance := make(map[string]any) + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + err = json.Unmarshal(b, &instance) + if err != nil { + return nil, err + } + + // The default JSON unmarshaler parses untyped number values as float64. + // We convert integer properties from float64 to int64 here. + for name, v := range instance { + propertySchema, ok := s.Properties[name] + if !ok { + continue + } + if propertySchema.Type != IntegerType { + continue + } + integerValue, err := toInteger(v) + if err != nil { + return nil, fmt.Errorf("failed to parse property %s: %w", name, err) + } + instance[name] = integerValue + } + return instance, s.ValidateInstance(instance) +} + +func (s *Schema) ValidateInstance(instance map[string]any) error { + if err := s.validateAdditionalProperties(instance); err != nil { + return err + } + if err := s.validateRequired(instance); err != nil { + return err + } + return s.validateTypes(instance) +} + +// If additional properties is set to false, this function validates instance only +// contains properties defined in the schema. +func (s *Schema) validateAdditionalProperties(instance map[string]any) error { + // Note: AdditionalProperties has the type any. + if s.AdditionalProperties != false { + return nil + } + for k := range instance { + _, ok := s.Properties[k] + if !ok { + return fmt.Errorf("property %s is not defined in the schema", k) + } + } + return nil +} + +// This function validates that all require properties in the schema have values +// in the instance. +func (s *Schema) validateRequired(instance map[string]any) error { + for _, name := range s.Required { + if _, ok := instance[name]; !ok { + return fmt.Errorf("no value provided for required property %s", name) + } + } + return nil +} + +// Validates the types of all input properties values match their types defined in the schema +func (s *Schema) validateTypes(instance map[string]any) error { + for k, v := range instance { + fieldInfo, ok := s.Properties[k] + if !ok { + continue + } + err := validateType(v, fieldInfo.Type) + if err != nil { + return fmt.Errorf("incorrect type for property %s: %w", k, err) + } + } + return nil +} diff --git a/libs/jsonschema/instance_test.go b/libs/jsonschema/instance_test.go new file mode 100644 index 00000000000..d5e0766dd23 --- /dev/null +++ b/libs/jsonschema/instance_test.go @@ -0,0 +1,129 @@ +package jsonschema + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateInstanceAdditionalPropertiesPermitted(t *testing.T) { + instance := map[string]any{ + "int_val": 1, + "float_val": 1.0, + "bool_val": false, + "an_additional_property": "abc", + } + + schema, err := Load("./testdata/instance-validate/test-schema.json") + require.NoError(t, err) + + err = schema.validateAdditionalProperties(instance) + assert.NoError(t, err) + + err = schema.ValidateInstance(instance) + assert.NoError(t, err) +} + +func TestValidateInstanceAdditionalPropertiesForbidden(t *testing.T) { + instance := map[string]any{ + "int_val": 1, + "float_val": 1.0, + "bool_val": false, + "an_additional_property": "abc", + } + + schema, err := Load("./testdata/instance-validate/test-schema-no-additional-properties.json") + require.NoError(t, err) + + err = schema.validateAdditionalProperties(instance) + assert.EqualError(t, err, "property an_additional_property is not defined in the schema") + + err = schema.ValidateInstance(instance) + assert.EqualError(t, err, "property an_additional_property is not defined in the schema") + + instanceWOAdditionalProperties := map[string]any{ + "int_val": 1, + "float_val": 1.0, + "bool_val": false, + } + + err = schema.validateAdditionalProperties(instanceWOAdditionalProperties) + assert.NoError(t, err) + + err = schema.ValidateInstance(instanceWOAdditionalProperties) + assert.NoError(t, err) +} + +func TestValidateInstanceTypes(t *testing.T) { + schema, err := Load("./testdata/instance-validate/test-schema.json") + require.NoError(t, err) + + validInstance := map[string]any{ + "int_val": 1, + "float_val": 1.0, + "bool_val": false, + } + + err = schema.validateTypes(validInstance) + assert.NoError(t, err) + + err = schema.ValidateInstance(validInstance) + assert.NoError(t, err) + + invalidInstance := map[string]any{ + "int_val": "abc", + "float_val": 1.0, + "bool_val": false, + } + + err = schema.validateTypes(invalidInstance) + assert.EqualError(t, err, "incorrect type for property int_val: expected type integer, but value is \"abc\"") + + err = schema.ValidateInstance(invalidInstance) + assert.EqualError(t, err, "incorrect type for property int_val: expected type integer, but value is \"abc\"") +} + +func TestValidateInstanceRequired(t *testing.T) { + schema, err := Load("./testdata/instance-validate/test-schema-some-fields-required.json") + require.NoError(t, err) + + validInstance := map[string]any{ + "int_val": 1, + "float_val": 1.0, + "bool_val": false, + } + err = schema.validateRequired(validInstance) + assert.NoError(t, err) + err = schema.ValidateInstance(validInstance) + assert.NoError(t, err) + + invalidInstance := map[string]any{ + "string_val": "abc", + "float_val": 1.0, + "bool_val": false, + } + err = schema.validateRequired(invalidInstance) + assert.EqualError(t, err, "no value provided for required property int_val") + err = schema.ValidateInstance(invalidInstance) + assert.EqualError(t, err, "no value provided for required property int_val") +} + +func TestLoadInstance(t *testing.T) { + schema, err := Load("./testdata/instance-validate/test-schema.json") + require.NoError(t, err) + + // Expect the instance to be loaded successfully. + instance, err := schema.LoadInstance("./testdata/instance-load/valid-instance.json") + assert.NoError(t, err) + assert.Equal(t, map[string]any{ + "bool_val": false, + "int_val": int64(1), + "string_val": "abc", + "float_val": 2.0, + }, instance) + + // Expect instance validation against the schema to fail. + _, err = schema.LoadInstance("./testdata/instance-load/invalid-type-instance.json") + assert.EqualError(t, err, "incorrect type for property string_val: expected type string, but value is 123") +} diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index 87e9acd566b..44c65ecc6a5 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -58,6 +58,7 @@ const ( ) func (schema *Schema) validate() error { + // Validate property types are all valid JSON schema types. for _, v := range schema.Properties { switch v.Type { case NumberType, BooleanType, StringType, IntegerType: @@ -72,6 +73,17 @@ func (schema *Schema) validate() error { return fmt.Errorf("type %s is not a recognized json schema type", v.Type) } } + + // Validate default property values are consistent with types. + for name, property := range schema.Properties { + if property.Default == nil { + continue + } + if err := validateType(property.Default, property.Type); err != nil { + return fmt.Errorf("type validation for default value of property %s failed: %w", name, err) + } + } + return nil } @@ -85,5 +97,25 @@ func Load(path string) (*Schema, error) { if err != nil { return nil, err } + + // Convert the default values of top-level properties to integers. + // This is required because the default JSON unmarshaler parses numbers + // as floats when the Golang field it's being loaded to is untyped. + // + // NOTE: properties can be recursively defined in a schema, but the current + // use-cases only uses the first layer of properties so we skip converting + // any recursive properties. + for name, property := range schema.Properties { + if property.Type != IntegerType { + continue + } + if property.Default != nil { + property.Default, err = toInteger(property.Default) + if err != nil { + return nil, fmt.Errorf("failed to parse default value for property %s: %w", name, err) + } + } + } + return schema, schema.validate() } diff --git a/libs/jsonschema/schema_test.go b/libs/jsonschema/schema_test.go index 76112492f57..5b92d84669e 100644 --- a/libs/jsonschema/schema_test.go +++ b/libs/jsonschema/schema_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestJsonSchemaValidate(t *testing.T) { +func TestSchemaValidateTypeNames(t *testing.T) { var err error toSchema := func(s string) *Schema { return &Schema{ @@ -42,3 +42,40 @@ func TestJsonSchemaValidate(t *testing.T) { err = toSchema("foobar").validate() assert.EqualError(t, err, "type foobar is not a recognized json schema type") } + +func TestSchemaLoadIntegers(t *testing.T) { + schema, err := Load("./testdata/schema-load-int/schema-valid.json") + assert.NoError(t, err) + assert.Equal(t, int64(1), schema.Properties["abc"].Default) +} + +func TestSchemaLoadIntegersWithInvalidDefault(t *testing.T) { + _, err := Load("./testdata/schema-load-int/schema-invalid-default.json") + assert.EqualError(t, err, "failed to parse default value for property abc: expected integer value, got: 1.1") +} + +func TestSchemaValidateDefaultType(t *testing.T) { + invalidSchema := &Schema{ + Properties: map[string]*Schema{ + "foo": { + Type: "number", + Default: "abc", + }, + }, + } + + err := invalidSchema.validate() + assert.EqualError(t, err, "type validation for default value of property foo failed: expected type float, but value is \"abc\"") + + validSchema := &Schema{ + Properties: map[string]*Schema{ + "foo": { + Type: "boolean", + Default: true, + }, + }, + } + + err = validSchema.validate() + assert.NoError(t, err) +} diff --git a/libs/jsonschema/testdata/instance-load/invalid-type-instance.json b/libs/jsonschema/testdata/instance-load/invalid-type-instance.json new file mode 100644 index 00000000000..c55b6fccb72 --- /dev/null +++ b/libs/jsonschema/testdata/instance-load/invalid-type-instance.json @@ -0,0 +1,6 @@ +{ + "int_val": 1, + "bool_val": false, + "string_val": 123, + "float_val": 3.0 +} diff --git a/libs/jsonschema/testdata/instance-load/valid-instance.json b/libs/jsonschema/testdata/instance-load/valid-instance.json new file mode 100644 index 00000000000..7d4dc818ad4 --- /dev/null +++ b/libs/jsonschema/testdata/instance-load/valid-instance.json @@ -0,0 +1,6 @@ +{ + "int_val": 1, + "bool_val": false, + "string_val": "abc", + "float_val": 2.0 +} diff --git a/libs/jsonschema/testdata/instance-validate/test-schema-no-additional-properties.json b/libs/jsonschema/testdata/instance-validate/test-schema-no-additional-properties.json new file mode 100644 index 00000000000..98b19d5a489 --- /dev/null +++ b/libs/jsonschema/testdata/instance-validate/test-schema-no-additional-properties.json @@ -0,0 +1,19 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "default": 123 + }, + "float_val": { + "type": "number" + }, + "bool_val": { + "type": "boolean" + }, + "string_val": { + "type": "string", + "default": "abc" + } + }, + "additionalProperties": false +} diff --git a/libs/jsonschema/testdata/instance-validate/test-schema-some-fields-required.json b/libs/jsonschema/testdata/instance-validate/test-schema-some-fields-required.json new file mode 100644 index 00000000000..46581103454 --- /dev/null +++ b/libs/jsonschema/testdata/instance-validate/test-schema-some-fields-required.json @@ -0,0 +1,19 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "default": 123 + }, + "float_val": { + "type": "number" + }, + "bool_val": { + "type": "boolean" + }, + "string_val": { + "type": "string", + "default": "abc" + } + }, + "required": ["int_val", "float_val", "bool_val"] +} diff --git a/libs/jsonschema/testdata/instance-validate/test-schema.json b/libs/jsonschema/testdata/instance-validate/test-schema.json new file mode 100644 index 00000000000..41eb825190f --- /dev/null +++ b/libs/jsonschema/testdata/instance-validate/test-schema.json @@ -0,0 +1,18 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "default": 123 + }, + "float_val": { + "type": "number" + }, + "bool_val": { + "type": "boolean" + }, + "string_val": { + "type": "string", + "default": "abc" + } + } +} diff --git a/libs/jsonschema/testdata/schema-load-int/schema-invalid-default.json b/libs/jsonschema/testdata/schema-load-int/schema-invalid-default.json new file mode 100644 index 00000000000..1e709f622c5 --- /dev/null +++ b/libs/jsonschema/testdata/schema-load-int/schema-invalid-default.json @@ -0,0 +1,9 @@ +{ + "type": "object", + "properties": { + "abc": { + "type": "integer", + "default": 1.1 + } + } +} diff --git a/libs/jsonschema/testdata/schema-load-int/schema-valid.json b/libs/jsonschema/testdata/schema-load-int/schema-valid.json new file mode 100644 index 00000000000..599ac04d0d3 --- /dev/null +++ b/libs/jsonschema/testdata/schema-load-int/schema-valid.json @@ -0,0 +1,9 @@ +{ + "type": "object", + "properties": { + "abc": { + "type": "integer", + "default": 1 + } + } +} diff --git a/libs/template/utils.go b/libs/jsonschema/utils.go similarity index 80% rename from libs/template/utils.go rename to libs/jsonschema/utils.go index ade6a573058..21866965e2a 100644 --- a/libs/template/utils.go +++ b/libs/jsonschema/utils.go @@ -1,11 +1,9 @@ -package template +package jsonschema import ( "errors" "fmt" "strconv" - - "github.com/databricks/cli/libs/jsonschema" ) // function to check whether a float value represents an integer @@ -40,41 +38,41 @@ func toInteger(v any) (int64, error) { } } -func toString(v any, T jsonschema.Type) (string, error) { +func ToString(v any, T Type) (string, error) { switch T { - case jsonschema.BooleanType: + case BooleanType: boolVal, ok := v.(bool) if !ok { return "", fmt.Errorf("expected bool, got: %#v", v) } return strconv.FormatBool(boolVal), nil - case jsonschema.StringType: + case StringType: strVal, ok := v.(string) if !ok { return "", fmt.Errorf("expected string, got: %#v", v) } return strVal, nil - case jsonschema.NumberType: + case NumberType: floatVal, ok := v.(float64) if !ok { return "", fmt.Errorf("expected float, got: %#v", v) } return strconv.FormatFloat(floatVal, 'f', -1, 64), nil - case jsonschema.IntegerType: + case IntegerType: intVal, err := toInteger(v) if err != nil { return "", err } return strconv.FormatInt(intVal, 10), nil - case jsonschema.ArrayType, jsonschema.ObjectType: + case ArrayType, ObjectType: return "", fmt.Errorf("cannot format object of type %s as a string. Value of object: %#v", T, v) default: return "", fmt.Errorf("unknown json schema type: %q", T) } } -func fromString(s string, T jsonschema.Type) (any, error) { - if T == jsonschema.StringType { +func FromString(s string, T Type) (any, error) { + if T == StringType { return s, nil } @@ -83,13 +81,13 @@ func fromString(s string, T jsonschema.Type) (any, error) { var err error switch T { - case jsonschema.BooleanType: + case BooleanType: v, err = strconv.ParseBool(s) - case jsonschema.NumberType: + case NumberType: v, err = strconv.ParseFloat(s, 32) - case jsonschema.IntegerType: + case IntegerType: v, err = strconv.ParseInt(s, 10, 64) - case jsonschema.ArrayType, jsonschema.ObjectType: + case ArrayType, ObjectType: return "", fmt.Errorf("cannot parse string as object of type %s. Value of string: %q", T, s) default: return "", fmt.Errorf("unknown json schema type: %q", T) diff --git a/libs/template/utils_test.go b/libs/jsonschema/utils_test.go similarity index 72% rename from libs/template/utils_test.go rename to libs/jsonschema/utils_test.go index 1e038aac6f8..9686cf39bbd 100644 --- a/libs/template/utils_test.go +++ b/libs/jsonschema/utils_test.go @@ -1,10 +1,9 @@ -package template +package jsonschema import ( "math" "testing" - "github.com/databricks/cli/libs/jsonschema" "github.com/stretchr/testify/assert" ) @@ -50,72 +49,72 @@ func TestTemplateToInteger(t *testing.T) { } func TestTemplateToString(t *testing.T) { - s, err := toString(true, jsonschema.BooleanType) + s, err := ToString(true, BooleanType) assert.NoError(t, err) assert.Equal(t, "true", s) - s, err = toString("abc", jsonschema.StringType) + s, err = ToString("abc", StringType) assert.NoError(t, err) assert.Equal(t, "abc", s) - s, err = toString(1.1, jsonschema.NumberType) + s, err = ToString(1.1, NumberType) assert.NoError(t, err) assert.Equal(t, "1.1", s) - s, err = toString(2, jsonschema.IntegerType) + s, err = ToString(2, IntegerType) assert.NoError(t, err) assert.Equal(t, "2", s) - _, err = toString([]string{}, jsonschema.ArrayType) + _, err = ToString([]string{}, ArrayType) assert.EqualError(t, err, "cannot format object of type array as a string. Value of object: []string{}") - _, err = toString("true", jsonschema.BooleanType) + _, err = ToString("true", BooleanType) assert.EqualError(t, err, "expected bool, got: \"true\"") - _, err = toString(123, jsonschema.StringType) + _, err = ToString(123, StringType) assert.EqualError(t, err, "expected string, got: 123") - _, err = toString(false, jsonschema.NumberType) + _, err = ToString(false, NumberType) assert.EqualError(t, err, "expected float, got: false") - _, err = toString("abc", jsonschema.IntegerType) + _, err = ToString("abc", IntegerType) assert.EqualError(t, err, "cannot convert \"abc\" to an integer") - _, err = toString("abc", "foobar") + _, err = ToString("abc", "foobar") assert.EqualError(t, err, "unknown json schema type: \"foobar\"") } func TestTemplateFromString(t *testing.T) { - v, err := fromString("true", jsonschema.BooleanType) + v, err := FromString("true", BooleanType) assert.NoError(t, err) assert.Equal(t, true, v) - v, err = fromString("abc", jsonschema.StringType) + v, err = FromString("abc", StringType) assert.NoError(t, err) assert.Equal(t, "abc", v) - v, err = fromString("1.1", jsonschema.NumberType) + v, err = FromString("1.1", NumberType) assert.NoError(t, err) // Floating point conversions are not perfect assert.True(t, (v.(float64)-1.1) < 0.000001) - v, err = fromString("12345", jsonschema.IntegerType) + v, err = FromString("12345", IntegerType) assert.NoError(t, err) assert.Equal(t, int64(12345), v) - v, err = fromString("123", jsonschema.NumberType) + v, err = FromString("123", NumberType) assert.NoError(t, err) assert.Equal(t, float64(123), v) - _, err = fromString("qrt", jsonschema.ArrayType) + _, err = FromString("qrt", ArrayType) assert.EqualError(t, err, "cannot parse string as object of type array. Value of string: \"qrt\"") - _, err = fromString("abc", jsonschema.IntegerType) + _, err = FromString("abc", IntegerType) assert.EqualError(t, err, "could not parse \"abc\" as a integer: strconv.ParseInt: parsing \"abc\": invalid syntax") - _, err = fromString("1.0", jsonschema.IntegerType) + _, err = FromString("1.0", IntegerType) assert.EqualError(t, err, "could not parse \"1.0\" as a integer: strconv.ParseInt: parsing \"1.0\": invalid syntax") - _, err = fromString("1.0", "foobar") + _, err = FromString("1.0", "foobar") assert.EqualError(t, err, "unknown json schema type: \"foobar\"") } diff --git a/libs/template/validators.go b/libs/jsonschema/validate_type.go similarity index 68% rename from libs/template/validators.go rename to libs/jsonschema/validate_type.go index 209700b63c7..125d6b20b26 100644 --- a/libs/template/validators.go +++ b/libs/jsonschema/validate_type.go @@ -1,17 +1,15 @@ -package template +package jsonschema import ( "fmt" "reflect" "slices" - - "github.com/databricks/cli/libs/jsonschema" ) -type validator func(v any) error +type validateTypeFunc func(v any) error -func validateType(v any, fieldType jsonschema.Type) error { - validateFunc, ok := validators[fieldType] +func validateType(v any, fieldType Type) error { + validateFunc, ok := validateTypeFuncs[fieldType] if !ok { return nil } @@ -50,9 +48,9 @@ func validateInteger(v any) error { return nil } -var validators map[jsonschema.Type]validator = map[jsonschema.Type]validator{ - jsonschema.StringType: validateString, - jsonschema.BooleanType: validateBoolean, - jsonschema.IntegerType: validateInteger, - jsonschema.NumberType: validateNumber, +var validateTypeFuncs map[Type]validateTypeFunc = map[Type]validateTypeFunc{ + StringType: validateString, + BooleanType: validateBoolean, + IntegerType: validateInteger, + NumberType: validateNumber, } diff --git a/libs/template/validators_test.go b/libs/jsonschema/validate_type_test.go similarity index 75% rename from libs/template/validators_test.go rename to libs/jsonschema/validate_type_test.go index f34f037a1ef..36d9e5758df 100644 --- a/libs/template/validators_test.go +++ b/libs/jsonschema/validate_type_test.go @@ -1,9 +1,8 @@ -package template +package jsonschema import ( "testing" - "github.com/databricks/cli/libs/jsonschema" "github.com/stretchr/testify/assert" ) @@ -77,53 +76,53 @@ func TestValidatorInt(t *testing.T) { func TestTemplateValidateType(t *testing.T) { // assert validation passing - err := validateType(int(0), jsonschema.IntegerType) + err := validateType(int(0), IntegerType) assert.NoError(t, err) - err = validateType(int32(1), jsonschema.IntegerType) + err = validateType(int32(1), IntegerType) assert.NoError(t, err) - err = validateType(int64(1), jsonschema.IntegerType) + err = validateType(int64(1), IntegerType) assert.NoError(t, err) - err = validateType(float32(1.1), jsonschema.NumberType) + err = validateType(float32(1.1), NumberType) assert.NoError(t, err) - err = validateType(float64(1.2), jsonschema.NumberType) + err = validateType(float64(1.2), NumberType) assert.NoError(t, err) - err = validateType(false, jsonschema.BooleanType) + err = validateType(false, BooleanType) assert.NoError(t, err) - err = validateType("abc", jsonschema.StringType) + err = validateType("abc", StringType) assert.NoError(t, err) // assert validation failing for integers - err = validateType(float64(1.2), jsonschema.IntegerType) + err = validateType(float64(1.2), IntegerType) assert.ErrorContains(t, err, "expected type integer, but value is 1.2") - err = validateType(true, jsonschema.IntegerType) + err = validateType(true, IntegerType) assert.ErrorContains(t, err, "expected type integer, but value is true") - err = validateType("abc", jsonschema.IntegerType) + err = validateType("abc", IntegerType) assert.ErrorContains(t, err, "expected type integer, but value is \"abc\"") // assert validation failing for floats - err = validateType(true, jsonschema.NumberType) + err = validateType(true, NumberType) assert.ErrorContains(t, err, "expected type float, but value is true") - err = validateType("abc", jsonschema.NumberType) + err = validateType("abc", NumberType) assert.ErrorContains(t, err, "expected type float, but value is \"abc\"") - err = validateType(int(1), jsonschema.NumberType) + err = validateType(int(1), NumberType) assert.ErrorContains(t, err, "expected type float, but value is 1") // assert validation failing for boolean - err = validateType(int(1), jsonschema.BooleanType) + err = validateType(int(1), BooleanType) assert.ErrorContains(t, err, "expected type boolean, but value is 1") - err = validateType(float64(1), jsonschema.BooleanType) + err = validateType(float64(1), BooleanType) assert.ErrorContains(t, err, "expected type boolean, but value is 1") - err = validateType("abc", jsonschema.BooleanType) + err = validateType("abc", BooleanType) assert.ErrorContains(t, err, "expected type boolean, but value is \"abc\"") // assert validation failing for string - err = validateType(int(1), jsonschema.StringType) + err = validateType(int(1), StringType) assert.ErrorContains(t, err, "expected type string, but value is 1") - err = validateType(float64(1), jsonschema.StringType) + err = validateType(float64(1), StringType) assert.ErrorContains(t, err, "expected type string, but value is 1") - err = validateType(false, jsonschema.StringType) + err = validateType(false, StringType) assert.ErrorContains(t, err, "expected type string, but value is false") } diff --git a/libs/template/config.go b/libs/template/config.go index 8a1ed6c82a8..6f980f61319 100644 --- a/libs/template/config.go +++ b/libs/template/config.go @@ -2,12 +2,11 @@ package template import ( "context" - "encoding/json" "fmt" - "os" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/jsonschema" + "golang.org/x/exp/maps" ) type config struct { @@ -26,6 +25,9 @@ func newConfig(ctx context.Context, schemaPath string) (*config, error) { return nil, err } + // Do not allow template input variables that are not defined in the schema. + schema.AdditionalProperties = false + // Return config return &config{ ctx: ctx, @@ -45,32 +47,10 @@ func validateSchema(schema *jsonschema.Schema) error { // Reads json file at path and assigns values from the file func (c *config) assignValuesFromFile(path string) error { - // Read the config file - configFromFile := make(map[string]any, 0) - b, err := os.ReadFile(path) - if err != nil { - return err - } - err = json.Unmarshal(b, &configFromFile) + // Load the config file. + configFromFile, err := c.schema.LoadInstance(path) if err != nil { - return err - } - - // Cast any integer properties, from float to integer. Required because - // the json unmarshaller treats all json numbers as floating point - for name, floatVal := range configFromFile { - property, ok := c.schema.Properties[name] - if !ok { - return fmt.Errorf("%s is not defined as an input parameter for the template", name) - } - if property.Type != jsonschema.IntegerType { - continue - } - v, err := toInteger(floatVal) - if err != nil { - return fmt.Errorf("failed to cast value %v of property %s from file %s to an integer: %w", floatVal, name, path, err) - } - configFromFile[name] = v + return fmt.Errorf("failed to load config from file %s: %w", path, err) } // Write configs from the file to the input map, not overwriting any existing @@ -91,26 +71,11 @@ func (c *config) assignDefaultValues() error { if _, ok := c.values[name]; ok { continue } - // No default value defined for the property if property.Default == nil { continue } - - // Assign default value if property is not an integer - if property.Type != jsonschema.IntegerType { - c.values[name] = property.Default - continue - } - - // Cast default value to int before assigning to an integer configuration. - // Required because untyped field Default will read all numbers as floats - // during unmarshalling - v, err := toInteger(property.Default) - if err != nil { - return fmt.Errorf("failed to cast default value %v of property %s to an integer: %w", property.Default, name, err) - } - c.values[name] = v + c.values[name] = property.Default } return nil } @@ -130,7 +95,7 @@ func (c *config) promptForValues() error { var defaultVal string var err error if property.Default != nil { - defaultVal, err = toString(property.Default, property.Type) + defaultVal, err = jsonschema.ToString(property.Default, property.Type) if err != nil { return err } @@ -143,7 +108,7 @@ func (c *config) promptForValues() error { } // Convert user input string back to a value - c.values[name], err = fromString(userInput, property.Type) + c.values[name], err = jsonschema.FromString(userInput, property.Type) if err != nil { return err } @@ -163,42 +128,10 @@ func (c *config) promptOrAssignDefaultValues() error { // Validates the configuration. If passes, the configuration is ready to be used // to initialize the template. func (c *config) validate() error { - validateFns := []func() error{ - c.validateValuesDefined, - c.validateValuesType, - } - - for _, fn := range validateFns { - err := fn() - if err != nil { - return err - } - } - return nil -} - -// Validates all input properties have a user defined value assigned to them -func (c *config) validateValuesDefined() error { - for k := range c.schema.Properties { - if _, ok := c.values[k]; ok { - continue - } - return fmt.Errorf("no value has been assigned to input parameter %s", k) - } - return nil -} - -// Validates the types of all input properties values match their types defined in the schema -func (c *config) validateValuesType() error { - for k, v := range c.values { - fieldInfo, ok := c.schema.Properties[k] - if !ok { - return fmt.Errorf("%s is not defined as an input parameter for the template", k) - } - err := validateType(v, fieldInfo.Type) - if err != nil { - return fmt.Errorf("incorrect type for %s. %w", k, err) - } + // All properties in the JSON schema should have a value defined. + c.schema.Required = maps.Keys(c.schema.Properties) + if err := c.schema.ValidateInstance(c.values); err != nil { + return fmt.Errorf("validation for template input parameters failed. %w", err) } return nil } diff --git a/libs/template/config_test.go b/libs/template/config_test.go index 335242467f8..bba22c75875 100644 --- a/libs/template/config_test.go +++ b/libs/template/config_test.go @@ -1,7 +1,7 @@ package template import ( - "encoding/json" + "context" "testing" "github.com/databricks/cli/libs/jsonschema" @@ -9,36 +9,14 @@ import ( "github.com/stretchr/testify/require" ) -func testSchema(t *testing.T) *jsonschema.Schema { - schemaJson := `{ - "properties": { - "int_val": { - "type": "integer", - "default": 123 - }, - "float_val": { - "type": "number" - }, - "bool_val": { - "type": "boolean" - }, - "string_val": { - "type": "string", - "default": "abc" - } - } - }` - var jsonSchema jsonschema.Schema - err := json.Unmarshal([]byte(schemaJson), &jsonSchema) +func testConfig(t *testing.T) *config { + c, err := newConfig(context.Background(), "./testdata/config-test-schema/test-schema.json") require.NoError(t, err) - return &jsonSchema + return c } func TestTemplateConfigAssignValuesFromFile(t *testing.T) { - c := config{ - schema: testSchema(t), - values: make(map[string]any), - } + c := testConfig(t) err := c.assignValuesFromFile("./testdata/config-assign-from-file/config.json") assert.NoError(t, err) @@ -49,32 +27,17 @@ func TestTemplateConfigAssignValuesFromFile(t *testing.T) { assert.Equal(t, "hello", c.values["string_val"]) } -func TestTemplateConfigAssignValuesFromFileForUnknownField(t *testing.T) { - c := config{ - schema: testSchema(t), - values: make(map[string]any), - } - - err := c.assignValuesFromFile("./testdata/config-assign-from-file-unknown-property/config.json") - assert.EqualError(t, err, "unknown_prop is not defined as an input parameter for the template") -} - func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T) { - c := config{ - schema: testSchema(t), - values: make(map[string]any), - } + c := testConfig(t) err := c.assignValuesFromFile("./testdata/config-assign-from-file-invalid-int/config.json") - assert.EqualError(t, err, "failed to cast value abc of property int_val from file ./testdata/config-assign-from-file-invalid-int/config.json to an integer: cannot convert \"abc\" to an integer") + assert.EqualError(t, err, "failed to load config from file ./testdata/config-assign-from-file-invalid-int/config.json: failed to parse property int_val: cannot convert \"abc\" to an integer") } func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *testing.T) { - c := config{ - schema: testSchema(t), - values: map[string]any{ - "string_val": "this-is-not-overwritten", - }, + c := testConfig(t) + c.values = map[string]any{ + "string_val": "this-is-not-overwritten", } err := c.assignValuesFromFile("./testdata/config-assign-from-file/config.json") @@ -87,10 +50,7 @@ func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *te } func TestTemplateConfigAssignDefaultValues(t *testing.T) { - c := config{ - schema: testSchema(t), - values: make(map[string]any), - } + c := testConfig(t) err := c.assignDefaultValues() assert.NoError(t, err) @@ -101,65 +61,55 @@ func TestTemplateConfigAssignDefaultValues(t *testing.T) { } func TestTemplateConfigValidateValuesDefined(t *testing.T) { - c := config{ - schema: testSchema(t), - values: map[string]any{ - "int_val": 1, - "float_val": 1.0, - "bool_val": false, - }, + c := testConfig(t) + c.values = map[string]any{ + "int_val": 1, + "float_val": 1.0, + "bool_val": false, } - err := c.validateValuesDefined() - assert.EqualError(t, err, "no value has been assigned to input parameter string_val") + err := c.validate() + assert.EqualError(t, err, "validation for template input parameters failed. no value provided for required property string_val") } func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) { - c := &config{ - schema: testSchema(t), - values: map[string]any{ - "int_val": 1, - "float_val": 1.1, - "bool_val": true, - "string_val": "abcd", - }, + c := testConfig(t) + c.values = map[string]any{ + "int_val": 1, + "float_val": 1.1, + "bool_val": true, + "string_val": "abcd", } - err := c.validateValuesType() - assert.NoError(t, err) - - err = c.validate() + err := c.validate() assert.NoError(t, err) } func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) { - c := &config{ - schema: testSchema(t), - values: map[string]any{ - "unknown_prop": 1, - }, + c := testConfig(t) + c.values = map[string]any{ + "unknown_prop": 1, + "int_val": 1, + "float_val": 1.1, + "bool_val": true, + "string_val": "abcd", } - err := c.validateValuesType() - assert.EqualError(t, err, "unknown_prop is not defined as an input parameter for the template") + err := c.validate() + assert.EqualError(t, err, "validation for template input parameters failed. property unknown_prop is not defined in the schema") } func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) { - c := &config{ - schema: testSchema(t), - values: map[string]any{ - "int_val": "this-should-be-an-int", - "float_val": 1.1, - "bool_val": true, - "string_val": "abcd", - }, + c := testConfig(t) + c.values = map[string]any{ + "int_val": "this-should-be-an-int", + "float_val": 1.1, + "bool_val": true, + "string_val": "abcd", } - err := c.validateValuesType() - assert.EqualError(t, err, `incorrect type for int_val. expected type integer, but value is "this-should-be-an-int"`) - - err = c.validate() - assert.EqualError(t, err, `incorrect type for int_val. expected type integer, but value is "this-should-be-an-int"`) + err := c.validate() + assert.EqualError(t, err, "validation for template input parameters failed. incorrect type for property int_val: expected type integer, but value is \"this-should-be-an-int\"") } func TestTemplateValidateSchema(t *testing.T) { diff --git a/libs/template/testdata/config-test-schema/test-schema.json b/libs/template/testdata/config-test-schema/test-schema.json new file mode 100644 index 00000000000..41eb825190f --- /dev/null +++ b/libs/template/testdata/config-test-schema/test-schema.json @@ -0,0 +1,18 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "default": 123 + }, + "float_val": { + "type": "number" + }, + "bool_val": { + "type": "boolean" + }, + "string_val": { + "type": "string", + "default": "abc" + } + } +}