core: one type row per expression, canonicalized per dialect, across every engine (#4618)

Author: kyleconroy

docs/howto/analyze.md

@@ -71,7 +71,7 @@ reports the result columns and parameters:
       {
         "name": "id",
         "type": {
-          "name": "bigserial"
+          "name": "bigint"
         },
         "table": "authors"
       },
@@ -97,7 +97,7 @@ reports the result columns and parameters:
         "column": {
           "name": "id",
           "type": {
-            "name": "bigserial"
+            "name": "bigint"
           },
           "table": "authors"
         }
@@ -109,10 +109,20 @@ reports the result columns and parameters:
 
 A column's `type` is written as a call expression: a `name` applied to
 `args`, each of which carries an optional `label` and exactly one of `type`,
-`int`, `bool` or `string`, with `nullable` set at whatever depth it applies.
-An array of text is `array` applied to `text`; a `Map(String, Nullable(UInt8))`
-in ClickHouse is `map` applied to `string` and a nullable `uint8`. Names are
-recorded as the engine reports them.
+`int`, `bool`, `string` or `ident`, with `nullable` set at whatever depth it
+applies. A `numeric(10,2)` column is `numeric` applied to `10` and `2`; an
+array of text is `array` applied to `text`, and an array of arrays nests
+one `array` per dimension; a `Map(String, Nullable(UInt8))` in ClickHouse is
+`map` applied to `string` and a nullable `uint8`; a `STRUCT<a INT64>` in
+GoogleSQL is `struct` applied to an `int64` labelled `a`; the `MAX` of SQL
+Server's `nvarchar(max)` is the identifier `max`.
+
+Types are reported the way the engine itself stores and reports them rather
+than the way the schema spelled them: PostgreSQL's `int` and `bigserial` are
+`integer` and `bigint`, as `format_type` prints them; MySQL's `BOOLEAN` is
+`tinyint(1)`; ClickHouse's `Decimal32(4)` is `decimal(9, 4)`; DuckDB's
+`TEXT` is `varchar`; SQL Server's `FLOAT(24)` is `real`. SQLite, which
+keeps a declared type as written, is reported as written.
 
 Pass `--ast` to also include each statement's parsed AST under an `ast` key. It
 has the same shape as the output of [`parse`](parse.md), with every node tagged

internal/codegen/golang/postgresql_type.go

@@ -216,7 +216,7 @@ func postgresType(req *plugin.GenerateRequest, options *opts.Options, col *plugi
 		}
 		return "sql.NullTime"
 
-	case "pg_catalog.time":
+	case "pg_catalog.time", "time", "time without time zone":
 		if driver == opts.SQLDriverPGXV5 {
 			return "pgtype.Time"
 		}
@@ -228,7 +228,7 @@ func postgresType(req *plugin.GenerateRequest, options *opts.Options, col *plugi
 		}
 		return "sql.NullTime"
 
-	case "pg_catalog.timetz":
+	case "pg_catalog.timetz", "timetz", "time with time zone":
 		if notNull {
 			return "time.Time"
 		}
@@ -237,7 +237,7 @@ func postgresType(req *plugin.GenerateRequest, options *opts.Options, col *plugi
 		}
 		return "sql.NullTime"
 
-	case "pg_catalog.timestamp", "timestamp":
+	case "pg_catalog.timestamp", "timestamp", "timestamp without time zone":
 		if driver == opts.SQLDriverPGXV5 {
 			return "pgtype.Timestamp"
 		}
@@ -249,7 +249,7 @@ func postgresType(req *plugin.GenerateRequest, options *opts.Options, col *plugi
 		}
 		return "sql.NullTime"
 
-	case "pg_catalog.timestamptz", "timestamptz":
+	case "pg_catalog.timestamptz", "timestamptz", "timestamp with time zone":
 		if driver == opts.SQLDriverPGXV5 {
 			return "pgtype.Timestamptz"
 		}
@@ -261,7 +261,7 @@ func postgresType(req *plugin.GenerateRequest, options *opts.Options, col *plugi
 		}
 		return "sql.NullTime"
 
-	case "text", "pg_catalog.varchar", "pg_catalog.bpchar", "string", "citext", "name":
+	case "text", "pg_catalog.varchar", "varchar", "character varying", "pg_catalog.bpchar", "bpchar", "character", "string", "citext", "name":
 		if notNull {
 			return "string"
 		}
@@ -470,7 +470,7 @@ func postgresType(req *plugin.GenerateRequest, options *opts.Options, col *plugi
 		}
 		return "any"
 
-	case "bit", "varbit", "pg_catalog.bit", "pg_catalog.varbit":
+	case "bit", "varbit", "bit varying", "pg_catalog.bit", "pg_catalog.varbit":
 		if driver == opts.SQLDriverPGXV5 {
 			return "pgtype.Bits"
 		}

internal/compiler/catalog_core.go

@@ -32,19 +32,25 @@ func coreResultCatalog(c *core.Catalog) (*catalog.Catalog, error) {
 			}
 			t := &catalog.Table{Rel: &ast.TableName{Schema: ns.Name, Name: table.Name}}
 			for _, col := range cols {
-				// The catalog names an array type after its element with the
-				// suffix appended, which is codegen's data type and array
-				// flag in one string. The core catalog holds one dimension,
-				// and codegen renders a "[]" per dimension.
-				dataType, isArray := strings.CutSuffix(col.TypeName, core.ArraySuffix)
+				// Codegen reads a data type and an array flag, and renders
+				// a "[]" per dimension, so an array of arrays of integers
+				// is the type integer with two dimensions.
+				expr, err := c.TypeExprOf(col.TypeOID)
+				if err != nil {
+					return nil, err
+				}
+				inner := expr.Innermost()
 				column := &catalog.Column{
-					Name:      col.Name,
-					Type:      ast.TypeName{Name: dataType},
-					IsNotNull: col.NotNull,
-					IsArray:   isArray,
+					Name:       col.Name,
+					Type:       ast.TypeName{Name: strings.TrimSuffix(inner.Name, " unsigned")},
+					IsNotNull:  col.NotNull,
+					IsArray:    expr.IsArray(),
+					ArrayDims:  expr.ArrayDims(),
+					IsUnsigned: strings.HasSuffix(inner.Name, " unsigned"),
 				}
-				if isArray {
-					column.ArrayDims = 1
+				if len(inner.Args) > 0 && inner.Args[0].Int != nil {
+					l := int(*inner.Args[0].Int)
+					column.Length = &l
 				}
 				t.Columns = append(t.Columns, column)
 			}

internal/compiler/parse_core.go

@@ -107,21 +107,33 @@ func coreColumn(c core.Column) *Column {
 		IsArray:  c.IsArray,
 		TypeExpr: c.Type,
 	}
-	// The core reports arrays without dimensions, and codegen renders one
-	// "[]" per dimension.
-	if c.IsArray {
-		col.ArrayDims = 1
-	}
+	describeType(col, c.Type)
 	if c.Source != nil && c.Source.Table != "" {
 		col.Table = &ast.TableName{Schema: c.Source.Schema, Name: c.Source.Table}
 		col.TableAlias = c.Source.TableAlias
 		col.OriginalName = c.Source.Column
 	}
-	if c.TypeLength > 0 {
-		l := c.TypeLength
+	return col
+}
+
+// describeType fills in what codegen reads about a type from its
+// expression: one array dimension per nesting, the length that is the
+// innermost type's first integer argument (which is how a MySQL tinyint(1)
+// is told from a tinyint), and whether the innermost type is unsigned.
+func describeType(col *Column, t *core.TypeExpr) {
+	if t == nil {
+		if col.IsArray {
+			col.ArrayDims = 1
+		}
+		return
+	}
+	col.ArrayDims = t.ArrayDims()
+	inner := t.Innermost()
+	if len(inner.Args) > 0 && inner.Args[0].Int != nil {
+		l := int(*inner.Args[0].Int)
 		col.Length = &l
 	}
-	return col
+	col.Unsigned = strings.HasSuffix(inner.Name, " unsigned")
 }
 
 func coreParamColumn(p core.Parameter, params *named.ParamSet) *Column {
@@ -132,9 +144,7 @@ func coreParamColumn(p core.Parameter, params *named.ParamSet) *Column {
 		IsArray:  p.IsArray,
 		TypeExpr: p.Type,
 	}
-	if p.IsArray {
-		col.ArrayDims = 1
-	}
+	describeType(col, p.Type)
 	if p.Source != nil && p.Source.Table != "" {
 		col.Table = &ast.TableName{Schema: p.Source.Schema, Name: p.Source.Table}
 		col.OriginalName = p.Source.Column

internal/core/analysis.go

@@ -65,8 +65,6 @@ type Column struct {
 	SourceAttributeOID int64         `json:"source_attribute_oid,omitempty"`
 	Source             *ColumnSource `json:"source,omitempty"`
 	DeclType           string        `json:"decl_type,omitempty"`
-	TypeLength         int           `json:"type_length,omitempty"`
-	TypeScale          int           `json:"type_scale,omitempty"`
 	IsPrimaryKey       bool          `json:"is_primary_key,omitempty"`
 	IsUnique           bool          `json:"is_unique,omitempty"`
 	IsAutoIncrement    bool          `json:"is_auto_increment,omitempty"`

internal/core/analyzer/analyzer.go

@@ -112,6 +112,7 @@ func derivedRel(alias string, cols []core.Column) scopeRel {
 			AttOID:  col.SourceAttributeOID,
 			Name:    col.Name,
 			TypeOID: col.TypeOID,
+			Type:    col.Type.WithNullable(false),
 			NotNull: col.NotNull,
 		})
 	}
@@ -123,12 +124,12 @@ func (a *analyzer) result() core.PrepareResult {
 	// the dialect has such a type.
 	if oid, ok := a.cat.UntypedTypeOID(); ok {
 		for n, p := range a.params {
-			if p.TypeOID == 0 && p.DataType == "" {
+			if p.TypeOID == 0 && p.Type == nil {
 				t := exprType{typeOID: oid, nullable: true}
 				p.TypeOID = oid
 				p.DataType, p.IsArray = a.typeNameOf(t)
 				p.NotNull = false
-				p.Type = a.typeExprOf(t, "")
+				p.Type = a.typeExprOf(t)
 				a.params[n] = p
 			}
 		}

internal/core/analyzer/dml.go

@@ -202,6 +202,7 @@ func findColumn(rel scopeRel, name string) (core.ClassColumn, bool) {
 func columnType(rel scopeRel, col core.ClassColumn) exprType {
 	return exprType{
 		typeOID:            col.TypeOID,
+		expr:               col.Type,
 		nullable:           !col.NotNull,
 		sourceClassOID:     rel.classOID,
 		sourceAttributeOID: col.AttOID,