From d35de979cf6ffbc35d50a56dbe5c705545ec1c84 Mon Sep 17 00:00:00 2001 From: Brendan O'Brien Date: Thu, 20 Jul 2017 08:53:42 -0400 Subject: [PATCH 1/4] so, will that work? --- orders.go | 33 +++++++++++++++++++++ sql_datastore.go | 77 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 100 insertions(+), 10 deletions(-) create mode 100644 orders.go diff --git a/orders.go b/orders.go new file mode 100644 index 0000000..8d06adb --- /dev/null +++ b/orders.go @@ -0,0 +1,33 @@ +package sql_datastore + +import ( + "fmt" + "github.com/ipfs/go-datastore/query" +) + +// Order a query by a field +type OrderBy string + +// String value +func (o OrderBy) String() string { + return string(o) +} + +// satisfy datastore.Order interface, this is a no-op b/c sql sorting +// will happen at query-time +// TODO - In the future this should be generalized to facilitate supplying +// sql_datastore.OrderBy orders to other datastores, providing parity +func (o OrderBy) Sort([]query.Entry) {} + +// Order a query by a field, descending +type OrderByDesc string + +func (o OrderByDesc) String() string { + return fmt.Sprintf("%s DESC", o) +} + +// satisfy datastore.Order interface, this is a no-op b/c sql sorting +// will happen at query-time +// TODO - In the future this should be generalized to facilitate supplying +// sql_datastore.OrderBy orders to other datastores, providing parity +func (o OrderByDesc) Sort([]query.Entry) {} diff --git a/sql_datastore.go b/sql_datastore.go index 4b1a7f6..262dede 100644 --- a/sql_datastore.go +++ b/sql_datastore.go @@ -140,14 +140,12 @@ func (ds Datastore) Delete(key datastore.Key) error { // Currently it's required that the passed-in prefix be equal to DatastoreType() // which query will use to determine what model to ask for a ListCmd func (ds Datastore) Query(q query.Query) (query.Results, error) { + var rows *sql.Rows + // TODO - support query Filters if len(q.Filters) > 0 { return nil, fmt.Errorf("sql datastore queries do not support filters") } - // TODO - support query Orders - if len(q.Orders) > 0 { - return nil, fmt.Errorf("sql datastore queries do not support ordering") - } // TODO - support KeysOnly if q.KeysOnly { return nil, fmt.Errorf("sql datastore doesn't support keysonly ordering") @@ -161,18 +159,29 @@ func (ds Datastore) Query(q query.Query) (query.Results, error) { // This is totally janky, but will work for now. It's expected that // the returned CmdList will have at least 2 bindvars: - // $1 : LIMIT - // $2 : OFFSET + // $1 : LIMIT value + // $2 : OFFSET value + // if compatible Orders are provided to the query, the order string + // will be provided as a third bindvar: + // $3 : ORDERBY value // From there it can provide zero or more additional bindvars to // organize the query, which should be returned by the SQLParams method // TODO - this seems to hint at a need for some sort of Controller-like // pattern in userland. Have a think. - rows, err := ds.query(m, CmdList, q.Limit, q.Offset) - if err != nil { - return nil, err + os := orderString(q) + if os != "" { + rows, err = ds.query(m, CmdList, q.Limit, q.Offset, os) + if err != nil { + return nil, err + } + } else { + rows, err = ds.query(m, CmdList, q.Limit, q.Offset) + if err != nil { + return nil, err + } } - // TODO - should this be q.Limit or query.NormalBufferSize + // TODO - should this be q.Limit or query.NormalBufferSize? reschan := make(chan query.Result, q.Limit) go func() { defer close(reschan) @@ -199,6 +208,54 @@ func (ds Datastore) Query(q query.Query) (query.Results, error) { return query.ResultsWithChan(q, reschan), nil } +// orderString generates orders from any OrderBy or OrderByDesc values +func orderString(q query.Query) string { + if len(q.Orders) == 0 { + return "" + } + orders := []string{} + for _, o := range q.Orders { + // TODO - should this cross-correct by casting based on + // the outer order? + switch ot := o.(type) { + case query.OrderByValue: + switch obv := ot.TypedOrder.(type) { + case OrderBy: + orders = append(orders, obv.String()) + case OrderByDesc: + orders = append(orders, obv.String()) + } + case query.OrderByValueDescending: + switch obv := ot.TypedOrder.(type) { + case OrderBy: + orders = append(orders, obv.String()) + case OrderByDesc: + orders = append(orders, obv.String()) + } + // if ob, ok := ob.TypedOrder.(OrderBy); ok { + // } else if obd, ok := obv.TypedOrder.(OrderByDesc); ok { + // orders = append(orders, obd.String()) + // } + } + // } else if obvd, ok := o.(OrderByDesc); ok { + // if ob, ok := obvd.TypedOrder.(OrderBy); ok { + // orders = append(orders, ob.String()) + // } else if obd, ok := obvd.TypedOrder.(OrderByDesc); ok { + // orders = append(orders, obd.String()) + // } + // } + } + os := "" + for i, o := range orders { + if i == 0 { + os += o + } else { + os += fmt.Sprintf(", %s", o) + } + } + return os +} + // Batch commands are currently not supported func (ds *Datastore) Batch() (datastore.Batch, error) { return nil, datastore.ErrBatchUnsupported From 7d9019da4e34cd7e7fb134a7f330e023288882c8 Mon Sep 17 00:00:00 2001 From: Brendan O'Brien Date: Thu, 20 Jul 2017 10:09:37 -0400 Subject: [PATCH 2/4] BREAKING CHANGE: NewSQLModel is now passed the whole key --- model.go | 19 ++++++++++++------- sql_datastore.go | 17 +++-------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/model.go b/model.go index f0a95f5..6d3e228 100644 --- a/model.go +++ b/model.go @@ -2,6 +2,7 @@ package sql_datastore import ( "github.com/datatogether/sqlutil" + datastore "github.com/ipfs/go-datastore" ) // Model is the interface that must be implemented to work @@ -12,23 +13,27 @@ type Model interface { // DatastoreType must return the "type" of object, which is a consistent // name for the object being stored. DatastoreType works in conjunction // with GetId to construct the key for storage. - // Since SQL doesn't support the "pathing" aspect of keys, any path - // values are ignored DatastoreType() string - // GetId should return the cannonical ID for the object. + // GetId should return the standalone cannonical ID for the object. GetId() string - // While not explicitly required by this package, most implementations - // will want to have a "Key" method that combines DatastoreType() and GetId() + // Key is a methoda that traditionally combines DatastoreType() and GetId() // to form a key that can be provided to Get & Has commands // eg: // func (m) Key() datastore.Key { // return datastore.NewKey(fmt.Sprintf("%s:%s", m.DatastoreType(), m.GetId())) // } + // in examples of "submodels" of another model it makes sense to leverage the + // POSIX structure of keys. for example: + // func (m) Key() datastore.Key { + // return datastore.NewKey(fmt.Sprintf("%s:%s/%s", m.DatastoreType(), m.ParentId(), m.GetId())) + // } + Key() datastore.Key // NewSQLModel must allocate & return a new instance of the - // model with id set such that GetId returns the passed-in id string - NewSQLModel(id string) Model + // model with id set such that GetId returns the passed-in Key + // NewSQLModel will be passed keys for creation of new blank models + NewSQLModel(key datastore.Key) Model // SQLQuery gives the datastore the query to execute for a given command type // As an example, if CmdSelectOne is passed in, something like diff --git a/sql_datastore.go b/sql_datastore.go index 262dede..daceae7 100644 --- a/sql_datastore.go +++ b/sql_datastore.go @@ -100,7 +100,7 @@ func (ds Datastore) Get(key datastore.Key) (value interface{}, err error) { return nil, err } - v := m.NewSQLModel(key.Name()) + v := m.NewSQLModel(key) if err := v.UnmarshalSQL(row); err != nil { return nil, err } @@ -188,7 +188,7 @@ func (ds Datastore) Query(q query.Query) (query.Results, error) { for rows.Next() { - model := m.NewSQLModel("") + model := m.NewSQLModel(datastore.NewKey("")) if err := model.UnmarshalSQL(rows); err != nil { reschan <- query.Result{ Error: err, @@ -232,18 +232,7 @@ func orderString(q query.Query) string { case OrderByDesc: orders = append(orders, obv.String()) } - // if ob, ok := ob.TypedOrder.(OrderBy); ok { - // } else if obd, ok := obv.TypedOrder.(OrderByDesc); ok { - // orders = append(orders, obd.String()) - // } } - // } else if obvd, ok := o.(OrderByDesc); ok { - // if ob, ok := obvd.TypedOrder.(OrderBy); ok { - // orders = append(orders, ob.String()) - // } else if obd, ok := obvd.TypedOrder.(OrderByDesc); ok { - // orders = append(orders, obd.String()) - // } - // } } os := "" for i, o := range orders { @@ -265,7 +254,7 @@ func (ds Datastore) modelForKey(key datastore.Key) (Model, error) { for _, m := range ds.models { if m.DatastoreType() == key.Type() { // return a model with "ID" set to the key param - return m.NewSQLModel(key.Name()), nil + return m.NewSQLModel(key), nil } } return nil, fmt.Errorf("no usable model found for key, did you call register on the model?: %s", key.String()) From bbed3cc90e8bd96e8cdd60997962c09e1298471a Mon Sep 17 00:00:00 2001 From: Brendan O'Brien Date: Thu, 20 Jul 2017 17:06:40 -0400 Subject: [PATCH 3/4] support query filtering for model type --- query_filters.go | 23 +++++++++++++++++++++++ orders.go => query_orders.go | 0 sql_datastore.go | 29 +++++++++++++++++++++-------- 3 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 query_filters.go rename orders.go => query_orders.go (100%) diff --git a/query_filters.go b/query_filters.go new file mode 100644 index 0000000..ac4d243 --- /dev/null +++ b/query_filters.go @@ -0,0 +1,23 @@ +package sql_datastore + +import ( + "fmt" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" +) + +// Filter +type FilterKeyTypeEq string + +func (f FilterKeyTypeEq) Key() datastore.Key { + return datastore.NewKey(fmt.Sprintf("/%s:", f.String())) +} + +func (f FilterKeyTypeEq) String() string { + return string(f) +} + +// TODO - make this work properly for the sake of other datastores +func (f FilterKeyTypeEq) Filter(e query.Entry) bool { + return true +} diff --git a/orders.go b/query_orders.go similarity index 100% rename from orders.go rename to query_orders.go diff --git a/sql_datastore.go b/sql_datastore.go index daceae7..62a5cd7 100644 --- a/sql_datastore.go +++ b/sql_datastore.go @@ -142,21 +142,22 @@ func (ds Datastore) Delete(key datastore.Key) error { func (ds Datastore) Query(q query.Query) (query.Results, error) { var rows *sql.Rows - // TODO - support query Filters - if len(q.Filters) > 0 { - return nil, fmt.Errorf("sql datastore queries do not support filters") - } // TODO - support KeysOnly if q.KeysOnly { - return nil, fmt.Errorf("sql datastore doesn't support keysonly ordering") + return nil, fmt.Errorf("sql datastore doesn't support keysonly querying") } - // TODO - ugh this so bad - m, err := ds.modelForKey(datastore.NewKey(fmt.Sprintf("/%s:", q.Prefix))) + // determine what type of model we're querying for + m, err := ds.modelForQuery(q) if err != nil { return nil, err } + // here we attach query information to a new model + // TODO - currently this is basing off of the prefix, might need + // to do smarter things in relation to filters? + m = m.NewSQLModel(datastore.NewKey(q.Prefix)) + // This is totally janky, but will work for now. It's expected that // the returned CmdList will have at least 2 bindvars: // $1 : LIMIT value @@ -188,7 +189,7 @@ func (ds Datastore) Query(q query.Query) (query.Results, error) { for rows.Next() { - model := m.NewSQLModel(datastore.NewKey("")) + model := m.NewSQLModel(datastore.NewKey(q.Prefix)) if err := model.UnmarshalSQL(rows); err != nil { reschan <- query.Result{ Error: err, @@ -208,6 +209,17 @@ func (ds Datastore) Query(q query.Query) (query.Results, error) { return query.ResultsWithChan(q, reschan), nil } +// modelForQuery determines what type of model this query should return +func (ds Datastore) modelForQuery(q query.Query) (Model, error) { + for _, f := range q.Filters { + switch t := f.(type) { + case FilterKeyTypeEq: + return ds.modelForKey(t.Key()) + } + } + return ds.modelForKey(datastore.NewKey(fmt.Sprintf("/%s:", q.Prefix))) +} + // orderString generates orders from any OrderBy or OrderByDesc values func orderString(q query.Query) string { if len(q.Orders) == 0 { @@ -300,6 +312,7 @@ func (ds Datastore) query(m Model, t Cmd, prebind ...interface{}) (*sql.Rows, er if err != nil { return nil, err } + return ds.DB.Query(query, append(prebind, params...)...) } From b0c91273ed161344348fb9c89e90a7b4edcb3a68 Mon Sep 17 00:00:00 2001 From: Brendan O'Brien Date: Fri, 21 Jul 2017 10:17:34 -0400 Subject: [PATCH 4/4] docs or it didn't happen --- cmd.go | 5 +- query_filters.go | 6 ++- query_orders.go | 3 +- readme.md | 124 +++++++++++++++++++++++++++++++++++++++++++---- sql_datastore.go | 21 +++++++- 5 files changed, 145 insertions(+), 14 deletions(-) diff --git a/cmd.go b/cmd.go index a3bf7fb..76d40e6 100644 --- a/cmd.go +++ b/cmd.go @@ -2,11 +2,12 @@ package sql_datastore // Cmd represents a set of standardized SQL queries these abstractions // define a common set of commands that a model can provide to sql_datastore -// for execution +// for execution. type Cmd int const ( - // Unknown as default, errored state + // Unknown as default, errored state. CmdUnknown should never + // be intentionally passed to... anything. CmdUnknown Cmd = iota // starting with DDL statements: // CREATE TABLE query diff --git a/query_filters.go b/query_filters.go index ac4d243..5c140d4 100644 --- a/query_filters.go +++ b/query_filters.go @@ -6,17 +6,21 @@ import ( "github.com/ipfs/go-datastore/query" ) -// Filter +// FilterTypeEq filters for a specific key Type (which should match a registerd model on the sql_datastore.Datastore) +// FilterTypeEq is a string that specifies the key type we're after type FilterKeyTypeEq string +// Key return s FilterKeyTypeEq formatted as a datastore.Key func (f FilterKeyTypeEq) Key() datastore.Key { return datastore.NewKey(fmt.Sprintf("/%s:", f.String())) } +// Satisfy the Stringer interface func (f FilterKeyTypeEq) String() string { return string(f) } +// Filter satisfies the query.Filter interface // TODO - make this work properly for the sake of other datastores func (f FilterKeyTypeEq) Filter(e query.Entry) bool { return true diff --git a/query_orders.go b/query_orders.go index 8d06adb..b32f6aa 100644 --- a/query_orders.go +++ b/query_orders.go @@ -8,7 +8,7 @@ import ( // Order a query by a field type OrderBy string -// String value +// String value, used to inject the field name istself as a SQL query param func (o OrderBy) String() string { return string(o) } @@ -22,6 +22,7 @@ func (o OrderBy) Sort([]query.Entry) {} // Order a query by a field, descending type OrderByDesc string +// String value, used to inject the field name istself as a SQL query param func (o OrderByDesc) String() string { return fmt.Sprintf("%s DESC", o) } diff --git a/readme.md b/readme.md index e014b1d..9e14d87 100644 --- a/readme.md +++ b/readme.md @@ -14,6 +14,18 @@ requiring implementers to provide a standard set of queries and parameters to glue everything together. Whenever the datastore interface is not expressive enough, one can always fall back to standard SQL work. +sql_datastore reconciles the key-value orientation of the datastore interface +with the tables/relational orientation of SQL databases through the concept of a +"Model". Model is a bit of an unfortunate name, as it implies this package is an +ORM, which isn't a design goal. + +Annnnnnnnyway, the important patterns of this approach are: + + 1. The Model interface defines how to get stuff into and out of SQL + 2. All Models that will be interacted with must be "Registered" to the store. + Registered Models map to a datastore.Key Type. + 3. All Get/Put/Delete/Has/Query to sql_datastore must map to a single Model + This implementation leads to a great deal of required boilerplate code to implement. In the future this package could be expanded to become syntax-aware, accepting a table name & schema definition for registered models. From here the @@ -33,6 +45,7 @@ Package Level Datastore. Be sure to call SetDB before using! ```go func Register(models ...Model) error ``` +Register a number of models to the DefaultStore #### func SetDB @@ -48,11 +61,12 @@ type Cmd int ``` Cmd represents a set of standardized SQL queries these abstractions define a -common set of commands that a model can provide to sql_datastore for execution +common set of commands that a model can provide to sql_datastore for execution. ```go const ( - // Unknown as default, errored state + // Unknown as default, errored state. CmdUnknown should never + // be intentionally passed to... anything. CmdUnknown Cmd = iota // starting with DDL statements: // CREATE TABLE query @@ -89,7 +103,7 @@ type Datastore struct { } ``` -Datastore +Datastore implements the ipfs datastore interface for SQL databases #### func NewDatastore @@ -152,6 +166,38 @@ func (ds *Datastore) Register(models ...Model) error Register one or more models that will be used by this datastore. Must be called before a model can be manipulated by the store +#### type FilterKeyTypeEq + +```go +type FilterKeyTypeEq string +``` + +FilterTypeEq filters for a specific key Type (which should match a registerd +model on the sql_datastore.Datastore) FilterTypeEq is a string that specifies +the key type we're after + +#### func (FilterKeyTypeEq) Filter + +```go +func (f FilterKeyTypeEq) Filter(e query.Entry) bool +``` +Filter satisfies the query.Filter interface TODO - make this work properly for +the sake of other datastores + +#### func (FilterKeyTypeEq) Key + +```go +func (f FilterKeyTypeEq) Key() datastore.Key +``` +Key return s FilterKeyTypeEq formatted as a datastore.Key + +#### func (FilterKeyTypeEq) String + +```go +func (f FilterKeyTypeEq) String() string +``` +Satisfy the Stringer interface + #### type Model ```go @@ -159,15 +205,27 @@ type Model interface { // DatastoreType must return the "type" of object, which is a consistent // name for the object being stored. DatastoreType works in conjunction // with GetId to construct the key for storage. - // Since SQL doesn't support the "pathing" aspect of keys, any path - // values are ignored DatastoreType() string - // GetId should return the cannonical ID for the object. + // GetId should return the standalone cannonical ID for the object. GetId() string + // Key is a methoda that traditionally combines DatastoreType() and GetId() + // to form a key that can be provided to Get & Has commands + // eg: + // func (m) Key() datastore.Key { + // return datastore.NewKey(fmt.Sprintf("%s:%s", m.DatastoreType(), m.GetId())) + // } + // in examples of "submodels" of another model it makes sense to leverage the + // POSIX structure of keys. for example: + // func (m) Key() datastore.Key { + // return datastore.NewKey(fmt.Sprintf("%s:%s/%s", m.DatastoreType(), m.ParentId(), m.GetId())) + // } + Key() datastore.Key + // NewSQLModel must allocate & return a new instance of the - // model with id set such that GetId returns the passed-in id string - NewSQLModel(id string) Model + // model with id set such that GetId returns the passed-in Key + // NewSQLModel will be passed keys for creation of new blank models + NewSQLModel(key datastore.Key) Model // SQLQuery gives the datastore the query to execute for a given command type // As an example, if CmdSelectOne is passed in, something like @@ -187,4 +245,52 @@ type Model interface { Model is the interface that must be implemented to work with sql_datastore. There are some fairly heavy constraints here. For a working example checkout: -https://github.com/datatogether/archive and have a look at primer.go +https://github.com/archivers-space/archive and have a look at primer.go + +#### type OrderBy + +```go +type OrderBy string +``` + +Order a query by a field + +#### func (OrderBy) Sort + +```go +func (o OrderBy) Sort([]query.Entry) +``` +satisfy datastore.Order interface, this is a no-op b/c sql sorting will happen +at query-time TODO - In the future this should be generalized to facilitate +supplying sql_datastore.OrderBy orders to other datastores, providing parity + +#### func (OrderBy) String + +```go +func (o OrderBy) String() string +``` +String value, used to inject the field name istself as a SQL query param + +#### type OrderByDesc + +```go +type OrderByDesc string +``` + +Order a query by a field, descending + +#### func (OrderByDesc) Sort + +```go +func (o OrderByDesc) Sort([]query.Entry) +``` +satisfy datastore.Order interface, this is a no-op b/c sql sorting will happen +at query-time TODO - In the future this should be generalized to facilitate +supplying sql_datastore.OrderBy orders to other datastores, providing parity + +#### func (OrderByDesc) String + +```go +func (o OrderByDesc) String() string +``` +String value, used to inject the field name istself as a SQL query param diff --git a/sql_datastore.go b/sql_datastore.go index 62a5cd7..b7ffda2 100644 --- a/sql_datastore.go +++ b/sql_datastore.go @@ -12,6 +12,17 @@ // together. Whenever the datastore interface is not expressive enough, // one can always fall back to standard SQL work. // +// sql_datastore reconciles the key-value orientation of the datastore +// interface with the tables/relational orientation of SQL databases +// through the concept of a "Model". Model is a bit of an unfortunate name, +// as it implies this package is an ORM, which isn't a design goal. +// +// Annnnnnnnyway, the important patterns of this approach are: +// 1. The Model interface defines how to get stuff into and out of SQL +// 2. All Models that will be interacted with must be "Registered" to the store. +// Registered Models map to a datastore.Key Type. +// 3. All Get/Put/Delete/Has/Query to sql_datastore must map to a single Model +// // This implementation leads to a great deal of required boilerplate code // to implement. In the future this package could be expanded to become // syntax-aware, accepting a table name & schema definition for registered @@ -37,11 +48,12 @@ func SetDB(db *sql.DB) { DefaultStore.DB = db } +// Register a number of models to the DefaultStore func Register(models ...Model) error { return DefaultStore.Register(models...) } -// Datastore +// Datastore implements the ipfs datastore interface for SQL databases type Datastore struct { // DB is the underlying DB handler // it should be safe for use outside of the @@ -262,6 +274,7 @@ func (ds *Datastore) Batch() (datastore.Batch, error) { return nil, datastore.ErrBatchUnsupported } +// for a given key, determine what kind of Model we're looking for func (ds Datastore) modelForKey(key datastore.Key) (Model, error) { for _, m := range ds.models { if m.DatastoreType() == key.Type() { @@ -272,6 +285,7 @@ func (ds Datastore) modelForKey(key datastore.Key) (Model, error) { return nil, fmt.Errorf("no usable model found for key, did you call register on the model?: %s", key.String()) } +// does this datastore func (ds Datastore) hasModel(m Model) (exists bool, err error) { row, err := ds.queryRow(m, CmdExistsOne) if err != nil { @@ -281,6 +295,7 @@ func (ds Datastore) hasModel(m Model) (exists bool, err error) { return } +// execute a Cmd against a given model func (ds Datastore) exec(m Model, t Cmd) error { if ds.DB == nil { return fmt.Errorf("datastore has no DB") @@ -293,6 +308,7 @@ func (ds Datastore) exec(m Model, t Cmd) error { return err } +// query for a single row, given a type of command and model func (ds Datastore) queryRow(m Model, t Cmd) (*sql.Row, error) { if ds.DB == nil { return nil, fmt.Errorf("datastore has no DB") @@ -304,6 +320,8 @@ func (ds Datastore) queryRow(m Model, t Cmd) (*sql.Row, error) { return ds.DB.QueryRow(query, params...), nil } +// run a query against the db for a given command and model, with optionally prebound +// arguments derived from the query func (ds Datastore) query(m Model, t Cmd, prebind ...interface{}) (*sql.Rows, error) { if ds.DB == nil { return nil, fmt.Errorf("datastore has no DB") @@ -316,6 +334,7 @@ func (ds Datastore) query(m Model, t Cmd, prebind ...interface{}) (*sql.Rows, er return ds.DB.Query(query, append(prebind, params...)...) } +// prepare a query, grabbing the command sql & params from the model func (ds Datastore) prepQuery(m Model, t Cmd) (string, []interface{}, error) { query := m.SQLQuery(t) if query == "" {