diff --git a/cmd.go b/cmd.go index a3bf7fb..76d40e6 100644 --- a/cmd.go +++ b/cmd.go @@ -2,11 +2,12 @@ package sql_datastore // Cmd represents a set of standardized SQL queries these abstractions // define a common set of commands that a model can provide to sql_datastore -// for execution +// for execution. type Cmd int const ( - // Unknown as default, errored state + // Unknown as default, errored state. CmdUnknown should never + // be intentionally passed to... anything. CmdUnknown Cmd = iota // starting with DDL statements: // CREATE TABLE query diff --git a/model.go b/model.go index f0a95f5..6d3e228 100644 --- a/model.go +++ b/model.go @@ -2,6 +2,7 @@ package sql_datastore import ( "github.com/datatogether/sqlutil" + datastore "github.com/ipfs/go-datastore" ) // Model is the interface that must be implemented to work @@ -12,23 +13,27 @@ type Model interface { // DatastoreType must return the "type" of object, which is a consistent // name for the object being stored. DatastoreType works in conjunction // with GetId to construct the key for storage. - // Since SQL doesn't support the "pathing" aspect of keys, any path - // values are ignored DatastoreType() string - // GetId should return the cannonical ID for the object. + // GetId should return the standalone cannonical ID for the object. GetId() string - // While not explicitly required by this package, most implementations - // will want to have a "Key" method that combines DatastoreType() and GetId() + // Key is a methoda that traditionally combines DatastoreType() and GetId() // to form a key that can be provided to Get & Has commands // eg: // func (m) Key() datastore.Key { // return datastore.NewKey(fmt.Sprintf("%s:%s", m.DatastoreType(), m.GetId())) // } + // in examples of "submodels" of another model it makes sense to leverage the + // POSIX structure of keys. for example: + // func (m) Key() datastore.Key { + // return datastore.NewKey(fmt.Sprintf("%s:%s/%s", m.DatastoreType(), m.ParentId(), m.GetId())) + // } + Key() datastore.Key // NewSQLModel must allocate & return a new instance of the - // model with id set such that GetId returns the passed-in id string - NewSQLModel(id string) Model + // model with id set such that GetId returns the passed-in Key + // NewSQLModel will be passed keys for creation of new blank models + NewSQLModel(key datastore.Key) Model // SQLQuery gives the datastore the query to execute for a given command type // As an example, if CmdSelectOne is passed in, something like diff --git a/query_filters.go b/query_filters.go new file mode 100644 index 0000000..5c140d4 --- /dev/null +++ b/query_filters.go @@ -0,0 +1,27 @@ +package sql_datastore + +import ( + "fmt" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" +) + +// FilterTypeEq filters for a specific key Type (which should match a registerd model on the sql_datastore.Datastore) +// FilterTypeEq is a string that specifies the key type we're after +type FilterKeyTypeEq string + +// Key return s FilterKeyTypeEq formatted as a datastore.Key +func (f FilterKeyTypeEq) Key() datastore.Key { + return datastore.NewKey(fmt.Sprintf("/%s:", f.String())) +} + +// Satisfy the Stringer interface +func (f FilterKeyTypeEq) String() string { + return string(f) +} + +// Filter satisfies the query.Filter interface +// TODO - make this work properly for the sake of other datastores +func (f FilterKeyTypeEq) Filter(e query.Entry) bool { + return true +} diff --git a/query_orders.go b/query_orders.go new file mode 100644 index 0000000..b32f6aa --- /dev/null +++ b/query_orders.go @@ -0,0 +1,34 @@ +package sql_datastore + +import ( + "fmt" + "github.com/ipfs/go-datastore/query" +) + +// Order a query by a field +type OrderBy string + +// String value, used to inject the field name istself as a SQL query param +func (o OrderBy) String() string { + return string(o) +} + +// satisfy datastore.Order interface, this is a no-op b/c sql sorting +// will happen at query-time +// TODO - In the future this should be generalized to facilitate supplying +// sql_datastore.OrderBy orders to other datastores, providing parity +func (o OrderBy) Sort([]query.Entry) {} + +// Order a query by a field, descending +type OrderByDesc string + +// String value, used to inject the field name istself as a SQL query param +func (o OrderByDesc) String() string { + return fmt.Sprintf("%s DESC", o) +} + +// satisfy datastore.Order interface, this is a no-op b/c sql sorting +// will happen at query-time +// TODO - In the future this should be generalized to facilitate supplying +// sql_datastore.OrderBy orders to other datastores, providing parity +func (o OrderByDesc) Sort([]query.Entry) {} diff --git a/readme.md b/readme.md index e014b1d..9e14d87 100644 --- a/readme.md +++ b/readme.md @@ -14,6 +14,18 @@ requiring implementers to provide a standard set of queries and parameters to glue everything together. Whenever the datastore interface is not expressive enough, one can always fall back to standard SQL work. +sql_datastore reconciles the key-value orientation of the datastore interface +with the tables/relational orientation of SQL databases through the concept of a +"Model". Model is a bit of an unfortunate name, as it implies this package is an +ORM, which isn't a design goal. + +Annnnnnnnyway, the important patterns of this approach are: + + 1. The Model interface defines how to get stuff into and out of SQL + 2. All Models that will be interacted with must be "Registered" to the store. + Registered Models map to a datastore.Key Type. + 3. All Get/Put/Delete/Has/Query to sql_datastore must map to a single Model + This implementation leads to a great deal of required boilerplate code to implement. In the future this package could be expanded to become syntax-aware, accepting a table name & schema definition for registered models. From here the @@ -33,6 +45,7 @@ Package Level Datastore. Be sure to call SetDB before using! ```go func Register(models ...Model) error ``` +Register a number of models to the DefaultStore #### func SetDB @@ -48,11 +61,12 @@ type Cmd int ``` Cmd represents a set of standardized SQL queries these abstractions define a -common set of commands that a model can provide to sql_datastore for execution +common set of commands that a model can provide to sql_datastore for execution. ```go const ( - // Unknown as default, errored state + // Unknown as default, errored state. CmdUnknown should never + // be intentionally passed to... anything. CmdUnknown Cmd = iota // starting with DDL statements: // CREATE TABLE query @@ -89,7 +103,7 @@ type Datastore struct { } ``` -Datastore +Datastore implements the ipfs datastore interface for SQL databases #### func NewDatastore @@ -152,6 +166,38 @@ func (ds *Datastore) Register(models ...Model) error Register one or more models that will be used by this datastore. Must be called before a model can be manipulated by the store +#### type FilterKeyTypeEq + +```go +type FilterKeyTypeEq string +``` + +FilterTypeEq filters for a specific key Type (which should match a registerd +model on the sql_datastore.Datastore) FilterTypeEq is a string that specifies +the key type we're after + +#### func (FilterKeyTypeEq) Filter + +```go +func (f FilterKeyTypeEq) Filter(e query.Entry) bool +``` +Filter satisfies the query.Filter interface TODO - make this work properly for +the sake of other datastores + +#### func (FilterKeyTypeEq) Key + +```go +func (f FilterKeyTypeEq) Key() datastore.Key +``` +Key return s FilterKeyTypeEq formatted as a datastore.Key + +#### func (FilterKeyTypeEq) String + +```go +func (f FilterKeyTypeEq) String() string +``` +Satisfy the Stringer interface + #### type Model ```go @@ -159,15 +205,27 @@ type Model interface { // DatastoreType must return the "type" of object, which is a consistent // name for the object being stored. DatastoreType works in conjunction // with GetId to construct the key for storage. - // Since SQL doesn't support the "pathing" aspect of keys, any path - // values are ignored DatastoreType() string - // GetId should return the cannonical ID for the object. + // GetId should return the standalone cannonical ID for the object. GetId() string + // Key is a methoda that traditionally combines DatastoreType() and GetId() + // to form a key that can be provided to Get & Has commands + // eg: + // func (m) Key() datastore.Key { + // return datastore.NewKey(fmt.Sprintf("%s:%s", m.DatastoreType(), m.GetId())) + // } + // in examples of "submodels" of another model it makes sense to leverage the + // POSIX structure of keys. for example: + // func (m) Key() datastore.Key { + // return datastore.NewKey(fmt.Sprintf("%s:%s/%s", m.DatastoreType(), m.ParentId(), m.GetId())) + // } + Key() datastore.Key + // NewSQLModel must allocate & return a new instance of the - // model with id set such that GetId returns the passed-in id string - NewSQLModel(id string) Model + // model with id set such that GetId returns the passed-in Key + // NewSQLModel will be passed keys for creation of new blank models + NewSQLModel(key datastore.Key) Model // SQLQuery gives the datastore the query to execute for a given command type // As an example, if CmdSelectOne is passed in, something like @@ -187,4 +245,52 @@ type Model interface { Model is the interface that must be implemented to work with sql_datastore. There are some fairly heavy constraints here. For a working example checkout: -https://github.com/datatogether/archive and have a look at primer.go +https://github.com/archivers-space/archive and have a look at primer.go + +#### type OrderBy + +```go +type OrderBy string +``` + +Order a query by a field + +#### func (OrderBy) Sort + +```go +func (o OrderBy) Sort([]query.Entry) +``` +satisfy datastore.Order interface, this is a no-op b/c sql sorting will happen +at query-time TODO - In the future this should be generalized to facilitate +supplying sql_datastore.OrderBy orders to other datastores, providing parity + +#### func (OrderBy) String + +```go +func (o OrderBy) String() string +``` +String value, used to inject the field name istself as a SQL query param + +#### type OrderByDesc + +```go +type OrderByDesc string +``` + +Order a query by a field, descending + +#### func (OrderByDesc) Sort + +```go +func (o OrderByDesc) Sort([]query.Entry) +``` +satisfy datastore.Order interface, this is a no-op b/c sql sorting will happen +at query-time TODO - In the future this should be generalized to facilitate +supplying sql_datastore.OrderBy orders to other datastores, providing parity + +#### func (OrderByDesc) String + +```go +func (o OrderByDesc) String() string +``` +String value, used to inject the field name istself as a SQL query param diff --git a/sql_datastore.go b/sql_datastore.go index 4b1a7f6..b7ffda2 100644 --- a/sql_datastore.go +++ b/sql_datastore.go @@ -12,6 +12,17 @@ // together. Whenever the datastore interface is not expressive enough, // one can always fall back to standard SQL work. // +// sql_datastore reconciles the key-value orientation of the datastore +// interface with the tables/relational orientation of SQL databases +// through the concept of a "Model". Model is a bit of an unfortunate name, +// as it implies this package is an ORM, which isn't a design goal. +// +// Annnnnnnnyway, the important patterns of this approach are: +// 1. The Model interface defines how to get stuff into and out of SQL +// 2. All Models that will be interacted with must be "Registered" to the store. +// Registered Models map to a datastore.Key Type. +// 3. All Get/Put/Delete/Has/Query to sql_datastore must map to a single Model +// // This implementation leads to a great deal of required boilerplate code // to implement. In the future this package could be expanded to become // syntax-aware, accepting a table name & schema definition for registered @@ -37,11 +48,12 @@ func SetDB(db *sql.DB) { DefaultStore.DB = db } +// Register a number of models to the DefaultStore func Register(models ...Model) error { return DefaultStore.Register(models...) } -// Datastore +// Datastore implements the ipfs datastore interface for SQL databases type Datastore struct { // DB is the underlying DB handler // it should be safe for use outside of the @@ -100,7 +112,7 @@ func (ds Datastore) Get(key datastore.Key) (value interface{}, err error) { return nil, err } - v := m.NewSQLModel(key.Name()) + v := m.NewSQLModel(key) if err := v.UnmarshalSQL(row); err != nil { return nil, err } @@ -140,46 +152,56 @@ func (ds Datastore) Delete(key datastore.Key) error { // Currently it's required that the passed-in prefix be equal to DatastoreType() // which query will use to determine what model to ask for a ListCmd func (ds Datastore) Query(q query.Query) (query.Results, error) { - // TODO - support query Filters - if len(q.Filters) > 0 { - return nil, fmt.Errorf("sql datastore queries do not support filters") - } - // TODO - support query Orders - if len(q.Orders) > 0 { - return nil, fmt.Errorf("sql datastore queries do not support ordering") - } + var rows *sql.Rows + // TODO - support KeysOnly if q.KeysOnly { - return nil, fmt.Errorf("sql datastore doesn't support keysonly ordering") + return nil, fmt.Errorf("sql datastore doesn't support keysonly querying") } - // TODO - ugh this so bad - m, err := ds.modelForKey(datastore.NewKey(fmt.Sprintf("/%s:", q.Prefix))) + // determine what type of model we're querying for + m, err := ds.modelForQuery(q) if err != nil { return nil, err } + // here we attach query information to a new model + // TODO - currently this is basing off of the prefix, might need + // to do smarter things in relation to filters? + m = m.NewSQLModel(datastore.NewKey(q.Prefix)) + // This is totally janky, but will work for now. It's expected that // the returned CmdList will have at least 2 bindvars: - // $1 : LIMIT - // $2 : OFFSET + // $1 : LIMIT value + // $2 : OFFSET value + // if compatible Orders are provided to the query, the order string + // will be provided as a third bindvar: + // $3 : ORDERBY value // From there it can provide zero or more additional bindvars to // organize the query, which should be returned by the SQLParams method // TODO - this seems to hint at a need for some sort of Controller-like // pattern in userland. Have a think. - rows, err := ds.query(m, CmdList, q.Limit, q.Offset) - if err != nil { - return nil, err + os := orderString(q) + if os != "" { + rows, err = ds.query(m, CmdList, q.Limit, q.Offset, os) + if err != nil { + return nil, err + } + } else { + rows, err = ds.query(m, CmdList, q.Limit, q.Offset) + if err != nil { + return nil, err + } } - // TODO - should this be q.Limit or query.NormalBufferSize + // TODO - should this be q.Limit or query.NormalBufferSize? reschan := make(chan query.Result, q.Limit) go func() { defer close(reschan) for rows.Next() { - model := m.NewSQLModel("") + model := m.NewSQLModel(datastore.NewKey(q.Prefix)) if err := model.UnmarshalSQL(rows); err != nil { reschan <- query.Result{ Error: err, @@ -199,21 +221,71 @@ func (ds Datastore) Query(q query.Query) (query.Results, error) { return query.ResultsWithChan(q, reschan), nil } +// modelForQuery determines what type of model this query should return +func (ds Datastore) modelForQuery(q query.Query) (Model, error) { + for _, f := range q.Filters { + switch t := f.(type) { + case FilterKeyTypeEq: + return ds.modelForKey(t.Key()) + } + } + return ds.modelForKey(datastore.NewKey(fmt.Sprintf("/%s:", q.Prefix))) +} + +// orderString generates orders from any OrderBy or OrderByDesc values +func orderString(q query.Query) string { + if len(q.Orders) == 0 { + return "" + } + orders := []string{} + for _, o := range q.Orders { + // TODO - should this cross-correct by casting based on + // the outer order? + switch ot := o.(type) { + case query.OrderByValue: + switch obv := ot.TypedOrder.(type) { + case OrderBy: + orders = append(orders, obv.String()) + case OrderByDesc: + orders = append(orders, obv.String()) + } + case query.OrderByValueDescending: + switch obv := ot.TypedOrder.(type) { + case OrderBy: + orders = append(orders, obv.String()) + case OrderByDesc: + orders = append(orders, obv.String()) + } + } + } + os := "" + for i, o := range orders { + if i == 0 { + os += o + } else { + os += fmt.Sprintf(", %s", o) + } + } + return os +} + // Batch commands are currently not supported func (ds *Datastore) Batch() (datastore.Batch, error) { return nil, datastore.ErrBatchUnsupported } +// for a given key, determine what kind of Model we're looking for func (ds Datastore) modelForKey(key datastore.Key) (Model, error) { for _, m := range ds.models { if m.DatastoreType() == key.Type() { // return a model with "ID" set to the key param - return m.NewSQLModel(key.Name()), nil + return m.NewSQLModel(key), nil } } return nil, fmt.Errorf("no usable model found for key, did you call register on the model?: %s", key.String()) } +// does this datastore func (ds Datastore) hasModel(m Model) (exists bool, err error) { row, err := ds.queryRow(m, CmdExistsOne) if err != nil { @@ -223,6 +295,7 @@ func (ds Datastore) hasModel(m Model) (exists bool, err error) { return } +// execute a Cmd against a given model func (ds Datastore) exec(m Model, t Cmd) error { if ds.DB == nil { return fmt.Errorf("datastore has no DB") @@ -235,6 +308,7 @@ func (ds Datastore) exec(m Model, t Cmd) error { return err } +// query for a single row, given a type of command and model func (ds Datastore) queryRow(m Model, t Cmd) (*sql.Row, error) { if ds.DB == nil { return nil, fmt.Errorf("datastore has no DB") @@ -246,6 +320,8 @@ func (ds Datastore) queryRow(m Model, t Cmd) (*sql.Row, error) { return ds.DB.QueryRow(query, params...), nil } +// run a query against the db for a given command and model, with optionally prebound +// arguments derived from the query func (ds Datastore) query(m Model, t Cmd, prebind ...interface{}) (*sql.Rows, error) { if ds.DB == nil { return nil, fmt.Errorf("datastore has no DB") @@ -254,9 +330,11 @@ func (ds Datastore) query(m Model, t Cmd, prebind ...interface{}) (*sql.Rows, er if err != nil { return nil, err } + return ds.DB.Query(query, append(prebind, params...)...) } +// prepare a query, grabbing the command sql & params from the model func (ds Datastore) prepQuery(m Model, t Cmd) (string, []interface{}, error) { query := m.SQLQuery(t) if query == "" {