From 558d54553c89decadc6a4e52153e5ab64a083a2a Mon Sep 17 00:00:00 2001 From: Arnaud Ysmal Date: Sun, 23 Mar 2014 18:58:53 +0100 Subject: [PATCH] Add support for the datastore API (early beta) --- README.md | 6 +- crypto.go | 2 +- datastores.go | 615 ++++++++++++++++++++++++++++++++++++++ datastores_changes.go | 516 ++++++++++++++++++++++++++++++++ datastores_parser.go | 303 +++++++++++++++++++ datastores_parser_test.go | 202 +++++++++++++ datastores_requests.go | 303 +++++++++++++++++++ datastores_test.go | 263 ++++++++++++++++ dropbox.go | 2 +- 9 files changed, 2208 insertions(+), 4 deletions(-) create mode 100644 datastores.go create mode 100644 datastores_changes.go create mode 100644 datastores_parser.go create mode 100644 datastores_parser_test.go create mode 100644 datastores_requests.go create mode 100644 datastores_test.go diff --git a/README.md b/README.md index 7254bc7..65b5700 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ dropbox ======= -Go client library for the Dropbox core API with support for uploading and downloading encrypted files. +Go client library for the Dropbox core and Datastore API with support for uploading and downloading encrypted files. + +Support of the Datastore API should be considered as a beta version. Prerequisite ------------ @@ -9,7 +11,7 @@ To register a new client application, please visit https://www.dropbox.com/devel Installation ------------ -This library depends on a fork of the goauth2 library, it can be installed with the go get command: +This library depends on a fork of the goauth2 library, it can be installed with the go get command: $ go get code.google.com/p/stacktic-goauth2 diff --git a/crypto.go b/crypto.go index 2978f39..5a83820 100644 --- a/crypto.go +++ b/crypto.go @@ -193,7 +193,7 @@ func (db *Dropbox) UploadFileAES(key []byte, src, dst string, overwrite bool, pa return db.FilesPutAES(key, fd, fsize, dst, overwrite, parentRev) } -// DownloadAES downloads and decrypts the file located in the src path on Dropbox and return a io.ReadCloser. +// DownloadAES downloads and decrypts the file located in the src path on Dropbox and returns a io.ReadCloser. func (db *Dropbox) DownloadAES(key []byte, src, rev string, offset int) (io.ReadCloser, error) { var in io.ReadCloser var size int64 diff --git a/datastores.go b/datastores.go new file mode 100644 index 0000000..0c2c1c1 --- /dev/null +++ b/datastores.go @@ -0,0 +1,615 @@ +/* +** Copyright (c) 2014 Arnaud Ysmal. All Rights Reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +** OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +** FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +** DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +** SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +** HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +** LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +** OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +** SUCH DAMAGE. + */ + +package dropbox + +import ( + "fmt" + "reflect" + "regexp" + "time" +) + +// List represents a value of type list. +type List struct { + record *Record + field string + values []interface{} +} + +// Fields represents a record. +type Fields map[string]value + +// Record represents an entry in a table. +type Record struct { + table *Table + recordID string + fields Fields + isDeleted bool +} + +// Table represents a list of records. +type Table struct { + datastore *Datastore + tableID string + records map[string]*Record +} + +// DatastoreInfo represents the information about a datastore. +type DatastoreInfo struct { + ID string + handle string + revision int + title string + mtime time.Time +} + +type datastoreDelta struct { + Revision int `json:"rev"` + Changes listOfChanges `json:"changes"` + Nonce *string `json:"nonce"` +} + +type listOfDelta []datastoreDelta + +// Datastore represents a datastore. +type Datastore struct { + manager *DatastoreManager + info DatastoreInfo + changes listOfChanges + tables map[string]*Table + isDeleted bool + autoCommit bool + changesQueue chan changeWork +} + +// DatastoreManager represents all datastores linked to the current account. +type DatastoreManager struct { + dropbox *Dropbox + datastores []*Datastore + token string +} + +const ( + defaultDatastoreID = "default" + maxGlobalIDLength = 63 + maxIDLength = 64 + + localIDPattern = `[a-z0-9_-]([a-z0-9._-]{0,62}[a-z0-9_-])?` + globalIDPattern = `.[A-Za-z0-9_-]{1,63}` + fieldsIDPattern = `[A-Za-z0-9._+/=-]{1,64}` + fieldsSpecialIDPattern = `:[A-Za-z0-9._+/=-]{1,63}` +) + +var ( + localIDRegexp *regexp.Regexp + globalIDRegexp *regexp.Regexp + fieldsIDRegexp *regexp.Regexp + fieldsSpecialIDRegexp *regexp.Regexp +) + +func init() { + var err error + if localIDRegexp, err = regexp.Compile(localIDPattern); err != nil { + fmt.Println(err) + } + if globalIDRegexp, err = regexp.Compile(globalIDPattern); err != nil { + fmt.Println(err) + } + if fieldsIDRegexp, err = regexp.Compile(fieldsIDPattern); err != nil { + fmt.Println(err) + } + if fieldsSpecialIDRegexp, err = regexp.Compile(fieldsSpecialIDPattern); err != nil { + fmt.Println(err) + } +} + +func isValidDatastoreID(ID string) bool { + if ID[0] == '.' { + return globalIDRegexp.MatchString(ID) + } + return localIDRegexp.MatchString(ID) +} + +func isValidID(ID string) bool { + if ID[0] == ':' { + return fieldsSpecialIDRegexp.MatchString(ID) + } + return fieldsIDRegexp.MatchString(ID) +} + +const ( + // TypeBoolean is the returned type when the value is a bool + TypeBoolean AtomType = iota + // TypeInteger is the returned type when the value is an int + TypeInteger + // TypeDouble is the returned type when the value is a float + TypeDouble + // TypeString is the returned type when the value is a string + TypeString + // TypeBytes is the returned type when the value is a []byte + TypeBytes + // TypeDate is the returned type when the value is a Date + TypeDate + // TypeList is the returned type when the value is a List + TypeList +) + +// AtomType represents the type of the value. +type AtomType int + +// NewDatastoreManager returns a new DatastoreManager linked to the current account. +func (db *Dropbox) NewDatastoreManager() *DatastoreManager { + return &DatastoreManager{ + dropbox: db, + } +} + +// OpenDatastore opens or creates a datastore. +func (dmgr *DatastoreManager) OpenDatastore(dsID string) (*Datastore, error) { + rev, handle, _, err := dmgr.dropbox.openOrCreateDatastore(dsID) + if err != nil { + return nil, err + } + rv := &Datastore{ + manager: dmgr, + info: DatastoreInfo{ + ID: dsID, + handle: handle, + revision: rev, + }, + tables: make(map[string]*Table), + changesQueue: make(chan changeWork), + } + if rev > 0 { + err = rv.LoadSnapshot() + } + go rv.doHandleChange() + return rv, err +} + +// OpenDefaultDatastore opens the default datastore. +func (dmgr *DatastoreManager) OpenDefaultDatastore() (*Datastore, error) { + return dmgr.OpenDatastore(defaultDatastoreID) +} + +// ListDatastores lists all datastores. +func (dmgr *DatastoreManager) ListDatastores() ([]DatastoreInfo, error) { + info, _, err := dmgr.dropbox.listDatastores() + return info, err +} + +// DeleteDatastore deletes a datastore. +func (dmgr *DatastoreManager) DeleteDatastore(dsID string) error { + _, err := dmgr.dropbox.deleteDatastore(dsID) + return err +} + +// CreateDatastore creates a global datastore with a unique ID, empty string for a random key. +func (dmgr *DatastoreManager) CreateDatastore(dsID string) (*Datastore, error) { + rev, handle, _, err := dmgr.dropbox.createDatastore(dsID) + if err != nil { + return nil, err + } + return &Datastore{ + manager: dmgr, + info: DatastoreInfo{ + ID: dsID, + handle: handle, + revision: rev, + }, + tables: make(map[string]*Table), + changesQueue: make(chan changeWork), + }, nil +} + +// AwaitDeltas awaits for deltas and applies them. +func (ds *Datastore) AwaitDeltas() error { + if len(ds.changes) != 0 { + return fmt.Errorf("changes already pending") + } + _, _, deltas, err := ds.manager.dropbox.await([]*Datastore{ds}, "") + if err != nil { + return err + } + changes, ok := deltas[ds.info.handle] + if !ok || len(changes) == 0 { + return nil + } + return ds.applyDelta(changes) +} + +func (ds *Datastore) applyDelta(dds []datastoreDelta) error { + if len(ds.changes) != 0 { + return fmt.Errorf("changes already pending") + } + for _, d := range dds { + if d.Revision < ds.info.revision { + continue + } + for _, c := range d.Changes { + ds.applyChange(c) + } + } + return nil +} + +// Close closes the datastore. +func (ds *Datastore) Close() { + close(ds.changesQueue) +} + +// SetTitle sets the datastore title to the given string. +func (ds *Datastore) SetTitle(t string) error { + if len(ds.info.title) == 0 { + return ds.insertRecord(":info", "info", Fields{ + "title": value{ + values: []interface{}{t}, + }, + }) + } + return ds.updateField(":info", "info", "title", t) +} + +// SetMTime sets the datastore mtime to the given time. +func (ds *Datastore) SetMTime(t time.Time) error { + if time.Time(ds.info.mtime).IsZero() { + return ds.insertRecord(":info", "info", Fields{ + "mtime": value{ + values: []interface{}{t}, + }, + }) + } + return ds.updateField(":info", "info", "mtime", t) +} + +// Rollback reverts all local changes and discards them. +func (ds *Datastore) Rollback() error { + if len(ds.changes) == 0 { + return nil + } + for i := len(ds.changes) - 1; i >= 0; i-- { + ds.applyChange(ds.changes[i].Revert) + } + ds.changes = ds.changes[:0] + return nil +} + +// GetTable returns the requested table. +func (ds *Datastore) GetTable(tableID string) (*Table, error) { + if !isValidID(tableID) { + return nil, fmt.Errorf("invalid table ID %s", tableID) + } + t, ok := ds.tables[tableID] + if ok { + return t, nil + } + t = &Table{ + datastore: ds, + tableID: tableID, + records: make(map[string]*Record), + } + ds.tables[tableID] = t + return t, nil +} + +// Commit commits the changes registered by sending them to the server. +func (ds *Datastore) Commit() error { + rev, err := ds.manager.dropbox.putDelta(ds.info.handle, ds.info.revision, ds.changes) + if err != nil { + return err + } + ds.changes = ds.changes[:0] + ds.info.revision = rev + return nil +} + +// LoadSnapshot updates the state of the datastore from the server. +func (ds *Datastore) LoadSnapshot() error { + if len(ds.changes) != 0 { + return fmt.Errorf("could not load snapshot when there are pending changes") + } + rows, rev, err := ds.manager.dropbox.getSnapshot(ds.info.handle) + if err != nil { + return err + } + + ds.tables = make(map[string]*Table) + for _, r := range rows { + if _, ok := ds.tables[r.TID]; !ok { + ds.tables[r.TID] = &Table{ + datastore: ds, + tableID: r.TID, + records: make(map[string]*Record), + } + } + ds.tables[r.TID].records[r.RowID] = &Record{ + table: ds.tables[r.TID], + recordID: r.RowID, + fields: r.Data, + } + } + ds.info.revision = rev + return nil +} + +// GetDatastore returns the datastore associated with this table. +func (t *Table) GetDatastore() *Datastore { + return t.datastore +} + +// GetID returns the ID of this table. +func (t *Table) GetID() string { + return t.tableID +} + +// Get returns the record with this ID. +func (t *Table) Get(recordID string) (*Record, error) { + if !isValidID(recordID) { + return nil, fmt.Errorf("invalid record ID %s", recordID) + } + return t.records[recordID], nil +} + +// GetOrInsert gets the requested record. +func (t *Table) GetOrInsert(recordID string) (*Record, error) { + if !isValidID(recordID) { + return nil, fmt.Errorf("invalid record ID %s", recordID) + } + return t.GetOrInsertWithFields(recordID, nil) +} + +// GetOrInsertWithFields gets the requested table. +func (t *Table) GetOrInsertWithFields(recordID string, fields Fields) (*Record, error) { + if !isValidID(recordID) { + return nil, fmt.Errorf("invalid record ID %s", recordID) + } + if r, ok := t.records[recordID]; ok { + return r, nil + } + if fields == nil { + fields = make(Fields) + } + if err := t.datastore.insertRecord(t.tableID, recordID, fields); err != nil { + return nil, err + } + return t.records[recordID], nil +} + +// Query returns a list of records matching all the given fields. +func (t *Table) Query(fields Fields) ([]*Record, error) { + var records []*Record + +next: + for _, record := range t.records { + for qf, qv := range fields { + if rv, ok := record.fields[qf]; !ok || !reflect.DeepEqual(qv, rv) { + continue next + } + } + records = append(records, record) + } + return records, nil +} + +// GetTable returns the table associated with this record. +func (r *Record) GetTable() *Table { + return r.table +} + +// GetID returns the ID of this record. +func (r *Record) GetID() string { + return r.recordID +} + +// IsDeleted returns whether this record was deleted. +func (r *Record) IsDeleted() bool { + return r.isDeleted +} + +// DeleteRecord deletes this record. +func (r *Record) DeleteRecord() { + r.table.datastore.deleteRecord(r.table.tableID, r.recordID) +} + +// HasField returns whether this field exists. +func (r *Record) HasField(field string) (bool, error) { + if !isValidID(field) { + return false, fmt.Errorf("invalid field %s", field) + } + _, ok := r.fields[field] + return ok, nil +} + +// Get gets the current value of this field. +func (r *Record) Get(field string) (interface{}, bool, error) { + if !isValidID(field) { + return nil, false, fmt.Errorf("invalid field %s", field) + } + v, ok := r.fields[field] + if !ok { + return nil, false, nil + } + if v.isList { + return &List{ + record: r, + field: field, + values: v.values, + }, true, nil + } + return v.values[0], true, nil +} + +// GetOrCreateList gets the current value of this field. +func (r *Record) GetOrCreateList(field string) (*List, error) { + if !isValidID(field) { + return nil, fmt.Errorf("invalid field %s", field) + } + v, ok := r.fields[field] + if ok && !v.isList { + return nil, fmt.Errorf("not a list") + } + if !ok { + if err := r.table.datastore.listCreate(r.table.tableID, r.recordID, field); err != nil { + return nil, err + } + v = r.fields[field] + } + return &List{ + record: r, + field: field, + values: v.values, + }, nil +} + +func getType(i interface{}) (AtomType, error) { + switch i.(type) { + case bool: + return TypeBoolean, nil + case int, int32, int64: + return TypeInteger, nil + case float32, float64: + return TypeDouble, nil + case string: + return TypeString, nil + case []byte: + return TypeBytes, nil + case time.Time: + return TypeDate, nil + } + return 0, fmt.Errorf("type %s not supported", reflect.TypeOf(i).Name()) +} + +// GetFieldType returns the type of the given field. +func (r *Record) GetFieldType(field string) (AtomType, error) { + if !isValidID(field) { + return 0, fmt.Errorf("invalid field %s", field) + } + v, ok := r.fields[field] + if !ok { + return 0, fmt.Errorf("no such field: %s", field) + } + if v.isList { + return TypeList, nil + } + return getType(v.values[0]) +} + +// Set sets the value of a field. +func (r *Record) Set(field string, value interface{}) error { + if !isValidID(field) { + return fmt.Errorf("invalid field %s", field) + } + return r.table.datastore.updateField(r.table.tableID, r.recordID, field, value) +} + +// DeleteField deletes the given field from this record. +func (r *Record) DeleteField(field string) error { + if !isValidID(field) { + return fmt.Errorf("invalid field %s", field) + } + return r.table.datastore.deleteField(r.table.tableID, r.recordID, field) +} + +// FieldNames returns a list of fields names. +func (r *Record) FieldNames() []string { + var rv []string + + rv = make([]string, 0, len(r.fields)) + for k := range r.fields { + rv = append(rv, k) + } + return rv +} + +// IsEmpty returns whether the list contains an element. +func (l *List) IsEmpty() bool { + return len(l.values) == 0 +} + +// Size returns the number of elements in the list. +func (l *List) Size() int { + return len(l.values) +} + +// GetType gets the type of the n-th element in the list. +func (l *List) GetType(n int) (AtomType, error) { + if n >= len(l.values) { + return 0, fmt.Errorf("out of bound index") + } + return getType(l.values[n]) +} + +// Get gets the n-th element in the list. +func (l *List) Get(n int) (interface{}, error) { + if n >= len(l.values) { + return 0, fmt.Errorf("out of bound index") + } + return l.values[n], nil +} + +// AddAtPos inserts the item at the n-th position in the list. +func (l *List) AddAtPos(n int, i interface{}) error { + if n > len(l.values) { + return fmt.Errorf("out of bound index") + } + err := l.record.table.datastore.listInsert(l.record.table.tableID, l.record.recordID, l.field, n, i) + if err != nil { + return err + } + l.values = l.record.fields[l.field].values + return nil +} + +// Add adds the item at the end of the list. +func (l *List) Add(i interface{}) error { + return l.AddAtPos(len(l.values), i) +} + +// Set sets the value of the n-th element of the list. +func (l *List) Set(n int, i interface{}) error { + if n >= len(l.values) { + return fmt.Errorf("out of bound index") + } + return l.record.table.datastore.listPut(l.record.table.tableID, l.record.recordID, l.field, n, i) +} + +// Remove removes the n-th element of the list. +func (l *List) Remove(n int) error { + if n >= len(l.values) { + return fmt.Errorf("out of bound index") + } + err := l.record.table.datastore.listDelete(l.record.table.tableID, l.record.recordID, l.field, n) + l.values = l.record.fields[l.field].values + return err +} + +// Move moves the element from the from-th position to the to-th. +func (l *List) Move(from, to int) error { + if from >= len(l.values) || to >= len(l.values) { + return fmt.Errorf("out of bound index") + } + return l.record.table.datastore.listMove(l.record.table.tableID, l.record.recordID, l.field, from, to) +} diff --git a/datastores_changes.go b/datastores_changes.go new file mode 100644 index 0000000..d507f03 --- /dev/null +++ b/datastores_changes.go @@ -0,0 +1,516 @@ +/* +** Copyright (c) 2014 Arnaud Ysmal. All Rights Reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +** OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +** FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +** DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +** SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +** HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +** LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +** OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +** SUCH DAMAGE. + */ + +package dropbox + +import ( + "fmt" + "reflect" +) + +type value struct { + values []interface{} + isList bool +} + +type fieldOp struct { + Op string + Index int + Index2 int + Data value +} + +type opDict map[string]fieldOp + +type change struct { + Op string + TID string + RecordID string + Ops opDict + Data Fields + Revert *change +} +type listOfChanges []*change + +type changeWork struct { + c *change + out chan error +} + +const ( + recordDelete = "D" + recordInsert = "I" + recordUpdate = "U" + fieldDelete = "D" + fieldPut = "P" + listCreate = "LC" + listDelete = "LD" + listInsert = "LI" + listMove = "LM" + listPut = "LP" +) + +func newValueFromInterface(i interface{}) *value { + if a, ok := i.([]byte); ok { + return &value{ + values: []interface{}{a}, + isList: false, + } + } + if reflect.TypeOf(i).Kind() == reflect.Slice || reflect.TypeOf(i).Kind() == reflect.Array { + val := reflect.ValueOf(i) + v := &value{ + values: make([]interface{}, val.Len()), + isList: true, + } + for i := range v.values { + v.values[i] = val.Index(i).Interface() + } + return v + } + return &value{ + values: []interface{}{i}, + isList: false, + } +} + +func newValue(v *value) *value { + var nv *value + + nv = &value{ + values: make([]interface{}, len(v.values)), + isList: v.isList, + } + copy(nv.values, v.values) + return nv +} + +func newFields(f Fields) Fields { + var n Fields + + n = make(Fields) + for k, v := range f { + n[k] = *newValue(&v) + } + return n +} + +func (ds *Datastore) deleteRecord(table, record string) error { + return ds.handleChange(&change{ + Op: recordDelete, + TID: table, + RecordID: record, + }) +} + +func (ds *Datastore) insertRecord(table, record string, values Fields) error { + return ds.handleChange(&change{ + Op: recordInsert, + TID: table, + RecordID: record, + Data: newFields(values), + }) +} + +func (ds *Datastore) updateFields(table, record string, values map[string]interface{}) error { + var dsval opDict + + dsval = make(opDict) + for k, v := range values { + dsval[k] = fieldOp{ + Op: fieldPut, + Data: *newValueFromInterface(v), + } + } + return ds.handleChange(&change{ + Op: recordUpdate, + TID: table, + RecordID: record, + Ops: dsval, + }) +} + +func (ds *Datastore) updateField(table, record, field string, i interface{}) error { + return ds.updateFields(table, record, map[string]interface{}{field: i}) +} + +func (ds *Datastore) deleteField(table, record, field string) error { + return ds.handleChange(&change{ + Op: recordUpdate, + TID: table, + RecordID: record, + Ops: opDict{ + field: fieldOp{ + Op: fieldDelete, + }, + }, + }) +} + +func (ds *Datastore) listCreate(table, record, field string) error { + return ds.handleChange(&change{ + Op: recordUpdate, + TID: table, + RecordID: record, + Ops: opDict{ + field: fieldOp{ + Op: listCreate, + }, + }, + }) +} + +func (ds *Datastore) listDelete(table, record, field string, pos int) error { + return ds.handleChange(&change{ + Op: recordUpdate, + TID: table, + RecordID: record, + Ops: opDict{ + field: fieldOp{ + Op: listDelete, + Index: pos, + }, + }, + }) +} + +func (ds *Datastore) listInsert(table, record, field string, pos int, i interface{}) error { + return ds.handleChange(&change{ + Op: recordUpdate, + TID: table, + RecordID: record, + Ops: opDict{ + field: fieldOp{ + Op: listInsert, + Index: pos, + Data: *newValueFromInterface(i), + }, + }, + }) +} + +func (ds *Datastore) listMove(table, record, field string, from, to int) error { + return ds.handleChange(&change{ + Op: recordUpdate, + TID: table, + RecordID: record, + Ops: opDict{ + field: fieldOp{ + Op: listMove, + Index: from, + Index2: to, + }, + }, + }) +} + +func (ds *Datastore) listPut(table, record, field string, pos int, i interface{}) error { + return ds.handleChange(&change{ + Op: recordUpdate, + TID: table, + RecordID: record, + Ops: opDict{ + field: fieldOp{ + Op: listPut, + Index: pos, + Data: *newValueFromInterface(i), + }, + }, + }) +} + +func (ds *Datastore) handleChange(c *change) error { + var out chan error + + if ds.changesQueue == nil { + return fmt.Errorf("datastore is closed") + } + out = make(chan error) + ds.changesQueue <- changeWork{ + c: c, + out: out, + } + return <-out +} + +func (ds *Datastore) doHandleChange() { + var err error + var c *change + + q := ds.changesQueue + for cw := range q { + c = cw.c + + if err = ds.validateChange(c); err != nil { + cw.out <- err + continue + } + if c.Revert, err = ds.inverseChange(c); err != nil { + cw.out <- err + continue + } + + if err = ds.applyChange(c); err != nil { + cw.out <- err + continue + } + + ds.changes = append(ds.changes, c) + + if ds.autoCommit { + if err = ds.Commit(); err != nil { + cw.out <- err + } + } + close(cw.out) + } +} + +func (ds *Datastore) validateChange(c *change) error { + var t *Table + var r *Record + var ok bool + + if t, ok = ds.tables[c.TID]; !ok { + t = &Table{ + datastore: ds, + tableID: c.TID, + records: make(map[string]*Record), + } + } + + r = t.records[c.RecordID] + + switch c.Op { + case recordInsert, recordDelete: + return nil + case recordUpdate: + if r == nil { + return fmt.Errorf("no such record: %s", c.RecordID) + } + for field, op := range c.Ops { + if op.Op == fieldPut || op.Op == fieldDelete { + continue + } + v, ok := r.fields[field] + if op.Op == listCreate { + if ok { + return fmt.Errorf("field %s already exists", field) + } + continue + } + if !ok { + return fmt.Errorf("no such field: %s", field) + } + if !v.isList { + return fmt.Errorf("field %s is not a list", field) + } + maxIndex := len(v.values) - 1 + if op.Op == listInsert { + maxIndex++ + } + if op.Index > maxIndex { + return fmt.Errorf("out of bound access index %d on [0:%s]", op.Index, maxIndex) + } + if op.Index2 > maxIndex { + return fmt.Errorf("out of bound access index %d on [0:%s]", op.Index, maxIndex) + } + } + } + return nil +} + +func (ds *Datastore) applyChange(c *change) error { + var t *Table + var r *Record + var ok bool + + if t, ok = ds.tables[c.TID]; !ok { + t = &Table{ + datastore: ds, + tableID: c.TID, + records: make(map[string]*Record), + } + ds.tables[c.TID] = t + } + + r = t.records[c.RecordID] + + switch c.Op { + case recordInsert: + t.records[c.RecordID] = &Record{ + table: t, + recordID: c.RecordID, + fields: newFields(c.Data), + } + case recordDelete: + if r == nil { + return nil + } + r.isDeleted = true + delete(t.records, c.RecordID) + case recordUpdate: + for field, op := range c.Ops { + v, ok := r.fields[field] + switch op.Op { + case fieldPut: + r.fields[field] = *newValue(&op.Data) + case fieldDelete: + if ok { + delete(r.fields, field) + } + case listCreate: + if !ok { + r.fields[field] = value{isList: true} + } + case listDelete: + copy(v.values[op.Index:], v.values[op.Index+1:]) + v.values = v.values[:len(v.values)-1] + r.fields[field] = v + case listInsert: + v.values = append(v.values, op.Data) + copy(v.values[op.Index+1:], v.values[op.Index:len(v.values)-1]) + v.values[op.Index] = op.Data.values[0] + r.fields[field] = v + case listMove: + val := v.values[op.Index] + if op.Index < op.Index2 { + copy(v.values[op.Index:op.Index2], v.values[op.Index+1:op.Index2+1]) + } else { + copy(v.values[op.Index2+1:op.Index+1], v.values[op.Index2:op.Index]) + } + v.values[op.Index2] = val + r.fields[field] = v + case listPut: + r.fields[field].values[op.Index] = op.Data.values[0] + } + } + } + return nil +} + +func (ds *Datastore) inverseChange(c *change) (*change, error) { + var t *Table + var r *Record + var ok bool + var rev *change + + if t, ok = ds.tables[c.TID]; !ok { + t = &Table{ + datastore: ds, + tableID: c.TID, + records: make(map[string]*Record), + } + ds.tables[c.TID] = t + } + + r = t.records[c.RecordID] + + switch c.Op { + case recordInsert: + return &change{ + Op: recordDelete, + TID: c.TID, + RecordID: c.RecordID, + }, nil + case recordDelete: + if r == nil { + return nil, nil + } + return &change{ + Op: recordInsert, + TID: c.TID, + RecordID: c.RecordID, + Data: newFields(r.fields), + }, nil + case recordUpdate: + rev = &change{ + Op: recordUpdate, + TID: c.TID, + RecordID: c.RecordID, + Ops: make(opDict), + } + for field, op := range c.Ops { + switch op.Op { + case fieldPut: + if v, ok := r.fields[field]; ok { + rev.Ops[field] = fieldOp{ + Op: fieldPut, + Data: *newValue(&v), + } + } else { + rev.Ops[field] = fieldOp{ + Op: fieldDelete, + } + } + case fieldDelete: + if v, ok := r.fields[field]; ok { + rev.Ops[field] = fieldOp{ + Op: fieldPut, + Data: *newValue(&v), + } + } + case listCreate: + if _, ok := r.fields[field]; !ok { + rev.Ops[field] = fieldOp{ + Op: fieldDelete, + } + } + case listDelete: + v := r.fields[field] + rev.Ops[field] = fieldOp{ + Op: listInsert, + Index: op.Index, + Data: value{ + values: []interface{}{v.values[op.Index]}, + isList: false, + }, + } + case listInsert: + rev.Ops[field] = fieldOp{ + Op: listDelete, + Index: op.Index, + } + case listMove: + rev.Ops[field] = fieldOp{ + Op: listMove, + Index: op.Index2, + Index2: op.Index, + } + case listPut: + v := r.fields[field] + rev.Ops[field] = fieldOp{ + Op: listPut, + Index: op.Index, + Data: value{ + values: []interface{}{v.values[op.Index]}, + isList: false, + }, + } + } + } + } + return rev, nil +} diff --git a/datastores_parser.go b/datastores_parser.go new file mode 100644 index 0000000..7a48dae --- /dev/null +++ b/datastores_parser.go @@ -0,0 +1,303 @@ +/* +** Copyright (c) 2014 Arnaud Ysmal. All Rights Reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +** OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +** FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +** DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +** SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +** HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +** LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +** OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +** SUCH DAMAGE. + */ + +package dropbox + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "math" + "strconv" + "strings" + "time" +) + +type atom struct { + Value interface{} +} + +func encodeDBase64(b []byte) string { + return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=") +} + +func decodeDBase64(s string) ([]byte, error) { + pad := 4 - len(s)%4 + if pad != 4 { + s += strings.Repeat("=", pad) + } + return base64.URLEncoding.DecodeString(s) +} + +// MarshalJSON returns the JSON encoding of a. +func (a atom) MarshalJSON() ([]byte, error) { + switch v := a.Value.(type) { + case bool, string: + return json.Marshal(v) + case float64: + if math.IsNaN(v) { + return []byte(`{"N": "nan"}`), nil + } else if math.IsInf(v, 1) { + return []byte(`{"N": "+inf"}`), nil + } else if math.IsInf(v, -1) { + return []byte(`{"N": "-inf"}`), nil + } + return json.Marshal(v) + case time.Time: + return []byte(fmt.Sprintf(`{"T": "%d"}`, v.UnixNano()/int64(time.Millisecond))), nil + case int, int32, int64: + return []byte(fmt.Sprintf(`{"I": "%d"}`, v)), nil + case []byte: + return []byte(fmt.Sprintf(`{"B": "%s"}`, encodeDBase64(v))), nil + } + return nil, fmt.Errorf("wrong format") +} + +// UnmarshalJSON parses the JSON-encoded data and stores the result in the value pointed to by a. +func (a *atom) UnmarshalJSON(data []byte) error { + var i interface{} + var err error + + if err = json.Unmarshal(data, &i); err != nil { + return err + } + switch v := i.(type) { + case bool, int, int32, int64, float32, float64, string: + a.Value = v + return nil + case map[string]interface{}: + for key, rval := range v { + val, ok := rval.(string) + if !ok { + return fmt.Errorf("could not parse atom") + } + switch key { + case "I": + a.Value, err = strconv.ParseInt(val, 10, 64) + return nil + case "N": + switch val { + case "nan": + a.Value = math.NaN() + return nil + case "+inf": + a.Value = math.Inf(1) + return nil + case "-inf": + a.Value = math.Inf(-1) + return nil + default: + return fmt.Errorf("unknown special type %s", val) + } + case "T": + t, err := strconv.ParseInt(val, 10, 64) + if err != nil { + return fmt.Errorf("could not parse atom") + } + a.Value = time.Unix(t/1000, (t%1000)*int64(time.Millisecond)) + return nil + + case "B": + a.Value, err = decodeDBase64(val) + return err + } + } + } + return fmt.Errorf("could not parse atom") +} + +// MarshalJSON returns the JSON encoding of v. +func (v value) MarshalJSON() ([]byte, error) { + if v.isList { + var a []atom + + a = make([]atom, len(v.values)) + for i := range v.values { + a[i].Value = v.values[i] + } + return json.Marshal(a) + } + return json.Marshal(atom{Value: v.values[0]}) +} + +// UnmarshalJSON parses the JSON-encoded data and stores the result in the value pointed to by v. +func (v *value) UnmarshalJSON(data []byte) error { + var isArray bool + var err error + var a atom + var as []atom + + for _, d := range data { + if d == ' ' { + continue + } + if d == '[' { + isArray = true + } + break + } + if isArray { + if err = json.Unmarshal(data, &as); err != nil { + return err + } + v.values = make([]interface{}, len(as)) + for i, at := range as { + v.values[i] = at.Value + } + v.isList = true + return nil + } + if err = json.Unmarshal(data, &a); err != nil { + return err + } + v.values = make([]interface{}, 1) + v.values[0] = a.Value + return nil +} + +// UnmarshalJSON parses the JSON-encoded data and stores the result in the value pointed to by f. +func (f *fieldOp) UnmarshalJSON(data []byte) error { + var i []json.RawMessage + var err error + + if err = json.Unmarshal(data, &i); err != nil { + return err + } + + if err = json.Unmarshal(i[0], &f.Op); err != nil { + return err + } + switch f.Op { + case fieldPut: + if len(i) != 2 { + return fmt.Errorf("wrong format") + } + return json.Unmarshal(i[1], &f.Data) + case fieldDelete, listCreate: + if len(i) != 1 { + return fmt.Errorf("wrong format") + } + case listInsert, listPut: + if len(i) != 3 { + return fmt.Errorf("wrong format") + } + if err = json.Unmarshal(i[1], &f.Index); err != nil { + return err + } + return json.Unmarshal(i[2], &f.Data) + case listDelete: + if len(i) != 2 { + return fmt.Errorf("wrong format") + } + return json.Unmarshal(i[1], &f.Index) + case listMove: + if len(i) != 3 { + return fmt.Errorf("wrong format") + } + if err = json.Unmarshal(i[1], &f.Index); err != nil { + return err + } + return json.Unmarshal(i[2], &f.Index2) + default: + return fmt.Errorf("wrong format") + } + return nil +} + +// MarshalJSON returns the JSON encoding of f. +func (f fieldOp) MarshalJSON() ([]byte, error) { + switch f.Op { + case fieldPut: + return json.Marshal([]interface{}{f.Op, f.Data}) + case fieldDelete, listCreate: + return json.Marshal([]interface{}{f.Op}) + case listInsert, listPut: + return json.Marshal([]interface{}{f.Op, f.Index, f.Data}) + case listDelete: + return json.Marshal([]interface{}{f.Op, f.Index}) + case listMove: + return json.Marshal([]interface{}{f.Op, f.Index, f.Index2}) + } + return nil, fmt.Errorf("could not marshal Change type") +} + +// UnmarshalJSON parses the JSON-encoded data and stores the result in the value pointed to by c. +func (c *change) UnmarshalJSON(data []byte) error { + var i []json.RawMessage + var err error + + if err = json.Unmarshal(data, &i); err != nil { + return err + } + if len(i) < 3 { + return fmt.Errorf("wrong format") + } + + if err = json.Unmarshal(i[0], &c.Op); err != nil { + return err + } + if err = json.Unmarshal(i[1], &c.TID); err != nil { + return err + } + if err = json.Unmarshal(i[2], &c.RecordID); err != nil { + return err + } + switch c.Op { + case recordInsert: + if len(i) != 4 { + return fmt.Errorf("wrong format") + } + if err = json.Unmarshal(i[3], &c.Data); err != nil { + return err + } + case recordUpdate: + if len(i) != 4 { + return fmt.Errorf("wrong format") + } + if err = json.Unmarshal(i[3], &c.Ops); err != nil { + return err + } + case recordDelete: + if len(i) != 3 { + return fmt.Errorf("wrong format") + } + default: + return fmt.Errorf("wrong format") + } + return nil +} + +// MarshalJSON returns the JSON encoding of c. +func (c change) MarshalJSON() ([]byte, error) { + switch c.Op { + case recordInsert: + return json.Marshal([]interface{}{recordInsert, c.TID, c.RecordID, c.Data}) + case recordUpdate: + return json.Marshal([]interface{}{recordUpdate, c.TID, c.RecordID, c.Ops}) + case recordDelete: + return json.Marshal([]interface{}{recordDelete, c.TID, c.RecordID}) + } + return nil, fmt.Errorf("could not marshal Change type") +} diff --git a/datastores_parser_test.go b/datastores_parser_test.go new file mode 100644 index 0000000..5a33998 --- /dev/null +++ b/datastores_parser_test.go @@ -0,0 +1,202 @@ +/* +** Copyright (c) 2014 Arnaud Ysmal. All Rights Reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +** OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +** FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +** DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +** SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +** HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +** LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +** OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +** SUCH DAMAGE. + */ + +package dropbox + +import ( + "encoding/json" + "fmt" + "math" + "reflect" + "testing" + "time" +) + +type equaller interface { + equals(equaller) bool +} + +func (s *atom) equals(o *atom) bool { + switch v1 := s.Value.(type) { + case float64: + if v2, ok := o.Value.(float64); ok { + if math.IsNaN(v1) && math.IsNaN(v2) { + return true + } + return v1 == v2 + } + default: + return reflect.DeepEqual(s, o) + } + return false +} + +func (s *value) equals(o *value) bool { + return reflect.DeepEqual(s, o) +} + +func (s Fields) equals(o Fields) bool { + return reflect.DeepEqual(s, o) +} + +func (s opDict) equals(o opDict) bool { + return reflect.DeepEqual(s, o) +} + +func (s *change) equals(o *change) bool { + return reflect.DeepEqual(s, o) +} + +func (s *fieldOp) equals(o *fieldOp) bool { + return reflect.DeepEqual(s, o) +} + +func testDSAtom(t *testing.T, c *atom, e string) { + var c2 atom + var err error + var js []byte + + if js, err = json.Marshal(c); err != nil { + t.Errorf("%s", err) + } + if err = c2.UnmarshalJSON(js); err != nil { + t.Errorf("%s", err) + } + if !c.equals(&c2) { + t.Errorf("expected %#v type %s got %#v of type %s", c.Value, reflect.TypeOf(c.Value).Name(), c2.Value, reflect.TypeOf(c2.Value).Name()) + } + c2 = atom{} + if err = c2.UnmarshalJSON([]byte(e)); err != nil { + t.Errorf("%s", err) + } + if !c.equals(&c2) { + t.Errorf("expected %#v type %s got %#v of type %s", c.Value, reflect.TypeOf(c.Value).Name(), c2.Value, reflect.TypeOf(c2.Value).Name()) + } +} + +func TestDSAtomUnmarshalJSON(t *testing.T) { + testDSAtom(t, &atom{Value: 32.5}, `32.5`) + testDSAtom(t, &atom{Value: true}, `true`) + testDSAtom(t, &atom{Value: int64(42)}, `{"I":"42"}`) + testDSAtom(t, &atom{Value: math.NaN()}, `{"N":"nan"}`) + testDSAtom(t, &atom{Value: math.Inf(1)}, `{"N":"+inf"}`) + testDSAtom(t, &atom{Value: math.Inf(-1)}, `{"N":"-inf"}`) + testDSAtom(t, &atom{Value: []byte(`random string converted to bytes`)}, `{"B":"cmFuZG9tIHN0cmluZyBjb252ZXJ0ZWQgdG8gYnl0ZXM="}`) + + now := time.Now().Round(time.Millisecond) + js := fmt.Sprintf(`{"T": "%d"}`, now.UnixNano()/int64(time.Millisecond)) + testDSAtom(t, &atom{Value: now}, js) +} + +func testDSChange(t *testing.T, c *change, e string) { + var c2 change + var err error + var js []byte + + if js, err = json.Marshal(c); err != nil { + t.Errorf("%s", err) + } + if err = c2.UnmarshalJSON(js); err != nil { + t.Errorf("%s", err) + } + if !c.equals(&c2) { + t.Errorf("mismatch: got:\n\t%#v\nexpected:\n\t%#v", c2, *c) + } + c2 = change{} + if err = c2.UnmarshalJSON([]byte(e)); err != nil { + t.Errorf("%s", err) + } + if !c.equals(&c2) { + t.Errorf("mismatch") + } +} + +func TestDSChangeUnmarshalJSON(t *testing.T) { + testDSChange(t, + &change{ + Op: recordInsert, + TID: "dropbox", + RecordID: "test", + Data: Fields{"float": value{values: []interface{}{float64(42)}, isList: false}}, + }, `["I","dropbox","test",{"float":42}]`) + testDSChange(t, + &change{ + Op: recordUpdate, + TID: "dropbox", + RecordID: "test", + Ops: opDict{"field": fieldOp{Op: fieldPut, Data: value{values: []interface{}{float64(42)}, isList: false}}}, + }, `["U","dropbox","test",{"field":["P", 42]}]`) + testDSChange(t, + &change{ + Op: recordUpdate, + TID: "dropbox", + RecordID: "test", + Ops: opDict{"field": fieldOp{Op: listCreate}}, + }, `["U","dropbox","test",{"field":["LC"]}]`) + + testDSChange(t, + &change{ + Op: recordDelete, + TID: "dropbox", + RecordID: "test", + }, `["D","dropbox","test"]`) +} + +func testCheckfieldOp(t *testing.T, fo *fieldOp, e string) { + var fo2 fieldOp + var js []byte + var err error + + if js, err = json.Marshal(fo); err != nil { + t.Errorf("%s", err) + } + if string(js) != e { + t.Errorf("marshalling error got %s expected %s", string(js), e) + } + if err = json.Unmarshal(js, &fo2); err != nil { + t.Errorf("%s %s", err, string(js)) + } + if !fo.equals(&fo2) { + t.Errorf("%#v != %#v\n", fo, fo2) + } + fo2 = fieldOp{} + if err = json.Unmarshal([]byte(e), &fo2); err != nil { + t.Errorf("%s %s", err, string(js)) + } + if !fo.equals(&fo2) { + t.Errorf("%#v != %#v\n", fo, fo2) + } +} + +func TestDSfieldOpMarshalling(t *testing.T) { + testCheckfieldOp(t, &fieldOp{Op: "P", Data: value{values: []interface{}{"bar"}, isList: false}}, `["P","bar"]`) + testCheckfieldOp(t, &fieldOp{Op: "P", Data: value{values: []interface{}{"ga", "bu", "zo", "meuh", int64(42), 4.5, true}, isList: true}}, `["P",["ga","bu","zo","meuh",{"I":"42"},4.5,true]]`) + testCheckfieldOp(t, &fieldOp{Op: "D"}, `["D"]`) + testCheckfieldOp(t, &fieldOp{Op: "LC"}, `["LC"]`) + testCheckfieldOp(t, &fieldOp{Op: "LP", Index: 1, Data: value{values: []interface{}{"baz"}}}, `["LP",1,"baz"]`) + testCheckfieldOp(t, &fieldOp{Op: "LI", Index: 1, Data: value{values: []interface{}{"baz"}}}, `["LI",1,"baz"]`) + testCheckfieldOp(t, &fieldOp{Op: "LD", Index: 1}, `["LD",1]`) + testCheckfieldOp(t, &fieldOp{Op: "LM", Index: 1, Index2: 2}, `["LM",1,2]`) +} diff --git a/datastores_requests.go b/datastores_requests.go new file mode 100644 index 0000000..e20d653 --- /dev/null +++ b/datastores_requests.go @@ -0,0 +1,303 @@ +/* +** Copyright (c) 2014 Arnaud Ysmal. All Rights Reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +** OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +** FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +** DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +** SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +** HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +** LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +** OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +** SUCH DAMAGE. + */ + +package dropbox + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/url" + "strconv" + "time" +) + +type row struct { + TID string `json:"tid"` + RowID string `json:"rowid"` + Data Fields `json:"data"` +} + +type infoDict struct { + Title string `json:"title"` + MTime struct { + Time DBTime `json:"T"` + } `json:"mtime"` +} + +type datastoreInfo struct { + ID string `json:"dsid"` + Handle string `json:"handle"` + Revision int `json:"rev"` + Info infoDict `json:"info"` +} + +func (db *Dropbox) openOrCreateDatastore(dsID string) (int, string, bool, error) { + var r struct { + Revision int `json:"rev"` + Handle string `json:"handle"` + Created bool `json:"created"` + } + + err := db.doRequest("POST", "datastores/get_or_create_datastore", &url.Values{"dsid": {dsID}}, &r) + return r.Revision, r.Handle, r.Created, err +} + +func (db *Dropbox) listDatastores() ([]DatastoreInfo, string, error) { + var rv []DatastoreInfo + + var dl struct { + Info []datastoreInfo `json:"datastores"` + Token string `json:"token"` + } + + if err := db.doRequest("GET", "datastores/list_datastores", nil, &dl); err != nil { + return nil, "", err + } + rv = make([]DatastoreInfo, len(dl.Info)) + for i, di := range dl.Info { + rv[i] = DatastoreInfo{ + ID: di.ID, + handle: di.Handle, + revision: di.Revision, + title: di.Info.Title, + mtime: time.Time(di.Info.MTime.Time), + } + } + return rv, dl.Token, nil +} + +func (db *Dropbox) deleteDatastore(handle string) (*string, error) { + var r struct { + NotFound string `json:"notfound"` + OK string `json:"ok"` + } + + if err := db.doRequest("POST", "datastores/delete_datastore", &url.Values{"handle": {handle}}, &r); err != nil { + return nil, err + } + if len(r.NotFound) != 0 { + return nil, fmt.Errorf(r.NotFound) + } + return &r.OK, nil +} + +func generateDatastoreID() (string, error) { + var b []byte + var blen int + + b = make([]byte, 1) + _, err := io.ReadFull(rand.Reader, b) + if err != nil { + return "", err + } + blen = (int(b[0]) % maxGlobalIDLength) + 1 + b = make([]byte, blen) + _, err = io.ReadFull(rand.Reader, b) + if err != nil { + return "", err + } + + return encodeDBase64(b), nil +} + +func (db *Dropbox) createDatastore(key string) (int, string, bool, error) { + var r struct { + Revision int `json:"rev"` + Handle string `json:"handle"` + Created bool `json:"created"` + NotFound string `json:"notfound"` + } + var b64key string + var err error + + if len(key) != 0 { + b64key = encodeDBase64([]byte(key)) + } else { + b64key, err = generateDatastoreID() + if err != nil { + return 0, "", false, err + } + } + rhash := sha256.Sum256([]byte(b64key)) + dsID := "." + encodeDBase64(rhash[:]) + + params := &url.Values{ + "key": {b64key}, + "dsid": {dsID}, + } + if err := db.doRequest("POST", "datastores/create_datastore", params, &r); err != nil { + return 0, "", false, err + } + if len(r.NotFound) != 0 { + return 0, "", false, fmt.Errorf("%s", r.NotFound) + } + return r.Revision, r.Handle, r.Created, nil +} + +func (db *Dropbox) putDelta(handle string, rev int, changes listOfChanges) (int, error) { + var r struct { + Revision int `json:"rev"` + NotFound string `json:"notfound"` + Conflict string `json:"conflict"` + Error string `json:"error"` + } + var js []byte + var err error + + if len(changes) == 0 { + return rev, nil + } + + if js, err = json.Marshal(changes); err != nil { + return 0, err + } + + params := &url.Values{ + "handle": {handle}, + "rev": {strconv.FormatInt(int64(rev), 10)}, + "changes": {string(js)}, + } + + if err = db.doRequest("POST", "datastores/put_delta", params, &r); err != nil { + return 0, err + } + if len(r.NotFound) != 0 { + return 0, fmt.Errorf("%s", r.NotFound) + } + if len(r.Conflict) != 0 { + return 0, fmt.Errorf("%s", r.Conflict) + } + if len(r.Error) != 0 { + return 0, fmt.Errorf("%s", r.Error) + } + return r.Revision, nil +} + +func (db *Dropbox) getDelta(handle string, rev int) ([]datastoreDelta, error) { + var rv struct { + Deltas []datastoreDelta `json:"deltas"` + NotFound string `json:"notfound"` + } + err := db.doRequest("GET", "datastores/get_deltas", + &url.Values{ + "handle": {handle}, + "rev": {strconv.FormatInt(int64(rev), 10)}, + }, &rv) + + if len(rv.NotFound) != 0 { + return nil, fmt.Errorf("%s", rv.NotFound) + } + return rv.Deltas, err +} + +func (db *Dropbox) getSnapshot(handle string) ([]row, int, error) { + var r struct { + Rows []row `json:"rows"` + Revision int `json:"rev"` + NotFound string `json:"notfound"` + } + + if err := db.doRequest("GET", "datastores/get_snapshot", + &url.Values{"handle": {handle}}, &r); err != nil { + return nil, 0, err + } + if len(r.NotFound) != 0 { + return nil, 0, fmt.Errorf("%s", r.NotFound) + } + return r.Rows, r.Revision, nil +} + +func (db *Dropbox) await(cursors []*Datastore, token string) (string, []DatastoreInfo, map[string][]datastoreDelta, error) { + var params *url.Values + var dis []DatastoreInfo + var dd map[string][]datastoreDelta + + type awaitResult struct { + Deltas struct { + Results map[string]struct { + Deltas []datastoreDelta `json:"deltas"` + NotFound string `json:"notfound"` + } `json:"deltas"` + } `json:"get_deltas"` + Datastores struct { + Info []datastoreInfo `json:"datastores"` + Token string `json:"token"` + } `json:"list_datastores"` + } + var r awaitResult + if len(token) == 0 && len(cursors) == 0 { + return "", nil, nil, fmt.Errorf("at least one parameter required") + } + params = &url.Values{} + if len(token) != 0 { + js, err := json.Marshal(map[string]string{"token": token}) + if err != nil { + return "", nil, nil, err + } + params.Set("list_datastores", string(js)) + } + if len(cursors) != 0 { + m := make(map[string]int) + for _, ds := range cursors { + m[ds.info.handle] = ds.info.revision + } + js, err := json.Marshal(map[string]map[string]int{"cursors": m}) + if err != nil { + return "", nil, nil, err + } + params.Set("get_deltas", string(js)) + } + if err := db.doRequest("GET", "datastores/await", params, &r); err != nil { + return "", nil, nil, err + } + if len(r.Deltas.Results) == 0 && len(r.Datastores.Info) == 0 { + return token, nil, nil, fmt.Errorf("await timed out") + } + if len(r.Datastores.Token) != 0 { + token = r.Datastores.Token + } + if len(r.Deltas.Results) != 0 { + dd = make(map[string][]datastoreDelta) + for k, v := range r.Deltas.Results { + dd[k] = v.Deltas + } + } + if len(r.Datastores.Info) != 0 { + dis = make([]DatastoreInfo, len(r.Datastores.Info)) + for i, di := range r.Datastores.Info { + dis[i] = DatastoreInfo{ + ID: di.ID, + handle: di.Handle, + revision: di.Revision, + title: di.Info.Title, + mtime: time.Time(di.Info.MTime.Time), + } + } + } + return token, dis, dd, nil +} diff --git a/datastores_test.go b/datastores_test.go new file mode 100644 index 0000000..3707240 --- /dev/null +++ b/datastores_test.go @@ -0,0 +1,263 @@ +/* +** Copyright (c) 2014 Arnaud Ysmal. All Rights Reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +** OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +** FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +** DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +** SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +** HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +** LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +** OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +** SUCH DAMAGE. + */ + +package dropbox + +import ( + "encoding/json" + "testing" +) + +func checkList(t *testing.T, l *List, e []interface{}) { + var elt1 interface{} + var err error + + if l.Size() != len(e) { + t.Errorf("wrong size") + } + for i := range e { + if elt1, err = l.Get(i); err != nil { + t.Errorf("%s", err) + } + if elt1 != e[i] { + t.Errorf("position %d mismatch got %#v, expected %#v", i, elt1, e[i]) + } + } +} + +func newDatastore(t *testing.T) *Datastore { + var ds *Datastore + + ds = &Datastore{ + manager: newDropbox(t).NewDatastoreManager(), + info: DatastoreInfo{ + ID: "dummyID", + handle: "dummyHandle", + title: "dummyTitle", + revision: 0, + }, + tables: make(map[string]*Table), + changesQueue: make(chan changeWork), + } + go ds.doHandleChange() + return ds +} + +func TestList(t *testing.T) { + var tbl *Table + var r *Record + var ds *Datastore + var l *List + var err error + + ds = newDatastore(t) + + if tbl, err = ds.GetTable("dummyTable"); err != nil { + t.Errorf("%s", err) + } + if r, err = tbl.GetOrInsert("dummyRecord"); err != nil { + t.Errorf("%s", err) + } + if l, err = r.GetOrCreateList("dummyList"); err != nil { + t.Errorf("%s", err) + } + for i := 0; i < 10; i++ { + if err = l.Add(i); err != nil { + t.Errorf("%s", err) + } + } + if ftype, err := r.GetFieldType("dummyList"); err != nil || ftype != TypeList { + t.Errorf("wrong type") + } + + ftype, err := l.GetType(0) + if err != nil { + t.Errorf("%s", err) + } + if ftype != TypeInteger { + t.Errorf("wrong type") + } + + checkList(t, l, []interface{}{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + if err = l.Remove(5); err != nil { + t.Errorf("could not remove element 5") + } + checkList(t, l, []interface{}{0, 1, 2, 3, 4, 6, 7, 8, 9}) + + if err = l.Remove(0); err != nil { + t.Errorf("could not remove element 0") + } + checkList(t, l, []interface{}{1, 2, 3, 4, 6, 7, 8, 9}) + + if err = l.Remove(7); err != nil { + t.Errorf("could not remove element 7") + } + checkList(t, l, []interface{}{1, 2, 3, 4, 6, 7, 8}) + + if err = l.Remove(7); err == nil { + t.Errorf("out of bound index must return an error") + } + checkList(t, l, []interface{}{1, 2, 3, 4, 6, 7, 8}) + + if err = l.Move(3, 6); err != nil { + t.Errorf("could not move element 3 to position 6") + } + checkList(t, l, []interface{}{1, 2, 3, 6, 7, 8, 4}) + + if err = l.Move(3, 9); err == nil { + t.Errorf("out of bound index must return an error") + } + checkList(t, l, []interface{}{1, 2, 3, 6, 7, 8, 4}) + + if err = l.Move(6, 3); err != nil { + t.Errorf("could not move element 6 to position 3") + } + checkList(t, l, []interface{}{1, 2, 3, 4, 6, 7, 8}) + + if err = l.AddAtPos(0, 0); err != nil { + t.Errorf("could not insert element at position 0") + } + checkList(t, l, []interface{}{0, 1, 2, 3, 4, 6, 7, 8}) + + if err = l.Add(9); err != nil { + t.Errorf("could not append element") + } + checkList(t, l, []interface{}{0, 1, 2, 3, 4, 6, 7, 8, 9}) + + if err = l.AddAtPos(5, 5); err != nil { + t.Errorf("could not insert element at position 5") + } + checkList(t, l, []interface{}{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + if err = l.Set(0, 3); err != nil { + t.Errorf("could not update element at position 0") + } + checkList(t, l, []interface{}{3, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + + if err = l.Set(9, 2); err != nil { + t.Errorf("could not update element at position 9") + } + checkList(t, l, []interface{}{3, 1, 2, 3, 4, 5, 6, 7, 8, 2}) + + if err = l.Set(10, 11); err == nil { + t.Errorf("out of bound index must return an error") + } + checkList(t, l, []interface{}{3, 1, 2, 3, 4, 5, 6, 7, 8, 2}) +} + +func TestGenerateID(t *testing.T) { + f, err := generateDatastoreID() + if err != nil { + t.Errorf("%s", err) + } + if !isValidDatastoreID(f) { + t.Errorf("generated ID is not correct") + } +} + +func TestUnmarshalAwait(t *testing.T) { + type awaitResult struct { + Deltas struct { + Results map[string]struct { + Deltas []datastoreDelta `json:"deltas"` + } `json:"deltas"` + } `json:"get_deltas"` + Datastores struct { + Info []datastoreInfo `json:"datastores"` + Token string `json:"token"` + } `json:"list_datastores"` + } + var r awaitResult + var datastoreID string + var res []datastoreDelta + + js := `{"get_deltas":{"deltas":{"12345678901234567890":{"deltas":[{"changes":[["I","dummyTable","dummyRecord",{}]],"nonce":"","rev":0},{"changes":[["U","dummyTable","dummyRecord",{"name":["P","dummy"]}],["U","dummyTable","dummyRecord",{"dummyList":["LC"]}]],"nonce":"","rev":1},{"changes":[["U","dummyTable","dummyRecord",{"dummyList":["LI",0,{"I":"0"}]}],["U","dummyTable","dummyRecord",{"dummyList":["LI",1,{"I":"1"}]}],["U","dummyTable","dummyRecord",{"dummyList":["LI",2,{"I":"2"}]}],["U","dummyTable","dummyRecord",{"dummyList":["LI",3,{"I":"3"}]}],["U","dummyTable","dummyRecord",{"dummyList":["LI",4,{"I":"4"}]}],["U","dummyTable","dummyRecord",{"dummyList":["LI",5,{"I":"5"}]}],["U","dummyTable","dummyRecord",{"dummyList":["LI",6,{"I":"6"}]}],["U","dummyTable","dummyRecord",{"dummyList":["LI",7,{"I":"7"}]}],["U","dummyTable","dummyRecord",{"dummyList":["LI",8,{"I":"8"}]}],["U","dummyTable","dummyRecord",{"dummyList":["LI",9,{"I":"9"}]}]],"nonce":"","rev":2},{"changes":[["D","dummyTable","dummyRecord"]],"nonce":"","rev":3}]}}}}` + datastoreID = "12345678901234567890" + + expected := []datastoreDelta{ + datastoreDelta{ + Revision: 0, + Changes: listOfChanges{ + &change{Op: recordInsert, TID: "dummyTable", RecordID: "dummyRecord", Data: Fields{}}, + }, + }, + datastoreDelta{ + Revision: 1, + Changes: listOfChanges{ + &change{Op: "U", TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"name": fieldOp{Op: "P", Index: 0, Data: value{values: []interface{}{"dummy"}}}}}, + &change{Op: "U", TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"dummyList": fieldOp{Op: "LC"}}}, + }, + }, + datastoreDelta{ + Revision: 2, + Changes: listOfChanges{ + &change{Op: recordUpdate, TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"dummyList": fieldOp{Op: listInsert, Index: 0, Data: value{values: []interface{}{int64(0)}}}}}, + &change{Op: recordUpdate, TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"dummyList": fieldOp{Op: listInsert, Index: 1, Data: value{values: []interface{}{int64(1)}}}}}, + &change{Op: recordUpdate, TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"dummyList": fieldOp{Op: listInsert, Index: 2, Data: value{values: []interface{}{int64(2)}}}}}, + &change{Op: recordUpdate, TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"dummyList": fieldOp{Op: listInsert, Index: 3, Data: value{values: []interface{}{int64(3)}}}}}, + &change{Op: recordUpdate, TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"dummyList": fieldOp{Op: listInsert, Index: 4, Data: value{values: []interface{}{int64(4)}}}}}, + &change{Op: recordUpdate, TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"dummyList": fieldOp{Op: listInsert, Index: 5, Data: value{values: []interface{}{int64(5)}}}}}, + &change{Op: recordUpdate, TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"dummyList": fieldOp{Op: listInsert, Index: 6, Data: value{values: []interface{}{int64(6)}}}}}, + &change{Op: recordUpdate, TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"dummyList": fieldOp{Op: listInsert, Index: 7, Data: value{values: []interface{}{int64(7)}}}}}, + &change{Op: recordUpdate, TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"dummyList": fieldOp{Op: listInsert, Index: 8, Data: value{values: []interface{}{int64(8)}}}}}, + &change{Op: recordUpdate, TID: "dummyTable", RecordID: "dummyRecord", Ops: opDict{"dummyList": fieldOp{Op: listInsert, Index: 9, Data: value{values: []interface{}{int64(9)}}}}}, + }, + }, + datastoreDelta{ + Revision: 3, + Changes: listOfChanges{ + &change{Op: "D", TID: "dummyTable", RecordID: "dummyRecord"}, + }, + }, + } + err := json.Unmarshal([]byte(js), &r) + if err != nil { + t.Errorf("%s", err) + } + if len(r.Deltas.Results) != 1 { + t.Errorf("wrong number of datastoreDelta") + } + + if tmp, ok := r.Deltas.Results[datastoreID]; !ok { + t.Fatalf("wrong datastore ID") + } else { + res = tmp.Deltas + } + if len(res) != len(expected) { + t.Fatalf("got %d results expected %s", len(res), len(expected)) + } + for i, d := range res { + ed := expected[i] + if d.Revision != ed.Revision { + t.Errorf("wrong revision got %d expected %s", d.Revision, expected[i].Revision) + } + for j, c := range d.Changes { + if !c.equals(ed.Changes[j]) { + t.Errorf("wrong change: got: %+v expected: %+v", *c, *ed.Changes[j]) + } + } + } +} diff --git a/dropbox.go b/dropbox.go index e29856d..51a9955 100644 --- a/dropbox.go +++ b/dropbox.go @@ -23,7 +23,7 @@ ** SUCH DAMAGE. */ -// Package dropbox implements the Dropbox core API. +// Package dropbox implements the Dropbox core and datastore API. package dropbox import (