Impl delete find filters for boltdb

This commit is contained in:
Bel LaPointe
2020-07-23 16:41:50 -06:00
parent 085150a7b5
commit bed265a228
13 changed files with 670 additions and 92 deletions

View File

@@ -6,6 +6,7 @@ type Config struct {
Port int Port int
DBURI string DBURI string
Database string Database string
DriverType string
FilePrefix string FilePrefix string
FileRoot string FileRoot string
} }
@@ -18,6 +19,7 @@ func New() Config {
as.Append(args.STRING, "fileprefix", "path prefix for file service", "/__files__") as.Append(args.STRING, "fileprefix", "path prefix for file service", "/__files__")
as.Append(args.STRING, "fileroot", "path to file hosting root", "/tmp/") as.Append(args.STRING, "fileroot", "path to file hosting root", "/tmp/")
as.Append(args.STRING, "database", "database name to use", "db") as.Append(args.STRING, "database", "database name to use", "db")
as.Append(args.STRING, "drivertype", "database driver to use", "boltdb")
if err := as.Parse(); err != nil { if err := as.Parse(); err != nil {
panic(err) panic(err)
@@ -29,5 +31,6 @@ func New() Config {
FilePrefix: as.GetString("fileprefix"), FilePrefix: as.GetString("fileprefix"),
FileRoot: as.GetString("fileroot"), FileRoot: as.GetString("fileroot"),
Database: as.GetString("database"), Database: as.GetString("database"),
DriverType: as.GetString("drivertype"),
} }
} }

172
storage/driver/boltdb.go Normal file
View File

@@ -0,0 +1,172 @@
package driver
import (
"context"
"errors"
"fmt"
"local/dndex/config"
"os"
"regexp"
"github.com/boltdb/bolt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type BoltDB struct {
db *bolt.DB
}
func NewBoltDB() *BoltDB {
config := config.New()
db, err := bolt.Open(config.DBURI, os.ModePerm, nil)
if err != nil {
panic(err)
}
return &BoltDB{
db: db,
}
}
func (bdb *BoltDB) count(ctx context.Context, namespace string, filter interface{}) (int, error) {
ch, err := bdb.Find(ctx, namespace, filter)
n := 0
for _ = range ch {
n += 1
}
return n, err
}
func (bdb *BoltDB) Find(_ context.Context, namespace string, filter interface{}) (chan bson.Raw, error) {
b, err := bson.Marshal(filter)
if err != nil {
return nil, err
}
m := bson.M{}
if err := bson.Unmarshal(b, &m); err != nil {
return nil, err
}
results := make([]bson.Raw, 0)
err = bdb.db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(namespace))
if bucket == nil {
return nil
}
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil && v != nil; k, v = cursor.Next() {
n := bson.M{}
if err := bson.Unmarshal(v, &n); err != nil {
return err
}
if matches(n, m) {
results = append(results, bson.Raw(v))
}
}
return nil
})
ch := make(chan bson.Raw)
go func() {
defer close(ch)
for i := range results {
ch <- results[i]
}
}()
return ch, err
}
func (bdb *BoltDB) Update(context.Context, string, interface{}, interface{}) error {
return errors.New("not impl")
}
func (bdb *BoltDB) Insert(context.Context, string, interface{}) error {
return errors.New("not impl")
}
func (bdb *BoltDB) Delete(_ context.Context, namespace string, filter interface{}) error {
b, err := bson.Marshal(filter)
if err != nil {
return err
}
m := bson.M{}
if err := bson.Unmarshal(b, &m); err != nil {
return err
}
return bdb.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(namespace))
if bucket == nil {
return nil
}
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil && v != nil; k, v = cursor.Next() {
n := bson.M{}
if err := bson.Unmarshal(v, &n); err != nil {
return err
}
if matches(n, m) {
if err := bucket.Delete(k); err != nil {
return err
}
}
}
return nil
})
}
func matches(doc, filter bson.M) bool {
for k, v := range filter {
if _, ok := doc[k]; !ok {
continue
}
switch v.(type) {
case map[string]interface{}, primitive.M:
m, ok := v.(map[string]interface{})
if !ok {
m = map[string]interface{}(v.(primitive.M))
}
for k2, v2 := range m {
switch k2 {
case "$regex":
pattern, ok := v2.(string)
if !ok {
return false
}
re, err := regexp.Compile(pattern)
if err != nil {
return false
}
if !re.MatchString(fmt.Sprint(doc[k])) {
return false
}
case "$in":
options, ok := v2.([]interface{})
if !ok {
options = []interface{}(v2.(primitive.A))
ok = true
}
matches := false
for _, option := range options {
if fmt.Sprint(doc[k]) == fmt.Sprint(option) {
matches = true
}
}
if !matches {
return false
}
default:
dock, ok := doc[k].(map[string]interface{})
if !ok {
return false
}
if !matches(dock, bson.M(m)) {
return false
}
}
}
default:
if fmt.Sprint(v) != fmt.Sprint(doc[k]) {
return false
}
}
}
return true
}

View File

@@ -0,0 +1,252 @@
package driver
import (
"context"
"io/ioutil"
"local/dndex/storage/entity"
"local/dndex/storage/operator"
"os"
"testing"
"time"
"github.com/boltdb/bolt"
"github.com/google/uuid"
"go.mongodb.org/mongo-driver/bson"
)
const (
testN = 5
testNS = "col"
)
func TestNewBoltDB(t *testing.T) {
_, can := tempBoltDB(t)
defer can()
}
func TestBoltDBCount(t *testing.T) {
bdb, can := tempBoltDB(t)
defer can()
ch, err := bdb.Find(context.TODO(), testNS, map[string]string{})
if err != nil {
t.Fatal(err)
}
ones := make([]entity.One, testN)
i := 0
for j := range ch {
var o entity.One
if err := bson.Unmarshal(j, &o); err != nil {
t.Fatal(err)
}
ones[i] = o
i += 1
}
for name, filter := range map[string]struct {
filter interface{}
match bool
}{
"one.Query": {
filter: ones[0].Query(),
match: true,
},
"title:title": {
filter: map[string]interface{}{entity.Title: ones[1].Title},
match: true,
},
"title:title, text:text": {
filter: map[string]interface{}{entity.Title: ones[2].Title, entity.Text: ones[2].Text},
match: true,
},
"title:title, text:gibberish": {
filter: map[string]interface{}{entity.Title: ones[3].Title, entity.Text: ones[2].Text},
match: false,
},
"name:$in[gibberish]": {
filter: operator.NewFilterIn(entity.Name, []string{ones[0].Name + ones[1].Name}),
match: false,
},
"name:$in[name]": {
filter: operator.NewFilterIn(entity.Name, []string{ones[0].Name}),
match: true,
},
"name:$regex[gibberish]": {
filter: operator.Regex{entity.Name, ones[3].Name + ones[4].Name},
match: false,
},
"name:$regex[name]": {
filter: operator.Regex{entity.Name, ones[3].Name},
match: true,
},
} {
f := filter
t.Run(name, func(t *testing.T) {
n, err := bdb.count(context.TODO(), testNS, f.filter)
if err != nil {
t.Fatal(err)
}
if f.match && n != 1 {
t.Fatalf("%v results for %+v, want match=%v", n, f, f.match)
} else if !f.match && n != 0 {
t.Fatalf("%v results for %+v, want match=%v", n, f, f.match)
}
})
}
}
func TestBoltDBFind(t *testing.T) {
bdb, can := tempBoltDB(t)
defer can()
ch, err := bdb.Find(context.TODO(), testNS, map[string]string{})
if err != nil {
t.Fatal(err)
}
n := 0
for b := range ch {
n += 1
o := entity.One{}
if err := bson.Unmarshal(b, &o); err != nil {
t.Fatal(err)
}
if o.Type == "" {
t.Error(o.Type)
}
if o.Title == "" {
t.Error(o.Title)
}
if o.Image == "" {
t.Error(o.Image)
}
if o.Text == "" {
t.Error(o.Text)
}
if o.Relationship != "" {
t.Error(o.Relationship)
}
if o.Modified == 0 {
t.Error(o.Modified)
}
if len(o.Connections) == 0 {
t.Error(o.Connections)
}
for k := range o.Connections {
if o.Connections[k].Name == "" {
t.Error(o.Connections[k])
}
if o.Connections[k].Title == "" {
t.Error(o.Connections[k])
}
if o.Connections[k].Relationship == "" {
t.Error(o.Connections[k])
}
if o.Connections[k].Type == "" {
t.Error(o.Connections[k])
}
}
}
if n != testN {
t.Fatal(n)
}
}
func TestBoltDBUpdate(t *testing.T) {
t.Fatal("not impl")
}
func TestBoltDBInsert(t *testing.T) {
t.Fatal("not impl")
}
func TestBoltDBDelete(t *testing.T) {
bdb, can := tempBoltDB(t)
defer can()
ch, err := bdb.Find(context.TODO(), testNS, map[string]string{})
if err != nil {
t.Fatal(err)
}
ones := make([]entity.One, testN)
i := 0
for j := range ch {
var o entity.One
if err := bson.Unmarshal(j, &o); err != nil {
t.Fatal(err)
}
ones[i] = o
i += 1
}
wantN := testN
for _, filter := range []interface{}{
ones[0].Query(),
operator.NewFilterIn(entity.Title, []string{ones[1].Title}),
operator.Regex{entity.Text, ones[2].Text},
} {
err = bdb.Delete(context.TODO(), testNS, filter)
if err != nil {
t.Fatal(err)
}
wantN -= 1
n, err := bdb.count(context.TODO(), testNS, map[string]string{})
if err != nil {
t.Fatal(err)
}
if n != wantN {
t.Error(n, filter)
}
}
}
func tempBoltDB(t *testing.T) (*BoltDB, func()) {
f, err := ioutil.TempFile(os.TempDir(), "*.bolt.db")
if err != nil {
t.Fatal(err)
}
f.Close()
os.Args = []string{"a", "-dburi", f.Name()}
bdb := NewBoltDB()
fillBoltDB(t, bdb)
return bdb, func() {
bdb.db.Close()
os.Remove(f.Name())
}
}
func fillBoltDB(t *testing.T, bdb *BoltDB) {
if err := bdb.db.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists([]byte(testNS))
if err != nil {
return err
}
for i := 0; i < testN; i++ {
o := entity.One{
Name: "name-" + uuid.New().String()[:5],
Type: "type-" + uuid.New().String()[:5],
Title: "titl-" + uuid.New().String()[:5],
Image: "imge-" + uuid.New().String()[:5],
Text: "text-" + uuid.New().String()[:5],
Modified: time.Now().UnixNano(),
Connections: map[string]entity.One{},
}
p := entity.One{
Name: "name-" + uuid.New().String()[:5],
Type: "type-" + uuid.New().String()[:5],
Relationship: "rshp-" + uuid.New().String()[:5],
Title: "titl-" + uuid.New().String()[:5],
}
o.Connections[p.Name] = p
b, err := bson.Marshal(o)
if err != nil {
return err
}
if err := bucket.Put([]byte(o.Name), b); err != nil {
return err
}
}
return nil
}); err != nil {
t.Fatal(err)
}
}

26
storage/driver/driver.go Normal file
View File

@@ -0,0 +1,26 @@
package driver
import (
"context"
"local/dndex/config"
"strings"
"go.mongodb.org/mongo-driver/bson"
)
type Driver interface {
Find(context.Context, string, interface{}) (chan bson.Raw, error)
Update(context.Context, string, interface{}, interface{}) error
Insert(context.Context, string, interface{}) error
Delete(context.Context, string, interface{}) error
}
func New() Driver {
switch strings.ToLower(config.New().DriverType) {
case "mongo":
return NewMongo()
case "boltdb":
return NewBoltDB()
}
panic("unknown driver type " + strings.ToLower(config.New().DriverType))
}

View File

@@ -0,0 +1,10 @@
package driver
import "testing"
func TestDriver(t *testing.T) {
var driver Driver
driver = &Mongo{}
driver = &BoltDB{}
t.Log(driver)
}

View File

@@ -1,4 +1,4 @@
package storage package driver
import ( import (
"context" "context"
@@ -85,7 +85,7 @@ func (m Mongo) Insert(ctx context.Context, namespace string, apply interface{})
func (m Mongo) Delete(ctx context.Context, namespace string, filter interface{}) error { func (m Mongo) Delete(ctx context.Context, namespace string, filter interface{}) error {
c := m.client.Database(m.db).Collection(namespace) c := m.client.Database(m.db).Collection(namespace)
_, err := c.DeleteOne(ctx, filter) _, err := c.DeleteMany(ctx, filter)
return err return err
} }

View File

@@ -3,6 +3,7 @@ package storage
import ( import (
"context" "context"
"fmt" "fmt"
"local/dndex/storage/driver"
"local/dndex/storage/entity" "local/dndex/storage/entity"
"local/dndex/storage/operator" "local/dndex/storage/operator"
@@ -10,13 +11,13 @@ import (
) )
type Graph struct { type Graph struct {
mongo Mongo driver driver.Driver
} }
func NewGraph() Graph { func NewGraph() Graph {
mongo := NewMongo() mongo := driver.NewMongo()
return Graph{ return Graph{
mongo: mongo, driver: mongo,
} }
} }
@@ -31,7 +32,7 @@ func (g Graph) List(ctx context.Context, namespace string, from ...string) ([]en
} }
func (g Graph) find(ctx context.Context, namespace string, filter interface{}) ([]entity.One, error) { func (g Graph) find(ctx context.Context, namespace string, filter interface{}) ([]entity.One, error) {
ch, err := g.mongo.Find(ctx, namespace, filter) ch, err := g.driver.Find(ctx, namespace, filter)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -51,13 +52,13 @@ func (g Graph) gatherOnes(ctx context.Context, ch <-chan bson.Raw) ([]entity.One
} }
func (g Graph) Insert(ctx context.Context, namespace string, one entity.One) error { func (g Graph) Insert(ctx context.Context, namespace string, one entity.One) error {
return g.mongo.Insert(ctx, namespace, one) return g.driver.Insert(ctx, namespace, one)
} }
func (g Graph) Update(ctx context.Context, namespace string, one entity.One, modify interface{}) error { func (g Graph) Update(ctx context.Context, namespace string, one entity.One, modify interface{}) error {
return g.mongo.Update(ctx, namespace, one, modify) return g.driver.Update(ctx, namespace, one, modify)
} }
func (g Graph) Delete(ctx context.Context, namespace string, filter interface{}) error { func (g Graph) Delete(ctx context.Context, namespace string, filter interface{}) error {
return g.mongo.Delete(ctx, namespace, filter) return g.driver.Delete(ctx, namespace, filter)
} }

View File

@@ -20,11 +20,10 @@ func TestIntegration(t *testing.T) {
os.Args = os.Args[:1] os.Args = os.Args[:1]
graph := NewGraph() graph := NewGraph()
graph.mongo.db = "db"
ctx, can := context.WithCancel(context.TODO()) ctx, can := context.WithCancel(context.TODO())
defer can() defer can()
clean := func() { clean := func() {
graph.mongo.client.Database(graph.mongo.db).Collection("col").DeleteMany(ctx, map[string]interface{}{}) graph.driver.Delete(context.TODO(), "col", map[string]string{})
} }
clean() clean()
defer clean() defer clean()

49
view/httpisnow.go Normal file
View File

@@ -0,0 +1,49 @@
package view
import (
"fmt"
"io/ioutil"
"local/dndex/storage"
"local/dndex/storage/entity"
"local/dndex/storage/operator"
"net/http"
"path"
"github.com/buger/jsonparser"
)
func httpisnow(g storage.Graph, w http.ResponseWriter, r *http.Request) error {
namespace := path.Base(r.URL.Path)
b, err := ioutil.ReadAll(r.Body)
if err != nil {
return err
}
name, err := jsonparser.GetString(b, entity.JSONName)
if err != nil {
return fmt.Errorf("cannot find %q: %v", entity.JSONName, err)
}
if name == "" {
http.Error(w, `{"error":"must provide a name"}`, http.StatusBadRequest)
return nil
}
key, err := jsonparser.GetString(b, "set", "key")
if err != nil {
return fmt.Errorf("cannot find %q: %v", "set.key", err)
}
if key == "" {
http.Error(w, `{"error":"must provide a set.key"}`, http.StatusBadRequest)
return nil
}
value, err := jsonparser.GetString(b, "set", "value")
if err != nil {
return fmt.Errorf("cannot find %q: %v", "set.value", err)
}
operator := operator.Set{Key: key, Value: value}
return g.Update(r.Context(), namespace, entity.One{Name: name}, operator)
}

13
view/httpmeet.go Normal file
View File

@@ -0,0 +1,13 @@
package view
import (
"local/dndex/storage"
"net/http"
"path"
)
func httpmeet(g storage.Graph, w http.ResponseWriter, r *http.Request) error {
namespace := path.Base(r.URL.Path)
w.Write([]byte(namespace))
return nil
}

73
view/httpwho.go Normal file
View File

@@ -0,0 +1,73 @@
package view
import (
"context"
"encoding/json"
"local/dndex/storage"
"local/dndex/storage/entity"
"log"
"net/http"
"path"
"strings"
)
func httpwho(g storage.Graph, w http.ResponseWriter, r *http.Request) error {
namespace := strings.TrimLeft(r.URL.Path, path.Dir(r.URL.Path))
if len(namespace) == 0 {
http.NotFound(w, r)
return nil
}
ids := r.URL.Query()["id"]
_, verbose := r.URL.Query()["v"]
_, one := r.URL.Query()["one"]
results := make(map[string]entity.One)
for i := 0; i < len(ids); i++ {
id := ids[i]
one, err := httpwhoOne(r.Context(), g, namespace, id, verbose)
if err != nil {
return err
}
results[id] = one
}
var marshalme interface{}
if one {
for k := range results {
marshalme = results[k]
break
}
} else {
marshalme = results
}
log.Printf("id=%+v, one=%v, verbose=%v, results:%+v", ids, one, verbose, marshalme)
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
return enc.Encode(marshalme)
}
func httpwhoOne(ctx context.Context, g storage.Graph, namespace, id string, verbose bool) (entity.One, error) {
ones, err := g.List(ctx, namespace, id)
if err != nil {
return entity.One{}, err
}
if len(ones) != 1 {
ones = append(ones, entity.One{})
}
one := ones[0]
if verbose {
ones, err := g.List(ctx, namespace, one.Peers()...)
if err != nil {
return entity.One{}, err
}
for _, another := range ones {
another.Connections = nil
another.Text = ""
for j := range one.Connections {
if one.Connections[j].Name == another.Name {
another.Relationship = one.Connections[j].Relationship
one.Connections[j] = another
}
}
}
}
return one, nil
}

View File

@@ -1,104 +1,64 @@
package view package view
import ( import (
"context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"local/dndex/config" "local/dndex/config"
"local/dndex/storage" "local/dndex/storage"
"local/dndex/storage/entity"
"local/gziphttp" "local/gziphttp"
"log" "log"
"net/http" "net/http"
"path"
"strings" "strings"
) )
func JSON(g storage.Graph) error { func JSON(g storage.Graph) error {
port := config.New().Port port := config.New().Port
log.Println("listening on", port) log.Println("listening on", port)
err := http.ListenAndServe(fmt.Sprintf(":%d", port), foo(g)) err := http.ListenAndServe(fmt.Sprintf(":%d", port), jsonHandler(g))
return err return err
} }
func foo(g storage.Graph) http.Handler { func jsonHandler(g storage.Graph) http.Handler {
mux := http.NewServeMux()
routes := []struct {
path string
foo func(g storage.Graph, w http.ResponseWriter, r *http.Request) error
}{
{
path: "/who/",
foo: httpwho,
},
{
path: "/meet/",
foo: httpmeet,
},
{
path: "/isnow/",
foo: httpisnow,
},
}
for _, route := range routes {
path := route.path
nopath := strings.TrimRight(route.path, "/")
foo := route.foo
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if err := foo(g, w, r); err != nil {
b, _ := json.Marshal(map[string]string{"error": err.Error()})
http.Error(w, string(b), http.StatusInternalServerError)
}
})
mux.HandleFunc(nopath, http.NotFound)
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Origin", "*")
if gziphttp.Can(r) { if gziphttp.Can(r) {
w = gziphttp.New(w) gz := gziphttp.New(w)
} defer gz.Close()
var err error w = gz
switch path.Base(r.URL.Path) {
case "who":
err = who(g, w, r)
default:
http.NotFound(w, r)
return
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
} }
mux.ServeHTTP(w, r)
}) })
} }
func who(g storage.Graph, w http.ResponseWriter, r *http.Request) error {
namespace := path.Dir(r.URL.Path)
if len(namespace) < 2 {
http.NotFound(w, r)
return nil
}
namespace = strings.Replace(namespace[1:], "/", ".", -1)
ids := r.URL.Query()["id"]
_, verbose := r.URL.Query()["v"]
_, one := r.URL.Query()["one"]
results := make(map[string]entity.One)
for i := 0; i < len(ids); i++ {
id := ids[i]
one, err := whoOne(r.Context(), g, namespace, id, verbose)
if err != nil {
return err
}
results[id] = one
}
var marshalme interface{}
if one {
for k := range results {
marshalme = results[k]
break
}
} else {
marshalme = results
}
log.Printf("id=%+v, one=%v, verbose=%v, results:%+v", ids, one, verbose, marshalme)
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
return enc.Encode(marshalme)
}
func whoOne(ctx context.Context, g storage.Graph, namespace, id string, verbose bool) (entity.One, error) {
ones, err := g.List(ctx, namespace, id)
if err != nil {
return entity.One{}, err
}
if len(ones) != 1 {
ones = append(ones, entity.One{})
}
one := ones[0]
if verbose {
ones, err := g.List(ctx, namespace, one.Peers()...)
if err != nil {
return entity.One{}, err
}
for _, another := range ones {
another.Connections = nil
another.Text = ""
for j := range one.Connections {
if one.Connections[j].Name == another.Name {
another.Relationship = one.Connections[j].Relationship
one.Connections[j] = another
}
}
}
}
return one, nil
}

View File

@@ -11,6 +11,7 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os" "os"
"strings"
"testing" "testing"
"time" "time"
@@ -28,8 +29,27 @@ func TestJSON(t *testing.T) {
ones := fillDB(t, g) ones := fillDB(t, g)
want := ones[len(ones)-1] want := ones[len(ones)-1]
handler := foo(g) handler := jsonHandler(g)
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%d/col/who?id=%s&v", config.New().Port, want.Name), nil)
t.Run("404 on /who", func(t *testing.T) {
r := httptest.NewRequest("GET", "/who", strings.NewReader(``))
w := httptest.NewRecorder()
handler.ServeHTTP(w, r)
if w.Code != http.StatusNotFound {
t.Error(w.Code)
}
})
t.Run("404 on /who/", func(t *testing.T) {
r := httptest.NewRequest("GET", "/who/", strings.NewReader(``))
w := httptest.NewRecorder()
handler.ServeHTTP(w, r)
if w.Code != http.StatusNotFound {
t.Error(w.Code)
}
})
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%d/who/col?id=%s&v", config.New().Port, want.Name), strings.NewReader(``))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }