Impl delete find filters for boltdb

storage/driver/boltdb.go

@@ -0,0 +1,172 @@
+package driver
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"local/dndex/config"
+	"os"
+	"regexp"
+
+	"github.com/boltdb/bolt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type BoltDB struct {
+	db *bolt.DB
+}
+
+func NewBoltDB() *BoltDB {
+	config := config.New()
+	db, err := bolt.Open(config.DBURI, os.ModePerm, nil)
+	if err != nil {
+		panic(err)
+	}
+	return &BoltDB{
+		db: db,
+	}
+}
+
+func (bdb *BoltDB) count(ctx context.Context, namespace string, filter interface{}) (int, error) {
+	ch, err := bdb.Find(ctx, namespace, filter)
+	n := 0
+	for _ = range ch {
+		n += 1
+	}
+	return n, err
+}
+
+func (bdb *BoltDB) Find(_ context.Context, namespace string, filter interface{}) (chan bson.Raw, error) {
+	b, err := bson.Marshal(filter)
+	if err != nil {
+		return nil, err
+	}
+	m := bson.M{}
+	if err := bson.Unmarshal(b, &m); err != nil {
+		return nil, err
+	}
+	results := make([]bson.Raw, 0)
+	err = bdb.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(namespace))
+		if bucket == nil {
+			return nil
+		}
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil && v != nil; k, v = cursor.Next() {
+			n := bson.M{}
+			if err := bson.Unmarshal(v, &n); err != nil {
+				return err
+			}
+			if matches(n, m) {
+				results = append(results, bson.Raw(v))
+			}
+		}
+		return nil
+	})
+	ch := make(chan bson.Raw)
+	go func() {
+		defer close(ch)
+		for i := range results {
+			ch <- results[i]
+		}
+	}()
+	return ch, err
+}
+
+func (bdb *BoltDB) Update(context.Context, string, interface{}, interface{}) error {
+	return errors.New("not impl")
+}
+
+func (bdb *BoltDB) Insert(context.Context, string, interface{}) error {
+	return errors.New("not impl")
+}
+
+func (bdb *BoltDB) Delete(_ context.Context, namespace string, filter interface{}) error {
+	b, err := bson.Marshal(filter)
+	if err != nil {
+		return err
+	}
+	m := bson.M{}
+	if err := bson.Unmarshal(b, &m); err != nil {
+		return err
+	}
+	return bdb.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(namespace))
+		if bucket == nil {
+			return nil
+		}
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil && v != nil; k, v = cursor.Next() {
+			n := bson.M{}
+			if err := bson.Unmarshal(v, &n); err != nil {
+				return err
+			}
+			if matches(n, m) {
+				if err := bucket.Delete(k); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	})
+}
+
+func matches(doc, filter bson.M) bool {
+	for k, v := range filter {
+		if _, ok := doc[k]; !ok {
+			continue
+		}
+		switch v.(type) {
+		case map[string]interface{}, primitive.M:
+			m, ok := v.(map[string]interface{})
+			if !ok {
+				m = map[string]interface{}(v.(primitive.M))
+			}
+			for k2, v2 := range m {
+				switch k2 {
+				case "$regex":
+					pattern, ok := v2.(string)
+					if !ok {
+						return false
+					}
+					re, err := regexp.Compile(pattern)
+					if err != nil {
+						return false
+					}
+					if !re.MatchString(fmt.Sprint(doc[k])) {
+						return false
+					}
+				case "$in":
+					options, ok := v2.([]interface{})
+					if !ok {
+						options = []interface{}(v2.(primitive.A))
+						ok = true
+					}
+					matches := false
+					for _, option := range options {
+						if fmt.Sprint(doc[k]) == fmt.Sprint(option) {
+							matches = true
+						}
+					}
+					if !matches {
+						return false
+					}
+				default:
+					dock, ok := doc[k].(map[string]interface{})
+					if !ok {
+						return false
+					}
+					if !matches(dock, bson.M(m)) {
+						return false
+					}
+				}
+			}
+		default:
+			if fmt.Sprint(v) != fmt.Sprint(doc[k]) {
+				return false
+			}
+		}
+	}
+	return true
+}

storage/driver/boltdb_test.go

@@ -0,0 +1,252 @@
+package driver
+
+import (
+	"context"
+	"io/ioutil"
+	"local/dndex/storage/entity"
+	"local/dndex/storage/operator"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/google/uuid"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+const (
+	testN  = 5
+	testNS = "col"
+)
+
+func TestNewBoltDB(t *testing.T) {
+	_, can := tempBoltDB(t)
+	defer can()
+}
+
+func TestBoltDBCount(t *testing.T) {
+	bdb, can := tempBoltDB(t)
+	defer can()
+
+	ch, err := bdb.Find(context.TODO(), testNS, map[string]string{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ones := make([]entity.One, testN)
+	i := 0
+	for j := range ch {
+		var o entity.One
+		if err := bson.Unmarshal(j, &o); err != nil {
+			t.Fatal(err)
+		}
+		ones[i] = o
+		i += 1
+	}
+
+	for name, filter := range map[string]struct {
+		filter interface{}
+		match  bool
+	}{
+		"one.Query": {
+			filter: ones[0].Query(),
+			match:  true,
+		},
+		"title:title": {
+			filter: map[string]interface{}{entity.Title: ones[1].Title},
+			match:  true,
+		},
+		"title:title, text:text": {
+			filter: map[string]interface{}{entity.Title: ones[2].Title, entity.Text: ones[2].Text},
+			match:  true,
+		},
+		"title:title, text:gibberish": {
+			filter: map[string]interface{}{entity.Title: ones[3].Title, entity.Text: ones[2].Text},
+			match:  false,
+		},
+		"name:$in[gibberish]": {
+			filter: operator.NewFilterIn(entity.Name, []string{ones[0].Name + ones[1].Name}),
+			match:  false,
+		},
+		"name:$in[name]": {
+			filter: operator.NewFilterIn(entity.Name, []string{ones[0].Name}),
+			match:  true,
+		},
+		"name:$regex[gibberish]": {
+			filter: operator.Regex{entity.Name, ones[3].Name + ones[4].Name},
+			match:  false,
+		},
+		"name:$regex[name]": {
+			filter: operator.Regex{entity.Name, ones[3].Name},
+			match:  true,
+		},
+	} {
+		f := filter
+		t.Run(name, func(t *testing.T) {
+			n, err := bdb.count(context.TODO(), testNS, f.filter)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if f.match && n != 1 {
+				t.Fatalf("%v results for %+v, want match=%v", n, f, f.match)
+			} else if !f.match && n != 0 {
+				t.Fatalf("%v results for %+v, want match=%v", n, f, f.match)
+			}
+		})
+	}
+}
+
+func TestBoltDBFind(t *testing.T) {
+	bdb, can := tempBoltDB(t)
+	defer can()
+
+	ch, err := bdb.Find(context.TODO(), testNS, map[string]string{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	n := 0
+	for b := range ch {
+		n += 1
+		o := entity.One{}
+		if err := bson.Unmarshal(b, &o); err != nil {
+			t.Fatal(err)
+		}
+		if o.Type == "" {
+			t.Error(o.Type)
+		}
+		if o.Title == "" {
+			t.Error(o.Title)
+		}
+		if o.Image == "" {
+			t.Error(o.Image)
+		}
+		if o.Text == "" {
+			t.Error(o.Text)
+		}
+		if o.Relationship != "" {
+			t.Error(o.Relationship)
+		}
+		if o.Modified == 0 {
+			t.Error(o.Modified)
+		}
+		if len(o.Connections) == 0 {
+			t.Error(o.Connections)
+		}
+		for k := range o.Connections {
+			if o.Connections[k].Name == "" {
+				t.Error(o.Connections[k])
+			}
+			if o.Connections[k].Title == "" {
+				t.Error(o.Connections[k])
+			}
+			if o.Connections[k].Relationship == "" {
+				t.Error(o.Connections[k])
+			}
+			if o.Connections[k].Type == "" {
+				t.Error(o.Connections[k])
+			}
+		}
+	}
+	if n != testN {
+		t.Fatal(n)
+	}
+}
+
+func TestBoltDBUpdate(t *testing.T) {
+	t.Fatal("not impl")
+}
+
+func TestBoltDBInsert(t *testing.T) {
+	t.Fatal("not impl")
+}
+
+func TestBoltDBDelete(t *testing.T) {
+	bdb, can := tempBoltDB(t)
+	defer can()
+
+	ch, err := bdb.Find(context.TODO(), testNS, map[string]string{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ones := make([]entity.One, testN)
+	i := 0
+	for j := range ch {
+		var o entity.One
+		if err := bson.Unmarshal(j, &o); err != nil {
+			t.Fatal(err)
+		}
+		ones[i] = o
+		i += 1
+	}
+
+	wantN := testN
+	for _, filter := range []interface{}{
+		ones[0].Query(),
+		operator.NewFilterIn(entity.Title, []string{ones[1].Title}),
+		operator.Regex{entity.Text, ones[2].Text},
+	} {
+		err = bdb.Delete(context.TODO(), testNS, filter)
+		if err != nil {
+			t.Fatal(err)
+		}
+		wantN -= 1
+		n, err := bdb.count(context.TODO(), testNS, map[string]string{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if n != wantN {
+			t.Error(n, filter)
+		}
+	}
+}
+
+func tempBoltDB(t *testing.T) (*BoltDB, func()) {
+	f, err := ioutil.TempFile(os.TempDir(), "*.bolt.db")
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+	os.Args = []string{"a", "-dburi", f.Name()}
+	bdb := NewBoltDB()
+	fillBoltDB(t, bdb)
+	return bdb, func() {
+		bdb.db.Close()
+		os.Remove(f.Name())
+	}
+}
+
+func fillBoltDB(t *testing.T, bdb *BoltDB) {
+	if err := bdb.db.Update(func(tx *bolt.Tx) error {
+		bucket, err := tx.CreateBucketIfNotExists([]byte(testNS))
+		if err != nil {
+			return err
+		}
+		for i := 0; i < testN; i++ {
+			o := entity.One{
+				Name:        "name-" + uuid.New().String()[:5],
+				Type:        "type-" + uuid.New().String()[:5],
+				Title:       "titl-" + uuid.New().String()[:5],
+				Image:       "imge-" + uuid.New().String()[:5],
+				Text:        "text-" + uuid.New().String()[:5],
+				Modified:    time.Now().UnixNano(),
+				Connections: map[string]entity.One{},
+			}
+			p := entity.One{
+				Name:         "name-" + uuid.New().String()[:5],
+				Type:         "type-" + uuid.New().String()[:5],
+				Relationship: "rshp-" + uuid.New().String()[:5],
+				Title:        "titl-" + uuid.New().String()[:5],
+			}
+			o.Connections[p.Name] = p
+			b, err := bson.Marshal(o)
+			if err != nil {
+				return err
+			}
+			if err := bucket.Put([]byte(o.Name), b); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}

storage/driver/driver.go

@@ -0,0 +1,26 @@
+package driver
+
+import (
+	"context"
+	"local/dndex/config"
+	"strings"
+
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+type Driver interface {
+	Find(context.Context, string, interface{}) (chan bson.Raw, error)
+	Update(context.Context, string, interface{}, interface{}) error
+	Insert(context.Context, string, interface{}) error
+	Delete(context.Context, string, interface{}) error
+}
+
+func New() Driver {
+	switch strings.ToLower(config.New().DriverType) {
+	case "mongo":
+		return NewMongo()
+	case "boltdb":
+		return NewBoltDB()
+	}
+	panic("unknown driver type " + strings.ToLower(config.New().DriverType))
+}

storage/driver/driver_test.go

@@ -0,0 +1,10 @@
+package driver
+
+import "testing"
+
+func TestDriver(t *testing.T) {
+	var driver Driver
+	driver = &Mongo{}
+	driver = &BoltDB{}
+	t.Log(driver)
+}

storage/driver/mon.go

@@ -0,0 +1,102 @@
+package driver
+
+import (
+	"context"
+	"errors"
+	"local/dndex/config"
+	"log"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type Mongo struct {
+	client *mongo.Client
+	db     string
+}
+
+func NewMongo() Mongo {
+	opts := options.Client().ApplyURI(config.New().DBURI)
+	c, err := mongo.NewClient(opts)
+	if err != nil {
+		panic(err)
+	}
+	if err := c.Connect(context.TODO()); err != nil {
+		panic(err)
+	}
+	ctx, can := context.WithTimeout(context.Background(), time.Second)
+	defer can()
+	if _, err := c.ListDatabaseNames(ctx, map[string]interface{}{}); err != nil {
+		panic(err)
+	}
+	return Mongo{
+		client: c,
+		db:     config.New().Database,
+	}
+}
+
+func (m Mongo) Find(ctx context.Context, namespace string, filter interface{}) (chan bson.Raw, error) {
+	c := m.client.Database(m.db).Collection(namespace)
+	cursor, err := c.Find(ctx, filter)
+	if err != nil {
+		return nil, err
+	}
+	return m.page(ctx, cursor), nil
+}
+
+func (m Mongo) page(ctx context.Context, cursor *mongo.Cursor) chan bson.Raw {
+	ch := make(chan bson.Raw)
+	go func(chan<- bson.Raw) {
+		defer close(ch)
+		defer cursor.Close(ctx)
+		for cursor.Next(ctx) {
+			ch <- cursor.Current
+		}
+	}(ch)
+	return ch
+}
+
+func (m Mongo) Update(ctx context.Context, namespace string, filter, apply interface{}) error {
+	c := m.client.Database(m.db).Collection(namespace)
+	_, err := c.UpdateOne(ctx, filter, apply)
+	return err
+}
+
+func (m Mongo) Insert(ctx context.Context, namespace string, apply interface{}) error {
+	b, err := bson.Marshal(apply)
+	if err != nil {
+		return err
+	}
+	var mapp map[string]interface{}
+	if err := bson.Unmarshal(b, &mapp); err != nil {
+		return err
+	}
+	if _, ok := mapp["_id"]; !ok {
+		return errors.New("no _id in new object")
+	} else if _, ok := mapp["_id"].(string); !ok {
+		return errors.New("non-string _id in new object")
+	}
+	c := m.client.Database(m.db).Collection(namespace)
+	_, err = c.InsertOne(ctx, apply)
+	return err
+}
+
+func (m Mongo) Delete(ctx context.Context, namespace string, filter interface{}) error {
+	c := m.client.Database(m.db).Collection(namespace)
+	_, err := c.DeleteMany(ctx, filter)
+	return err
+}
+
+func peekBSON(v interface{}) {
+	b, err := bson.Marshal(v)
+	if err != nil {
+		panic(err)
+	}
+	var m map[string]interface{}
+	if err := bson.Unmarshal(b, &m); err != nil {
+		panic(err)
+	}
+	log.Printf("PEEK: %+v", m)
+}