Change config to use local/storage

config/config.go

@@ -4,14 +4,14 @@ import (
 	"io/ioutil"
 	"local/args"
 	"os"
+	"strings"
 	"time"
 )
 
 type Config struct {
 	Port         int
-	DBURI        string
 	Database     string
-	DriverType   string
+	Driver       []string
 	FilePrefix   string
 	FileRoot     string
 	Auth         bool
@@ -32,11 +32,10 @@ func New() Config {
 	f.Close()
 
 	as.Append(args.INT, "p", "port to listen on", 18114)
-	as.Append(args.STRING, "dburi", "database uri", f.Name())
 	as.Append(args.STRING, "fileprefix", "path prefix for file service", "/__files__")
 	as.Append(args.STRING, "fileroot", "path to file hosting root", "/tmp/")
 	as.Append(args.STRING, "database", "database name to use", "db")
-	as.Append(args.STRING, "driver-type", "database driver to use, [boltdb mongo map]", "map")
+	as.Append(args.STRING, "driver", "database driver args to use, like [local/storage.Type,arg1,arg2...] or [/path/to/boltdb]", "map")
 	as.Append(args.BOOL, "auth", "check for authorized access", false)
 	as.Append(args.DURATION, "authlifetime", "duration auth is valid for", time.Hour)
 	as.Append(args.DURATION, "delay", "time to delay requests", time.Duration(0))
@@ -51,11 +50,10 @@ func New() Config {
 
 	return Config{
 		Port:         as.GetInt("p"),
-		DBURI:        as.GetString("dburi"),
 		FilePrefix:   as.GetString("fileprefix"),
 		FileRoot:     as.GetString("fileroot"),
 		Database:     as.GetString("database"),
-		DriverType:   as.GetString("driver-type"),
+		Driver:       strings.Split(as.GetString("driver"), ","),
 		Auth:         as.GetBool("auth"),
 		AuthLifetime: as.GetDuration("authlifetime"),
 		Delay:        as.GetDuration("delay"),

main_test.go

@@ -27,7 +27,7 @@ func Test(t *testing.T) {
 	s := httptest.NewServer(http.HandlerFunc(nil))
 	s.Close()
 	p := strings.Split(s.URL, ":")[2]
-	os.Args = strings.Split(fmt.Sprintf(`dndex -auth=true -database db -delay 5ms -driver-type map -fileprefix /files -fileroot %s -p %v -rps 50 -sys-rps 40`, d, p), " ")
+	os.Args = strings.Split(fmt.Sprintf(`dndex -auth=true -database db -delay 5ms -driver map -fileprefix /files -fileroot %s -p %v -rps 50 -sys-rps 40`, d, p), " ")
 
 	go main()
 

storage/driver/boltdb_test.go

@@ -30,7 +30,7 @@ func TestBoltDBCount(t *testing.T) {
 	bdb, can := tempBoltDB(t)
 	defer can()
 
-	for _, driver := range []Driver{bdb, tempMap(t)} {
+	for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} {
 		t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) {
 			ch, err := driver.Find(context.TODO(), testNS, map[string]string{})
 			if err != nil {
@@ -121,7 +121,7 @@ func TestBoltDBFind(t *testing.T) {
 	bdb, can := tempBoltDB(t)
 	defer can()
 
-	for _, driver := range []Driver{bdb, tempMap(t)} {
+	for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} {
 		t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) {
 			ch, err := driver.Find(context.TODO(), testNS, map[string]string{})
 			if err != nil {
@@ -169,7 +169,7 @@ func TestBoltDBUpdate(t *testing.T) {
 	bdb, can := tempBoltDB(t)
 	defer can()
 
-	for _, driver := range []Driver{bdb, tempMap(t)} {
+	for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} {
 		t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) {
 			ch, err := driver.Find(context.TODO(), testNS, map[string]string{})
 			if err != nil {
@@ -250,7 +250,7 @@ func TestBoltDBInsert(t *testing.T) {
 	bdb, can := tempBoltDB(t)
 	defer can()
 
-	for _, driver := range []Driver{bdb, tempMap(t)} {
+	for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} {
 		t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) {
 			ch, err := driver.Find(context.TODO(), testNS, map[string]string{})
 			if err != nil {
@@ -320,7 +320,7 @@ func TestBoltDBDelete(t *testing.T) {
 	bdb, can := tempBoltDB(t)
 	defer can()
 
-	for _, driver := range []Driver{bdb, tempMap(t)} {
+	for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} {
 		t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) {
 			ch, err := driver.Find(context.TODO(), testNS, map[string]string{})
 			if err != nil {

storage/driver/driver.go

@@ -3,6 +3,7 @@ package driver
 import (
 	"context"
 	"local/dndex/config"
+	"local/storage"
 	"strings"
 
 	"go.mongodb.org/mongo-driver/bson"
@@ -18,9 +19,12 @@ type Driver interface {
 
 func New(path ...string) Driver {
 	if len(path) == 0 {
-		path = []string{config.New().DBURI}
+		path = config.New().Driver
 	}
-	switch strings.ToLower(config.New().DriverType) {
+	if t := storage.TypeFromString(path[0]); t >= 0 {
+		return NewStorage(path...)
+	}
+	switch strings.ToLower(config.New().Driver[0]) {
 	case "map":
 		return NewMap()
 	case "mongo":
@@ -28,5 +32,5 @@ func New(path ...string) Driver {
 	case "boltdb":
 		return NewBoltDB(path[0])
 	}
-	panic("unknown driver type " + strings.ToLower(config.New().DriverType))
+	panic("unknown driver type " + strings.ToLower(config.New().Driver[0]))
 }

storage/driver/driver_test.go

@@ -6,5 +6,6 @@ func TestDriver(t *testing.T) {
 	var driver Driver
 	driver = &Mongo{}
 	driver = &BoltDB{}
+	driver = &Storage{}
 	t.Log(driver)
 }

storage/driver/storage.go

@@ -0,0 +1,135 @@
+package driver
+
+import (
+	"context"
+	"errors"
+	"local/dndex/storage/entity"
+	"local/storage"
+	"log"
+
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+type Storage struct {
+	db storage.DB
+}
+
+func NewStorage(args ...string) *Storage {
+	if len(args) == 0 {
+		args = []string{"map"}
+	}
+	typed := storage.TypeFromString(args[0])
+	args = args[1:]
+	db, err := storage.New(typed, args...)
+	if err != nil {
+		panic(err)
+	}
+	return &Storage{
+		db: db,
+	}
+}
+
+func (s *Storage) Count(ctx context.Context, ns string, filter interface{}) (int, error) {
+	ch, err := s.Find(ctx, ns, filter)
+	n := 0
+	for _ = range ch {
+		n++
+	}
+	return n, err
+}
+
+func (s *Storage) Find(ctx context.Context, ns string, filter interface{}) (chan bson.Raw, error) {
+	ch := make(chan bson.Raw)
+	go func() {
+		defer close(ch)
+		if err := s.forEach(ctx, ns, filter, func(_ string, v []byte) error {
+			ch <- v
+			return nil
+		}); err != nil {
+			log.Println(err)
+		}
+	}()
+	return ch, nil
+}
+
+func (s *Storage) Update(ctx context.Context, ns string, filter, operator interface{}) error {
+	return s.forEach(ctx, ns, filter, func(id string, v []byte) error {
+		n := bson.M{}
+		if err := bson.Unmarshal(v, &n); err != nil {
+			return err
+		}
+		n, err := apply(n, operator)
+		if err != nil {
+			return err
+		}
+		v, err = bson.Marshal(n)
+		if err != nil {
+			return err
+		}
+		return s.db.Set(id, v, ns)
+	})
+}
+
+func (s *Storage) Insert(ctx context.Context, ns string, doc interface{}) error {
+	b, err := bson.Marshal(doc)
+	if err != nil {
+		return err
+	}
+	m := bson.M{}
+	if err := bson.Unmarshal(b, &m); err != nil {
+		return err
+	}
+
+	idi, ok := m[entity.ID]
+	if !ok {
+		return errors.New("primary key required to insert: did not find " + entity.ID)
+	}
+	id, ok := idi.(string)
+	if !ok {
+		return errors.New("primary key must be a string")
+	}
+
+	if _, err := s.db.Get(id, ns); err == nil {
+		return errors.New("collision")
+	}
+	return s.db.Set(id, b, ns)
+}
+
+func (s *Storage) Delete(ctx context.Context, ns string, filter interface{}) error {
+	return s.forEach(ctx, ns, filter, func(id string, v []byte) error {
+		return s.db.Set(id, nil, ns)
+	})
+}
+
+func (s *Storage) forEach(ctx context.Context, ns string, filter interface{}, foo func(string, []byte) error) error {
+	b, err := bson.Marshal(filter)
+	if err != nil {
+		return err
+	}
+	m := bson.M{}
+	if err := bson.Unmarshal(b, &m); err != nil {
+		return err
+	}
+	ids, err := s.db.List([]string{ns})
+	if err != nil {
+		return err
+	}
+
+	for _, id := range ids {
+		v, err := s.db.Get(id, ns)
+		if err != nil {
+			return err
+		} else {
+			n := bson.M{}
+			if err := bson.Unmarshal(v, &n); err != nil {
+				return err
+			}
+			if matches(n, m) {
+				if err := foo(id, append(bson.Raw{}, bson.Raw(v)...)); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}

storage/driver/storage_test.go

@@ -0,0 +1,48 @@
+package driver
+
+import (
+	"local/dndex/storage/entity"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+func TestNewStorage(t *testing.T) {
+	tempStorage(t)
+}
+
+func tempStorage(t *testing.T) *Storage {
+	s := NewStorage()
+	fillStorage(t, s)
+	return s
+}
+
+func fillStorage(t *testing.T, s *Storage) {
+	for i := 0; i < testN; i++ {
+		p := entity.One{
+			ID:    "iddd-" + uuid.New().String()[:5],
+			Name:  "name-" + uuid.New().String()[:5],
+			Type:  "type-" + uuid.New().String()[:5],
+			Title: "titl-" + uuid.New().String()[:5],
+		}
+		o := entity.One{
+			ID:          "iddd-" + uuid.New().String()[:5],
+			Name:        "name-" + uuid.New().String()[:5],
+			Type:        "type-" + uuid.New().String()[:5],
+			Title:       "titl-" + uuid.New().String()[:5],
+			Text:        "text-" + uuid.New().String()[:5],
+			Modified:    time.Now().UnixNano(),
+			Connections: map[string]entity.Connection{p.ID: entity.Connection{p.Name}},
+			Attachments: map[string]entity.Attachment{"filename": {"/path/to/file"}},
+		}
+		b, err := bson.Marshal(o)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := s.db.Set(o.ID, b, testNS); err != nil {
+			t.Fatal(err)
+		}
+	}
+}