local
/
dndex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change config to use local/storage

master
breel 2020-08-27 12:32:33 -06:00
parent bb9b91eef5
commit 3a99bf5e33
7 changed files with 201 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
config/config.go
View File

@ -4,14 +4,14 @@ import (
"io/ioutil"
"local/args"
"os"
"strings"
"time"
)
type Config struct {
Port int
DBURI string
Database string
DriverType string
Driver []string
FilePrefix string
FileRoot string
Auth bool
@ -32,11 +32,10 @@ func New() Config {
f.Close()
as.Append(args.INT, "p", "port to listen on", 18114)
as.Append(args.STRING, "dburi", "database uri", f.Name())
as.Append(args.STRING, "fileprefix", "path prefix for file service", "/__files__")
as.Append(args.STRING, "fileroot", "path to file hosting root", "/tmp/")
as.Append(args.STRING, "database", "database name to use", "db")
as.Append(args.STRING, "driver-type", "database driver to use, [boltdb mongo map]", "map")
as.Append(args.STRING, "driver", "database driver args to use, like [local/storage.Type,arg1,arg2...] or [/path/to/boltdb]", "map")
as.Append(args.BOOL, "auth", "check for authorized access", false)
as.Append(args.DURATION, "authlifetime", "duration auth is valid for", time.Hour)
as.Append(args.DURATION, "delay", "time to delay requests", time.Duration(0))
@ -51,11 +50,10 @@ func New() Config {
return Config{
Port: as.GetInt("p"),
DBURI: as.GetString("dburi"),
FilePrefix: as.GetString("fileprefix"),
FileRoot: as.GetString("fileroot"),
Database: as.GetString("database"),
DriverType: as.GetString("driver-type"),
Driver: strings.Split(as.GetString("driver"), ","),
Auth: as.GetBool("auth"),
AuthLifetime: as.GetDuration("authlifetime"),
Delay: as.GetDuration("delay"),

2
main_test.go
View File

@ -27,7 +27,7 @@ func Test(t *testing.T) {
s := httptest.NewServer(http.HandlerFunc(nil))
s.Close()
p := strings.Split(s.URL, ":")[2]
os.Args = strings.Split(fmt.Sprintf(`dndex -auth=true -database db -delay 5ms -driver-type map -fileprefix /files -fileroot %s -p %v -rps 50 -sys-rps 40`, d, p), " ")
os.Args = strings.Split(fmt.Sprintf(`dndex -auth=true -database db -delay 5ms -driver map -fileprefix /files -fileroot %s -p %v -rps 50 -sys-rps 40`, d, p), " ")
go main()

10
storage/driver/boltdb_test.go
View File

@ -30,7 +30,7 @@ func TestBoltDBCount(t *testing.T) {
bdb, can := tempBoltDB(t)
defer can()
for _, driver := range []Driver{bdb, tempMap(t)} {
for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} {
t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) {
ch, err := driver.Find(context.TODO(), testNS, map[string]string{})
if err != nil {
@ -121,7 +121,7 @@ func TestBoltDBFind(t *testing.T) {
bdb, can := tempBoltDB(t)
defer can()
for _, driver := range []Driver{bdb, tempMap(t)} {
for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} {
t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) {
ch, err := driver.Find(context.TODO(), testNS, map[string]string{})
if err != nil {
@ -169,7 +169,7 @@ func TestBoltDBUpdate(t *testing.T) {
bdb, can := tempBoltDB(t)
defer can()
for _, driver := range []Driver{bdb, tempMap(t)} {
for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} {
t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) {
ch, err := driver.Find(context.TODO(), testNS, map[string]string{})
if err != nil {
@ -250,7 +250,7 @@ func TestBoltDBInsert(t *testing.T) {
bdb, can := tempBoltDB(t)
defer can()
for _, driver := range []Driver{bdb, tempMap(t)} {
for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} {
t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) {
ch, err := driver.Find(context.TODO(), testNS, map[string]string{})
if err != nil {
@ -320,7 +320,7 @@ func TestBoltDBDelete(t *testing.T) {
bdb, can := tempBoltDB(t)
defer can()
for _, driver := range []Driver{bdb, tempMap(t)} {
for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} {
t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) {
ch, err := driver.Find(context.TODO(), testNS, map[string]string{})
if err != nil {

10
storage/driver/driver.go
View File

@ -3,6 +3,7 @@ package driver
import (
"context"
"local/dndex/config"
"local/storage"
"strings"
"go.mongodb.org/mongo-driver/bson"
@ -18,9 +19,12 @@ type Driver interface {
func New(path ...string) Driver {
if len(path) == 0 {
path = []string{config.New().DBURI}
path = config.New().Driver
}
switch strings.ToLower(config.New().DriverType) {
if t := storage.TypeFromString(path[0]); t >= 0 {
return NewStorage(path...)
}
switch strings.ToLower(config.New().Driver[0]) {
case "map":
return NewMap()
case "mongo":
@ -28,5 +32,5 @@ func New(path ...string) Driver {
case "boltdb":
return NewBoltDB(path[0])
}
panic("unknown driver type " + strings.ToLower(config.New().DriverType))
panic("unknown driver type " + strings.ToLower(config.New().Driver[0]))
}

1
storage/driver/driver_test.go
View File

@ -6,5 +6,6 @@ func TestDriver(t *testing.T) {
var driver Driver
driver = &Mongo{}
driver = &BoltDB{}
driver = &Storage{}
t.Log(driver)
}

135
storage/driver/storage.go Normal file
View File

@ -0,0 +1,135 @@
package driver
import (
"context"
"errors"
"local/dndex/storage/entity"
"local/storage"
"log"
"go.mongodb.org/mongo-driver/bson"
)
type Storage struct {
db storage.DB
}
func NewStorage(args ...string) *Storage {
if len(args) == 0 {
args = []string{"map"}
}
typed := storage.TypeFromString(args[0])
args = args[1:]
db, err := storage.New(typed, args...)
if err != nil {
panic(err)
}
return &Storage{
db: db,
}
}
func (s *Storage) Count(ctx context.Context, ns string, filter interface{}) (int, error) {
ch, err := s.Find(ctx, ns, filter)
n := 0
for _ = range ch {
n++
}
return n, err
}
func (s *Storage) Find(ctx context.Context, ns string, filter interface{}) (chan bson.Raw, error) {
ch := make(chan bson.Raw)
go func() {
defer close(ch)
if err := s.forEach(ctx, ns, filter, func(_ string, v []byte) error {
ch <- v
return nil
}); err != nil {
log.Println(err)
}
}()
return ch, nil
}
func (s *Storage) Update(ctx context.Context, ns string, filter, operator interface{}) error {
return s.forEach(ctx, ns, filter, func(id string, v []byte) error {
n := bson.M{}
if err := bson.Unmarshal(v, &n); err != nil {
return err
}
n, err := apply(n, operator)
if err != nil {
return err
}
v, err = bson.Marshal(n)
if err != nil {
return err
}
return s.db.Set(id, v, ns)
})
}
func (s *Storage) Insert(ctx context.Context, ns string, doc interface{}) error {
b, err := bson.Marshal(doc)
if err != nil {
return err
}
m := bson.M{}
if err := bson.Unmarshal(b, &m); err != nil {
return err
}
idi, ok := m[entity.ID]
if !ok {
return errors.New("primary key required to insert: did not find " + entity.ID)
}
id, ok := idi.(string)
if !ok {
return errors.New("primary key must be a string")
}
if _, err := s.db.Get(id, ns); err == nil {
return errors.New("collision")
}
return s.db.Set(id, b, ns)
}
func (s *Storage) Delete(ctx context.Context, ns string, filter interface{}) error {
return s.forEach(ctx, ns, filter, func(id string, v []byte) error {
return s.db.Set(id, nil, ns)
})
}
func (s *Storage) forEach(ctx context.Context, ns string, filter interface{}, foo func(string, []byte) error) error {
b, err := bson.Marshal(filter)
if err != nil {
return err
}
m := bson.M{}
if err := bson.Unmarshal(b, &m); err != nil {
return err
}
ids, err := s.db.List([]string{ns})
if err != nil {
return err
}
for _, id := range ids {
v, err := s.db.Get(id, ns)
if err != nil {
return err
} else {
n := bson.M{}
if err := bson.Unmarshal(v, &n); err != nil {
return err
}
if matches(n, m) {
if err := foo(id, append(bson.Raw{}, bson.Raw(v)...)); err != nil {
return err
}
}
}
}
return nil
}

48
storage/driver/storage_test.go Normal file
View File

@ -0,0 +1,48 @@
package driver
import (
"local/dndex/storage/entity"
"testing"
"time"
"github.com/google/uuid"
"go.mongodb.org/mongo-driver/bson"
)
func TestNewStorage(t *testing.T) {
tempStorage(t)
}
func tempStorage(t *testing.T) *Storage {
s := NewStorage()
fillStorage(t, s)
return s
}
func fillStorage(t *testing.T, s *Storage) {
for i := 0; i < testN; i++ {
p := entity.One{
ID: "iddd-" + uuid.New().String()[:5],
Name: "name-" + uuid.New().String()[:5],
Type: "type-" + uuid.New().String()[:5],
Title: "titl-" + uuid.New().String()[:5],
}
o := entity.One{
ID: "iddd-" + uuid.New().String()[:5],
Name: "name-" + uuid.New().String()[:5],
Type: "type-" + uuid.New().String()[:5],
Title: "titl-" + uuid.New().String()[:5],
Text: "text-" + uuid.New().String()[:5],
Modified: time.Now().UnixNano(),
Connections: map[string]entity.Connection{p.ID: entity.Connection{p.Name}},
Attachments: map[string]entity.Attachment{"filename": {"/path/to/file"}},
}
b, err := bson.Marshal(o)
if err != nil {
t.Fatal(err)
}
if err := s.db.Set(o.ID, b, testNS); err != nil {
t.Fatal(err)
}
}
}