From 3a99bf5e33ca4d14f1d04333e94f583faf4c7fb9 Mon Sep 17 00:00:00 2001 From: breel Date: Thu, 27 Aug 2020 12:32:33 -0600 Subject: [PATCH] Change config to use local/storage --- config/config.go | 10 +-- main_test.go | 2 +- storage/driver/boltdb_test.go | 10 +-- storage/driver/driver.go | 10 ++- storage/driver/driver_test.go | 1 + storage/driver/storage.go | 135 +++++++++++++++++++++++++++++++++ storage/driver/storage_test.go | 48 ++++++++++++ 7 files changed, 201 insertions(+), 15 deletions(-) create mode 100644 storage/driver/storage.go create mode 100644 storage/driver/storage_test.go diff --git a/config/config.go b/config/config.go index b7450c2..1345055 100644 --- a/config/config.go +++ b/config/config.go @@ -4,14 +4,14 @@ import ( "io/ioutil" "local/args" "os" + "strings" "time" ) type Config struct { Port int - DBURI string Database string - DriverType string + Driver []string FilePrefix string FileRoot string Auth bool @@ -32,11 +32,10 @@ func New() Config { f.Close() as.Append(args.INT, "p", "port to listen on", 18114) - as.Append(args.STRING, "dburi", "database uri", f.Name()) as.Append(args.STRING, "fileprefix", "path prefix for file service", "/__files__") as.Append(args.STRING, "fileroot", "path to file hosting root", "/tmp/") as.Append(args.STRING, "database", "database name to use", "db") - as.Append(args.STRING, "driver-type", "database driver to use, [boltdb mongo map]", "map") + as.Append(args.STRING, "driver", "database driver args to use, like [local/storage.Type,arg1,arg2...] or [/path/to/boltdb]", "map") as.Append(args.BOOL, "auth", "check for authorized access", false) as.Append(args.DURATION, "authlifetime", "duration auth is valid for", time.Hour) as.Append(args.DURATION, "delay", "time to delay requests", time.Duration(0)) @@ -51,11 +50,10 @@ func New() Config { return Config{ Port: as.GetInt("p"), - DBURI: as.GetString("dburi"), FilePrefix: as.GetString("fileprefix"), FileRoot: as.GetString("fileroot"), Database: as.GetString("database"), - DriverType: as.GetString("driver-type"), + Driver: strings.Split(as.GetString("driver"), ","), Auth: as.GetBool("auth"), AuthLifetime: as.GetDuration("authlifetime"), Delay: as.GetDuration("delay"), diff --git a/main_test.go b/main_test.go index e483660..4c13559 100644 --- a/main_test.go +++ b/main_test.go @@ -27,7 +27,7 @@ func Test(t *testing.T) { s := httptest.NewServer(http.HandlerFunc(nil)) s.Close() p := strings.Split(s.URL, ":")[2] - os.Args = strings.Split(fmt.Sprintf(`dndex -auth=true -database db -delay 5ms -driver-type map -fileprefix /files -fileroot %s -p %v -rps 50 -sys-rps 40`, d, p), " ") + os.Args = strings.Split(fmt.Sprintf(`dndex -auth=true -database db -delay 5ms -driver map -fileprefix /files -fileroot %s -p %v -rps 50 -sys-rps 40`, d, p), " ") go main() diff --git a/storage/driver/boltdb_test.go b/storage/driver/boltdb_test.go index bee44eb..2bda956 100644 --- a/storage/driver/boltdb_test.go +++ b/storage/driver/boltdb_test.go @@ -30,7 +30,7 @@ func TestBoltDBCount(t *testing.T) { bdb, can := tempBoltDB(t) defer can() - for _, driver := range []Driver{bdb, tempMap(t)} { + for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} { t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) { ch, err := driver.Find(context.TODO(), testNS, map[string]string{}) if err != nil { @@ -121,7 +121,7 @@ func TestBoltDBFind(t *testing.T) { bdb, can := tempBoltDB(t) defer can() - for _, driver := range []Driver{bdb, tempMap(t)} { + for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} { t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) { ch, err := driver.Find(context.TODO(), testNS, map[string]string{}) if err != nil { @@ -169,7 +169,7 @@ func TestBoltDBUpdate(t *testing.T) { bdb, can := tempBoltDB(t) defer can() - for _, driver := range []Driver{bdb, tempMap(t)} { + for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} { t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) { ch, err := driver.Find(context.TODO(), testNS, map[string]string{}) if err != nil { @@ -250,7 +250,7 @@ func TestBoltDBInsert(t *testing.T) { bdb, can := tempBoltDB(t) defer can() - for _, driver := range []Driver{bdb, tempMap(t)} { + for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} { t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) { ch, err := driver.Find(context.TODO(), testNS, map[string]string{}) if err != nil { @@ -320,7 +320,7 @@ func TestBoltDBDelete(t *testing.T) { bdb, can := tempBoltDB(t) defer can() - for _, driver := range []Driver{bdb, tempMap(t)} { + for _, driver := range []Driver{bdb, tempMap(t), tempStorage(t)} { t.Run(fmt.Sprintf("%T", driver), func(t *testing.T) { ch, err := driver.Find(context.TODO(), testNS, map[string]string{}) if err != nil { diff --git a/storage/driver/driver.go b/storage/driver/driver.go index 71e0527..2d57c8a 100644 --- a/storage/driver/driver.go +++ b/storage/driver/driver.go @@ -3,6 +3,7 @@ package driver import ( "context" "local/dndex/config" + "local/storage" "strings" "go.mongodb.org/mongo-driver/bson" @@ -18,9 +19,12 @@ type Driver interface { func New(path ...string) Driver { if len(path) == 0 { - path = []string{config.New().DBURI} + path = config.New().Driver } - switch strings.ToLower(config.New().DriverType) { + if t := storage.TypeFromString(path[0]); t >= 0 { + return NewStorage(path...) + } + switch strings.ToLower(config.New().Driver[0]) { case "map": return NewMap() case "mongo": @@ -28,5 +32,5 @@ func New(path ...string) Driver { case "boltdb": return NewBoltDB(path[0]) } - panic("unknown driver type " + strings.ToLower(config.New().DriverType)) + panic("unknown driver type " + strings.ToLower(config.New().Driver[0])) } diff --git a/storage/driver/driver_test.go b/storage/driver/driver_test.go index 2f0dfad..1ecb459 100644 --- a/storage/driver/driver_test.go +++ b/storage/driver/driver_test.go @@ -6,5 +6,6 @@ func TestDriver(t *testing.T) { var driver Driver driver = &Mongo{} driver = &BoltDB{} + driver = &Storage{} t.Log(driver) } diff --git a/storage/driver/storage.go b/storage/driver/storage.go new file mode 100644 index 0000000..1399784 --- /dev/null +++ b/storage/driver/storage.go @@ -0,0 +1,135 @@ +package driver + +import ( + "context" + "errors" + "local/dndex/storage/entity" + "local/storage" + "log" + + "go.mongodb.org/mongo-driver/bson" +) + +type Storage struct { + db storage.DB +} + +func NewStorage(args ...string) *Storage { + if len(args) == 0 { + args = []string{"map"} + } + typed := storage.TypeFromString(args[0]) + args = args[1:] + db, err := storage.New(typed, args...) + if err != nil { + panic(err) + } + return &Storage{ + db: db, + } +} + +func (s *Storage) Count(ctx context.Context, ns string, filter interface{}) (int, error) { + ch, err := s.Find(ctx, ns, filter) + n := 0 + for _ = range ch { + n++ + } + return n, err +} + +func (s *Storage) Find(ctx context.Context, ns string, filter interface{}) (chan bson.Raw, error) { + ch := make(chan bson.Raw) + go func() { + defer close(ch) + if err := s.forEach(ctx, ns, filter, func(_ string, v []byte) error { + ch <- v + return nil + }); err != nil { + log.Println(err) + } + }() + return ch, nil +} + +func (s *Storage) Update(ctx context.Context, ns string, filter, operator interface{}) error { + return s.forEach(ctx, ns, filter, func(id string, v []byte) error { + n := bson.M{} + if err := bson.Unmarshal(v, &n); err != nil { + return err + } + n, err := apply(n, operator) + if err != nil { + return err + } + v, err = bson.Marshal(n) + if err != nil { + return err + } + return s.db.Set(id, v, ns) + }) +} + +func (s *Storage) Insert(ctx context.Context, ns string, doc interface{}) error { + b, err := bson.Marshal(doc) + if err != nil { + return err + } + m := bson.M{} + if err := bson.Unmarshal(b, &m); err != nil { + return err + } + + idi, ok := m[entity.ID] + if !ok { + return errors.New("primary key required to insert: did not find " + entity.ID) + } + id, ok := idi.(string) + if !ok { + return errors.New("primary key must be a string") + } + + if _, err := s.db.Get(id, ns); err == nil { + return errors.New("collision") + } + return s.db.Set(id, b, ns) +} + +func (s *Storage) Delete(ctx context.Context, ns string, filter interface{}) error { + return s.forEach(ctx, ns, filter, func(id string, v []byte) error { + return s.db.Set(id, nil, ns) + }) +} + +func (s *Storage) forEach(ctx context.Context, ns string, filter interface{}, foo func(string, []byte) error) error { + b, err := bson.Marshal(filter) + if err != nil { + return err + } + m := bson.M{} + if err := bson.Unmarshal(b, &m); err != nil { + return err + } + ids, err := s.db.List([]string{ns}) + if err != nil { + return err + } + + for _, id := range ids { + v, err := s.db.Get(id, ns) + if err != nil { + return err + } else { + n := bson.M{} + if err := bson.Unmarshal(v, &n); err != nil { + return err + } + if matches(n, m) { + if err := foo(id, append(bson.Raw{}, bson.Raw(v)...)); err != nil { + return err + } + } + } + } + return nil +} diff --git a/storage/driver/storage_test.go b/storage/driver/storage_test.go new file mode 100644 index 0000000..4e0eb8b --- /dev/null +++ b/storage/driver/storage_test.go @@ -0,0 +1,48 @@ +package driver + +import ( + "local/dndex/storage/entity" + "testing" + "time" + + "github.com/google/uuid" + "go.mongodb.org/mongo-driver/bson" +) + +func TestNewStorage(t *testing.T) { + tempStorage(t) +} + +func tempStorage(t *testing.T) *Storage { + s := NewStorage() + fillStorage(t, s) + return s +} + +func fillStorage(t *testing.T, s *Storage) { + for i := 0; i < testN; i++ { + p := entity.One{ + ID: "iddd-" + uuid.New().String()[:5], + Name: "name-" + uuid.New().String()[:5], + Type: "type-" + uuid.New().String()[:5], + Title: "titl-" + uuid.New().String()[:5], + } + o := entity.One{ + ID: "iddd-" + uuid.New().String()[:5], + Name: "name-" + uuid.New().String()[:5], + Type: "type-" + uuid.New().String()[:5], + Title: "titl-" + uuid.New().String()[:5], + Text: "text-" + uuid.New().String()[:5], + Modified: time.Now().UnixNano(), + Connections: map[string]entity.Connection{p.ID: entity.Connection{p.Name}}, + Attachments: map[string]entity.Attachment{"filename": {"/path/to/file"}}, + } + b, err := bson.Marshal(o) + if err != nil { + t.Fatal(err) + } + if err := s.db.Set(o.ID, b, testNS); err != nil { + t.Fatal(err) + } + } +}