local/dndex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Storage to uuids

Browse Source
This commit is contained in:
breel
2020-08-07 16:15:52 -06:00
parent 953d2d1365
commit 304956da74
17 changed files with 862 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
storage/driver/boltdb.go
View File

@@ -289,7 +289,7 @@ func applyUnset(doc, operator bson.M) (bson.M, error) {
ok = pmok
}
if !ok {
return nil, fmt.Errorf("subpath cannot be followed for non object: %s (%s): %+v (%T)", k, nesting[0], mInterface, mInterface)
return nil, fmt.Errorf("subpath of %v (%v) cannot be followed for non object: %s (%s): %+v (%T)", doc, doc[nesting[0]], k, nesting[0], mInterface, mInterface)
}
subdoc, err := applyUnset(bson.M(m), bson.M{strings.Join(nesting[1:], "."): ""})
if err != nil {
@@ -322,7 +322,7 @@ func applySet(doc, operator bson.M) (bson.M, error) {
ok = pmok
}
if !ok {
return nil, fmt.Errorf("subpath cannot be followed for non object: %s (%s): %+v (%T)", k, nesting[0], mInterface, mInterface)
return nil, fmt.Errorf("subpath of %v (%v) cannot be followed for non object: %s (%s): %+v (%T)", doc, doc[nesting[0]], k, nesting[0], mInterface, mInterface)
}
subdoc, err := applySet(bson.M(m), bson.M{strings.Join(nesting[1:], "."): v})
if err != nil {

34
storage/driver/boltdb_test.go
View File

@@ -143,9 +143,6 @@ func TestBoltDBFind(t *testing.T) {
if o.Text == "" {
t.Error(o.Text)
}
if o.Relationship != "" {
t.Error(o.Relationship)
}
if o.Modified == 0 {
t.Error(o.Modified)
}
@@ -156,18 +153,9 @@ func TestBoltDBFind(t *testing.T) {
t.Error(o.Connections)
}
for k := range o.Connections {
if o.Connections[k].Name == "" {
t.Error(o.Connections[k])
}
if o.Connections[k].Title == "" {
t.Error(o.Connections[k])
}
if o.Connections[k].Relationship == "" {
t.Error(o.Connections[k])
}
if o.Connections[k].Type == "" {
t.Error(o.Connections[k])
}
}
}
if n != testN {
@@ -317,13 +305,6 @@ func TestBoltDBInsert(t *testing.T) {
if _, ok := ones[0].Connections[k]; !ok {
t.Fatalf("db had more connections than real: %s", k)
}
c := o.Connections[k]
c.Modified = 0
o.Connections[k] = c
c = ones[0].Connections[k]
c.Modified = 0
ones[0].Connections[k] = c
}
o.Modified = 0
ones[0].Modified = 0
@@ -372,7 +353,7 @@ func TestBoltDBDelete(t *testing.T) {
t.Fatal(err)
}
if n != wantN {
t.Error(n, filter)
t.Error(wantN, n, filter)
}
}
})
@@ -401,11 +382,10 @@ func fillBoltDB(t *testing.T, bdb *BoltDB) {
}
for i := 0; i < testN; i++ {
p := entity.One{
ID: "iddd-" + uuid.New().String()[:5],
Name: "name-" + uuid.New().String()[:5],
Type: "type-" + uuid.New().String()[:5],
Relationship: "rshp-" + uuid.New().String()[:5],
Title: "titl-" + uuid.New().String()[:5],
ID: "iddd-" + uuid.New().String()[:5],
Name: "name-" + uuid.New().String()[:5],
Type: "type-" + uuid.New().String()[:5],
Title: "titl-" + uuid.New().String()[:5],
}
o := entity.One{
ID: "iddd-" + uuid.New().String()[:5],
@@ -414,8 +394,8 @@ func fillBoltDB(t *testing.T, bdb *BoltDB) {
Title: "titl-" + uuid.New().String()[:5],
Text: "text-" + uuid.New().String()[:5],
Modified: time.Now().UnixNano(),
Connections: map[string]entity.One{p.ID: p},
Attachments: map[string]string{"filename": "/path/to/file"},
Connections: map[string]entity.Connection{p.ID: entity.Connection{p.Name}},
Attachments: map[string]entity.Attachment{"filename": {"/path/to/file"}},
}
b, err := bson.Marshal(o)
if err != nil {

13
storage/driver/map_test.go
View File

@@ -14,11 +14,10 @@ func tempMap(t *testing.T) *Map {
mp.db[testNS] = map[string][]byte{}
for i := 0; i < testN; i++ {
p := entity.One{
ID: "iddd-" + uuid.New().String()[:5],
Name: "name-" + uuid.New().String()[:5],
Type: "type-" + uuid.New().String()[:5],
Relationship: "rshp-" + uuid.New().String()[:5],
Title: "titl-" + uuid.New().String()[:5],
ID: "iddd-" + uuid.New().String()[:5],
Name: "name-" + uuid.New().String()[:5],
Type: "type-" + uuid.New().String()[:5],
Title: "titl-" + uuid.New().String()[:5],
}
o := entity.One{
ID: "iddd-" + uuid.New().String()[:5],
@@ -27,8 +26,8 @@ func tempMap(t *testing.T) *Map {
Title: "titl-" + uuid.New().String()[:5],
Text: "text-" + uuid.New().String()[:5],
Modified: time.Now().UnixNano(),
Connections: map[string]entity.One{p.ID: p},
Attachments: map[string]string{"filename": "/path/to/file"},
Connections: map[string]entity.Connection{p.ID: entity.Connection{p.Name}},
Attachments: map[string]entity.Attachment{"filename": {"/path/to/file"}},
}
b, err := bson.Marshal(o)
if err != nil {

79
storage/entity/one.go
View File

@@ -19,42 +19,40 @@ const (
Modified = "modified"
Connections = "connections"
Attachments = "attachments"
Location = "location"
)
type One struct {
ID string `bson:"_id,omitempty" json:"_id,omitempty"`
Name string `bson:"name,omitempty" json:"name,omitempty"`
Type string `bson:"type,omitempty" json:"type,omitempty"`
Title string `bson:"title,omitempty" json:"title,omitempty"`
Text string `bson:"text,omitempty" json:"text,omitempty"`
Relationship string `bson:"relationship,omitempty" json:"relationship,omitempty"`
Modified int64 `bson:"modified,omitempty" json:"modified,omitempty"`
Connections map[string]One `bson:"connections" json:"connections,omitempty"`
Attachments map[string]string `bson:"attachments" json:"attachments,omitempty"`
ID string `bson:"_id,omitempty" json:"_id"`
Name string `bson:"name,omitempty" json:"name"`
Type string `bson:"type,omitempty" json:"type"`
Title string `bson:"title,omitempty" json:"title"`
Text string `bson:"text,omitempty" json:"text"`
Modified int64 `bson:"modified,omitempty" json:"modified"`
Connections map[string]Connection `bson:"connections" json:"connections"`
Attachments map[string]Attachment `bson:"attachments" json:"attachments"`
}
func (o One) Query() One {
return One{Name: o.Name}
type Connection struct {
Relationship string `bson:"relationship,omitempty" json:"relationship"`
}
func (o One) Peer() One {
return One{
Name: o.Name,
Type: o.Type,
Title: o.Title,
Relationship: o.Relationship,
Modified: o.Modified,
}
type Attachment struct {
Location string `bson:"location,omitempty" json:"location"`
}
func (o One) Query() bson.M {
return bson.M{ID: o.ID}
}
func (o One) Peers() []string {
names := make([]string, len(o.Connections))
ids := make([]string, len(o.Connections))
i := 0
for k := range o.Connections {
names[i] = o.Connections[k].Name
ids[i] = k
i += 1
}
return names
return ids
}
func (o One) MarshalBSON() ([]byte, error) {
@@ -62,6 +60,12 @@ func (o One) MarshalBSON() ([]byte, error) {
if !isMin {
o.Modified = time.Now().UnixNano()
}
if o.Connections == nil {
o.Connections = make(map[string]Connection)
}
if o.Attachments == nil {
o.Attachments = make(map[string]Attachment)
}
b, err := json.Marshal(o)
if err != nil {
return nil, err
@@ -76,36 +80,5 @@ func (o One) MarshalBSON() ([]byte, error) {
m[k] = strings.TrimSpace(v.(string))
}
}
if !isMin {
connections := map[string]interface{}{}
switch m[Connections].(type) {
case nil:
case map[string]interface{}:
connections = m[Connections].(map[string]interface{})
default:
return nil, fmt.Errorf("bad connections type %T", m[Connections])
}
delete(connections, "")
for k := range connections {
if k == "" {
continue
}
if o.Connections[k].Name == "" {
p := o.Connections[k]
p.Name = k
o.Connections[k] = p
}
b, err := bson.Marshal(o.Connections[k])
if err != nil {
return nil, err
}
m := bson.M{}
if err := bson.Unmarshal(b, &m); err != nil {
return nil, err
}
connections[k] = m
}
m[Connections] = connections
}
return bson.Marshal(m)
}

92
storage/entity/one_test.go
View File

@@ -9,11 +9,11 @@ import (
func TestOne(t *testing.T) {
one := One{
Name: "myname",
ID: "myname",
Type: "mytype",
}
q := one.Query()
if want := fmt.Sprint(One{Name: one.Name}); want != fmt.Sprint(q) {
if want := fmt.Sprint(bson.M{ID: one.ID}); want != fmt.Sprint(q) {
t.Error(want, q)
}
}
@@ -21,7 +21,7 @@ func TestOne(t *testing.T) {
func TestOneMarshalBSON(t *testing.T) {
cases := map[string]struct {
sameAsQuery bool
one One
one interface{}
}{
"query no modified change": {
sameAsQuery: true,
@@ -31,18 +31,16 @@ func TestOneMarshalBSON(t *testing.T) {
one: One{Name: "hello", Type: "world", Modified: 1},
},
"w/ connections": {
one: One{Name: "hello", Type: "world", Modified: 1, Connections: map[string]One{"hi": One{Name: "hi", Relationship: "mom"}}},
one: One{Name: "hello", Type: "world", Modified: 1, Connections: map[string]Connection{"hi": Connection{Relationship: "mom"}}},
},
"w/ attachments": {
one: One{Name: "hello", Type: "world", Modified: 1, Attachments: map[string]string{"hello": "/world"}},
one: One{Name: "hello", Type: "world", Modified: 1, Attachments: map[string]Attachment{"hello": Attachment{"/world"}}},
},
}
for name, d := range cases {
c := d
t.Run(name, func(t *testing.T) {
var bm bson.Marshaler = c.one
t.Log(bm)
b, err := bson.Marshal(c.one)
if err != nil {
t.Fatal(err)
@@ -51,85 +49,11 @@ func TestOneMarshalBSON(t *testing.T) {
if err := bson.Unmarshal(b, &one); err != nil {
t.Fatal(err)
}
if c.sameAsQuery && (fmt.Sprint(one) != fmt.Sprint(one.Query()) || fmt.Sprint(one) != fmt.Sprint(c.one)) {
t.Error(c.sameAsQuery, c.one, one)
} else if !c.sameAsQuery {
if c.one.Modified == one.Modified {
t.Error(c.one.Modified, one.Modified)
}
c.one.Modified = 0
one.Modified = 0
for k := range one.Connections {
temp := one.Connections[k]
temp.Modified = 0
one.Connections[k] = temp
}
if fmt.Sprint(c.one) != fmt.Sprint(one) {
t.Error(c.one, one)
if !c.sameAsQuery {
if one.Modified < 2 {
t.Error(one.Modified)
}
}
})
}
}
func TestOneMarshalBSONBadConnections(t *testing.T) {
t.Run("connections has an empty string for a key that should die", func(t *testing.T) {
input := One{Name: "hello", Connections: map[string]One{"": One{Name: "teehee"}}}
b, err := bson.Marshal(input)
if err != nil {
t.Fatal(err)
}
output := One{}
if err := bson.Unmarshal(b, &output); err != nil {
t.Fatal(err)
}
if len(output.Connections) != 0 {
t.Fatal(output.Connections)
}
input.Connections = nil
output.Connections = nil
input.Modified = 0
output.Modified = 0
if fmt.Sprint(input) != fmt.Sprint(output) {
t.Fatal(input, output)
}
})
t.Run("connections has a key but empty name that should correct", func(t *testing.T) {
input := One{Name: "hello", Connections: map[string]One{"teehee": One{Name: ""}}}
b, err := bson.Marshal(input)
if err != nil {
t.Fatal(err)
}
output := One{}
if err := bson.Unmarshal(b, &output); err != nil {
t.Fatal(err)
}
if len(output.Connections) != 1 {
t.Fatal(output.Connections)
} else {
for k := range output.Connections {
if k != output.Connections[k].Name {
t.Fatal(k, output.Connections)
}
}
}
input.Connections = nil
output.Connections = nil
input.Modified = 0
output.Modified = 0
if fmt.Sprint(input) != fmt.Sprint(output) {
t.Fatal(input, output)
}
})
}

16
storage/graph.go
View File

@@ -31,6 +31,20 @@ func (g Graph) ListCaseInsensitive(ctx context.Context, namespace string, from .
return g.find(ctx, namespace, filter)
}
func (g Graph) Get(ctx context.Context, namespace, id string) (entity.One, error) {
ones, err := g.find(ctx, namespace, bson.M{entity.ID: id})
if err != nil {
return entity.One{}, err
}
if len(ones) == 0 {
return entity.One{}, errors.New("not found")
}
if len(ones) > 1 {
return entity.One{}, errors.New("primary key collision detected")
}
return ones[0], nil
}
func (g Graph) List(ctx context.Context, namespace string, from ...string) ([]entity.One, error) {
filter := operator.NewFilterIn(entity.Name, from)
return g.find(ctx, namespace, filter)
@@ -68,7 +82,7 @@ func (g Graph) Insert(ctx context.Context, namespace string, one entity.One) err
return g.driver.Insert(ctx, namespace, one)
}
func (g Graph) Update(ctx context.Context, namespace string, one entity.One, modify interface{}) error {
func (g Graph) Update(ctx context.Context, namespace string, one, modify interface{}) error {
return g.driver.Update(ctx, namespace, one, modify)
}

46
storage/graph_test.go
View File

@@ -29,7 +29,7 @@ func TestIntegration(t *testing.T) {
randomOne(),
randomOne(),
}
ones[0].Connections = map[string]entity.One{ones[2].Name: entity.One{Name: ones[2].Name, Relationship: ":("}}
ones[0].Connections = map[string]entity.Connection{ones[2].Name: entity.Connection{Relationship: ":("}}
ones[1].Name = ones[0].Name[1 : len(ones[0].Name)-1]
cleanFill := func() {
clean()
@@ -62,6 +62,30 @@ func TestIntegration(t *testing.T) {
}
})
t.Run("graph.Get 404", func(t *testing.T) {
cleanFill()
_, err := graph.Get(ctx, "col", "fake_here")
if err == nil {
t.Fatal(err)
}
})
t.Run("graph.Get", func(t *testing.T) {
cleanFill()
all, err := graph.List(ctx, "col")
if err != nil {
t.Fatal(err)
}
want := all[0]
got, err := graph.Get(ctx, "col", want.ID)
if err != nil {
t.Fatal(err)
}
if got.ID != want.ID {
t.Fatal(got)
}
})
t.Run("graph.ListCaseInsensitive", func(t *testing.T) {
cleanFill()
all, err := graph.ListCaseInsensitive(ctx, "col")
@@ -201,9 +225,9 @@ func TestIntegration(t *testing.T) {
t.Run("graph.Update(foo, +=2); graph.Update(foo, -=1)", func(t *testing.T) {
cleanFill()
err := graph.Update(ctx, "col", ones[0].Query(), operator.Set{entity.Connections, map[string]entity.One{
"hello": entity.One{Name: "hello", Relationship: ":("},
"world": entity.One{Name: "world", Relationship: ":("},
err := graph.Update(ctx, "col", ones[0].Query(), operator.Set{entity.Connections, map[string]entity.Connection{
"hello": entity.Connection{Relationship: ":("},
"world": entity.Connection{Relationship: ":("},
}})
if err != nil {
t.Fatal(err)
@@ -240,7 +264,7 @@ func TestIntegration(t *testing.T) {
t.Run("graph.Update(new attachment), Update(--new attachment)", func(t *testing.T) {
cleanFill()
err := graph.Update(ctx, "col", ones[0].Query(), operator.Set{Key: fmt.Sprintf("%s.new attachment", entity.Attachments), Value: "my new attachment"})
err := graph.Update(ctx, "col", ones[0].Query(), operator.Set{Key: fmt.Sprintf("%s.new attachment", entity.Attachments), Value: entity.Attachment{Location: "my new attachment"}})
if err != nil {
t.Fatal(err)
}
@@ -255,8 +279,8 @@ func TestIntegration(t *testing.T) {
}
if v, ok := some1[0].Attachments["new attachment"]; !ok {
t.Fatal(ok, some1[0].Attachments)
} else if v != "my new attachment" {
t.Fatal(v, some1[0].Attachments)
} else if v.Location != "my new attachment" {
t.Fatalf("when listing from DB, did not find updated attachment: got %+v from %+v", v, some1[0].Attachments)
}
err = graph.Update(ctx, "col", ones[0].Query(), operator.Unset(fmt.Sprintf("%s.new attachment", entity.Attachments)))
@@ -372,10 +396,10 @@ func randomOne() entity.One {
Title: "Biggus",
Text: "tee hee xd",
Modified: time.Now().UnixNano(),
Connections: map[string]entity.One{},
Attachments: map[string]string{
"pdf file": "/path/to.pdf",
"png file": "/path/to.png",
Connections: map[string]entity.Connection{},
Attachments: map[string]entity.Attachment{
"pdf file": entity.Attachment{Location: "/path/to.pdf"},
"png file": entity.Attachment{Location: "/path/to.png"},
},
}
}

6
storage/ratelimitedgraph.go
View File

@@ -46,6 +46,10 @@ func (rlg RateLimitedGraph) Insert(ctx context.Context, namespace string, one en
return rlg.g.Insert(ctx, namespace, one)
}
func (rlg RateLimitedGraph) Get(ctx context.Context, namespace, id string) (entity.One, error) {
return rlg.g.Get(ctx, namespace, id)
}
func (rlg RateLimitedGraph) List(ctx context.Context, namespace string, from ...string) ([]entity.One, error) {
return rlg.g.List(ctx, namespace, from...)
}
@@ -58,6 +62,6 @@ func (rlg RateLimitedGraph) Search(ctx context.Context, namespace string, nameCo
return rlg.g.Search(ctx, namespace, nameContains)
}
func (rlg RateLimitedGraph) Update(ctx context.Context, namespace string, one entity.One, modify interface{}) error {
func (rlg RateLimitedGraph) Update(ctx context.Context, namespace string, one, modify interface{}) error {
return rlg.g.Update(ctx, namespace, one, modify)
}