Storage to uuids

server/auth.go

@@ -0,0 +1,194 @@
+package server
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"io"
+	"local/dndex/config"
+	"local/dndex/storage"
+	"local/dndex/storage/entity"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+func Auth(g storage.RateLimitedGraph, w http.ResponseWriter, r *http.Request) error {
+	if !config.New().Auth {
+		return nil
+	}
+	if err := auth(g, w, r); err != nil {
+		json.NewEncoder(w).Encode(map[string]interface{}{"error": "error when authorizing: " + err.Error()})
+		return err
+	}
+	return nil
+}
+
+func auth(g storage.RateLimitedGraph, w http.ResponseWriter, r *http.Request) error {
+	if isPublic(g, r) {
+		return nil
+	}
+	if !hasAuth(r) {
+		return requestAuth(g, w, r)
+	}
+	return checkAuth(g, w, r)
+}
+
+func isPublic(g storage.RateLimitedGraph, r *http.Request) bool {
+	namespace, err := getAuthNamespace(r)
+
+	if err != nil {
+		return false
+	}
+	ones, err := g.List(r.Context(), namespace, UserKey)
+	if err != nil {
+		return false
+	}
+	if len(ones) == 0 {
+		return false
+	}
+	return ones[0].Title == ""
+}
+
+func hasAuth(r *http.Request) bool {
+	_, err := r.Cookie(AuthKey)
+	return err == nil
+}
+
+func checkAuth(g storage.RateLimitedGraph, w http.ResponseWriter, r *http.Request) error {
+	namespace, err := getAuthNamespace(r)
+	if err != nil {
+		return err
+	}
+	token, _ := r.Cookie(AuthKey)
+	results, err := g.List(r.Context(), namespace, token.Value)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return requestAuth(g, w, r)
+	}
+	modified := time.Unix(0, results[0].Modified)
+	if time.Since(modified) > config.New().AuthLifetime {
+		return requestAuth(g, w, r)
+	}
+	return nil
+}
+
+func requestAuth(g storage.RateLimitedGraph, w http.ResponseWriter, r *http.Request) error {
+	namespace, err := getAuthNamespace(r)
+	if err != nil {
+		http.Error(w, `{"error": "namespace required"}`, http.StatusBadRequest)
+		return err
+	}
+
+	ones, err := g.List(r.Context(), namespace, UserKey)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return err
+	}
+	if len(ones) != 1 {
+		http.NotFound(w, r)
+		return errors.New("namespace not established")
+	}
+	userKey := ones[0]
+
+	id := uuid.New().String()
+	token := entity.One{
+		ID:    id,
+		Name:  id,
+		Title: namespace,
+	}
+	if err := g.Insert(r.Context(), namespace, token); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return err
+	}
+
+	encodedToken, err := aesEnc(userKey.Title, token.Name)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return err
+	}
+	http.SetCookie(w, &http.Cookie{Name: NewAuthKey, Value: encodedToken})
+
+	http.Redirect(w, r, r.URL.String(), http.StatusSeeOther)
+	return errors.New("auth requested")
+}
+
+func aesEnc(key, payload string) (string, error) {
+	if len(key) == 0 {
+		return "", errors.New("key required")
+	}
+	key = strings.Repeat(key, 32)[:32]
+
+	block, err := aes.NewCipher([]byte(key))
+	if err != nil {
+		return "", err
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return "", err
+	}
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+		return "", err
+	}
+	b := gcm.Seal(nonce, nonce, []byte(payload), nil)
+
+	return base64.StdEncoding.EncodeToString(b), nil
+}
+
+func aesDec(key, payload string) (string, error) {
+	if len(key) == 0 {
+		return "", errors.New("key required")
+	}
+	key = strings.Repeat(key, 32)[:32]
+
+	ciphertext, err := base64.StdEncoding.DecodeString(payload)
+	if err != nil {
+		return "", err
+	}
+
+	block, err := aes.NewCipher([]byte(key))
+	if err != nil {
+		return "", err
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return "", err
+	}
+	if len(ciphertext) < gcm.NonceSize() {
+		return "", errors.New("short ciphertext")
+	}
+	b, err := gcm.Open(nil, ciphertext[:gcm.NonceSize()], ciphertext[gcm.NonceSize():], nil)
+	return string(b), err
+}
+
+func getAuthNamespace(r *http.Request) (string, error) {
+	namespace, err := getNamespace(r)
+	return strings.Join([]string{namespace, AuthKey}, "."), err
+}
+
+func getNamespace(r *http.Request) (string, error) {
+	if strings.HasPrefix(r.URL.Path, config.New().FilePrefix) {
+		path := strings.TrimPrefix(r.URL.Path, config.New().FilePrefix+"/")
+		if path == r.URL.Path {
+			return "", errors.New("no namespace on files")
+		}
+		path = strings.Split(path, "/")[0]
+		if path == "" {
+			return "", errors.New("empty namespace on files")
+		}
+		return path, nil
+	}
+	namespace := r.URL.Query().Get("namespace")
+	if len(namespace) == 0 {
+		return "", errors.New("no namespace found")
+	}
+	return namespace, nil
+}

server/auth_test.go

@@ -0,0 +1,184 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"local/dndex/storage/entity"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+func TestAuth(t *testing.T) {
+	os.Args = os.Args[:1]
+
+	rest, clean := testREST(t)
+	defer clean()
+
+	handler := rest.router
+	g := rest.g
+
+	os.Setenv("AUTH", "true")
+	defer os.Setenv("AUTH", "false")
+
+	if err := g.Insert(context.TODO(), "col."+AuthKey, entity.One{ID: UserKey, Name: UserKey, Title: "password"}); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("auth: no namespace", func(t *testing.T) {
+		r := httptest.NewRequest(http.MethodGet, "/who", nil)
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		if w.Code != http.StatusBadRequest {
+			t.Fatalf("%d: %s", w.Code, w.Body.Bytes())
+		}
+	})
+
+	t.Run("auth: bad provided", func(t *testing.T) {
+		r := httptest.NewRequest(http.MethodGet, "/who?namespace=col", nil)
+		r.Header.Set("Cookie", fmt.Sprintf("%s=not-a-real-token", AuthKey))
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		if w.Code != http.StatusSeeOther {
+			t.Fatalf("%d: %s", w.Code, w.Body.Bytes())
+		}
+	})
+
+	t.Run("auth: expired provided", func(t *testing.T) {
+		os.Setenv("AUTHLIFETIME", "1ms")
+		defer os.Setenv("AUTHLIFETIME", "1h")
+		one := entity.One{ID: uuid.New().String(), Name: uuid.New().String(), Title: "title"}
+		if err := g.Insert(context.TODO(), "col", one); err != nil {
+			t.Fatal(err)
+		}
+		time.Sleep(time.Millisecond * 50)
+		r := httptest.NewRequest(http.MethodGet, "/who?namespace=col", nil)
+		r.Header.Set("Cookie", fmt.Sprintf("%s=%s", AuthKey, one.ID))
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		if w.Code != http.StatusSeeOther {
+			t.Fatalf("%d: %s", w.Code, w.Body.Bytes())
+		}
+	})
+
+	t.Run("auth: none provided: who", func(t *testing.T) {
+		r := httptest.NewRequest(http.MethodGet, "/who?namespace=col", nil)
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		if w.Code != http.StatusSeeOther {
+			t.Fatalf("%d: %s", w.Code, w.Body.Bytes())
+		}
+	})
+
+	t.Run("auth: none provided: files", func(t *testing.T) {
+		r := httptest.NewRequest(http.MethodGet, "/__files__/col/myfile", nil)
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		if w.Code != http.StatusSeeOther {
+			t.Fatalf("%d: %s", w.Code, w.Body.Bytes())
+		}
+	})
+
+	t.Run("auth: provided", func(t *testing.T) {
+		os.Setenv("AUTHLIFETIME", "1h")
+		one := entity.One{ID: uuid.New().String(), Name: uuid.New().String(), Title: "title"}
+		if err := g.Insert(context.TODO(), "col."+AuthKey, one); err != nil {
+			t.Fatal(err)
+		}
+		r := httptest.NewRequest(http.MethodTrace, "/who?namespace=col", nil)
+		r.Header.Set("Cookie", fmt.Sprintf("%s=%s", AuthKey, one.Name))
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		if w.Code != http.StatusOK {
+			t.Fatalf("%d: %s", w.Code, w.Body.Bytes())
+		}
+	})
+
+	t.Run("auth: request unknown namespace", func(t *testing.T) {
+		os.Setenv("AUTHLIFETIME", "1h")
+		r := httptest.NewRequest(http.MethodTrace, "/who?namespace=not-col", nil)
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		if w.Code != http.StatusNotFound {
+			t.Fatalf("%d: %s", w.Code, w.Body.Bytes())
+		}
+	})
+
+	t.Run("auth: request", func(t *testing.T) {
+		os.Setenv("AUTHLIFETIME", "1h")
+		r := httptest.NewRequest(http.MethodTrace, "/who?namespace=col", nil)
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		if w.Code != http.StatusSeeOther {
+			t.Fatalf("%d: %s", w.Code, w.Body.Bytes())
+		}
+
+		rawtoken := getCookie(NewAuthKey, w.Header())
+		if rawtoken == "" {
+			t.Fatal(w.Header())
+		}
+		token, err := aesDec("password", rawtoken)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		r = httptest.NewRequest(http.MethodTrace, "/who?namespace=col", nil)
+		w = httptest.NewRecorder()
+		r.Header.Set("Cookie", fmt.Sprintf("%s=%s", AuthKey, token))
+		handler.ServeHTTP(w, r)
+		if w.Code != http.StatusOK {
+			t.Fatalf("%d: %s", w.Code, w.Body.Bytes())
+		}
+
+		r = httptest.NewRequest(http.MethodTrace, "/__files__/col/myfile", nil)
+		w = httptest.NewRecorder()
+		r.Header.Set("Cookie", fmt.Sprintf("%s=%s", AuthKey, token))
+		handler.ServeHTTP(w, r)
+		if w.Code != http.StatusNotFound {
+			t.Fatalf("%d: %s", w.Code, w.Body.Bytes())
+		}
+	})
+}
+
+func TestAES(t *testing.T) {
+	for _, plaintext := range []string{"", "payload!", "a really long payload here"} {
+		key := "password"
+
+		enc, err := aesEnc(key, plaintext)
+		if err != nil {
+			t.Fatal("cannot enc:", err)
+		}
+		if enc == plaintext {
+			t.Fatal(enc)
+		}
+
+		dec, err := aesDec(key, enc)
+		if err != nil {
+			t.Fatal("cannot dec:", err)
+		}
+		if dec != plaintext {
+			t.Fatalf("want decrypted %q, got %q", plaintext, dec)
+		}
+	}
+}
+
+func getCookie(key string, header http.Header) string {
+	cookies, _ := header["Set-Cookie"]
+	if len(cookies) == 0 {
+		cookies, _ = header["Cookie"]
+	}
+	for i := range cookies {
+		value := strings.Split(cookies[i], ";")[0]
+		k := value[:strings.Index(value, "=")]
+		v := value[strings.Index(value, "=")+1:]
+		if k == key {
+			return v
+		}
+	}
+	return ""
+}

server/const.go

@@ -0,0 +1,11 @@
+package server
+
+const (
+	AuthKey = "DnDex-Auth"
+	UserKey = "DnDex-User"
+)
+
+var (
+	NewAuthKey = "New-" + AuthKey
+	GitCommit  string
+)

server/middleware.go

@@ -0,0 +1,47 @@
+package server
+
+import (
+	"io"
+	"local/dndex/config"
+	"local/gziphttp"
+	"net/http"
+	"time"
+)
+
+func (rest *REST) delay(foo http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case <-time.After(config.New().Delay):
+			foo(w, r)
+		case <-r.Context().Done():
+			http.Error(w, r.Context().Err().Error(), 499)
+		}
+	}
+}
+
+func (rest *REST) defend(foo http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if gziphttp.Can(r) {
+			gz := gziphttp.New(w)
+			defer gz.Close()
+			w = gz
+		}
+		r.Body = struct {
+			io.Reader
+			io.Closer
+		}{
+			Reader: io.LimitReader(r.Body, config.New().MaxFileSize),
+			Closer: r.Body,
+		}
+		foo(w, r)
+	}
+}
+
+func (rest *REST) auth(foo http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if err := Auth(rest.g, w, r); err != nil {
+			return
+		}
+		foo(w, r)
+	}
+}

server/response.go

@@ -0,0 +1,16 @@
+package server
+
+import (
+	"encoding/json"
+	"net/http"
+)
+
+func (rest *REST) respMap(w http.ResponseWriter, key string, value interface{}) {
+	rest.resp(w, map[string]interface{}{key: value})
+}
+
+func (rest *REST) resp(w http.ResponseWriter, body interface{}) {
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "   ")
+	enc.Encode(body)
+}

server/rest.go

@@ -0,0 +1,77 @@
+package server
+
+import (
+	"fmt"
+	"local/dndex/config"
+	"local/dndex/storage"
+	"local/router"
+	"net/http"
+)
+
+type REST struct {
+	port   int
+	router *router.Router
+	g      storage.RateLimitedGraph
+}
+
+type RESTScope struct {
+	entity scope
+	user   scope
+}
+type scope struct {
+	name string
+	id   string
+}
+
+func Listen(g storage.RateLimitedGraph) error {
+	rest, err := NewREST(g)
+	if err != nil {
+		return err
+	}
+	return http.ListenAndServe(fmt.Sprintf(":%d", rest.port), rest.router)
+}
+
+func NewREST(g storage.RateLimitedGraph) (*REST, error) {
+	rest := &REST{
+		g:      g,
+		port:   config.New().Port,
+		router: router.New(),
+	}
+
+	param := router.Wildcard
+	params := router.Wildcard + router.Wildcard
+	_, _ = param, params
+
+	paths := map[string]http.HandlerFunc{
+		fmt.Sprintf("version"): rest.version,
+		fmt.Sprintf("files/%s/%s", config.New().FilePrefix, params): rest.files,
+		fmt.Sprintf("users"):               rest.users,
+		fmt.Sprintf("entities/%s", params): rest.entities,
+	}
+
+	for path, foo := range paths {
+		bar := foo
+		bar = rest.auth(bar)
+		bar = rest.defend(bar)
+		bar = rest.delay(bar)
+		if err := rest.router.Add(path, bar); err != nil {
+			return nil, err
+		}
+	}
+
+	return rest, nil
+}
+
+func (rest *REST) scope(r *http.Request) RESTScope {
+	value, _ := r.Context().Value(AuthKey).(RESTScope)
+	return value
+}
+
+func (rest *REST) files(w http.ResponseWriter, _ *http.Request) {
+}
+
+func (rest *REST) users(w http.ResponseWriter, _ *http.Request) {
+}
+
+func (rest *REST) entities(w http.ResponseWriter, _ *http.Request) {
+}

server/rest_test.go

@@ -0,0 +1,217 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"local/dndex/config"
+	"local/dndex/storage"
+	"local/dndex/storage/entity"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+var (
+	testNamespaceName = "col-name-" + uuid.New().String()[:10]
+	testNamespaceID   = "col-id-" + uuid.New().String()[:10]
+	testEntityName    = "ent-name-" + uuid.New().String()[:10]
+	testEntityID      = "ent-id-" + uuid.New().String()[:10]
+	testFilename      = "filename-" + uuid.New().String()[:10]
+	testContent       = "content-" + uuid.New().String()[:10]
+)
+
+func TestRESTRouter(t *testing.T) {
+	rest, clean := testREST(t)
+	defer clean()
+	cases := map[string]struct {
+		method string
+		is404  bool
+	}{
+		"/version": {
+			method: http.MethodGet,
+		},
+		fmt.Sprintf(`%s`, config.New().FilePrefix): {
+			method: http.MethodGet,
+			is404:  true,
+		},
+		fmt.Sprintf(`%s/`, config.New().FilePrefix): {
+			method: http.MethodGet,
+			is404:  true,
+		},
+		fmt.Sprintf(`%s/%s`, config.New().FilePrefix, testFilename): {
+			method: http.MethodGet,
+		},
+		fmt.Sprintf(`%s/fake.fake`, config.New().FilePrefix): {
+			method: http.MethodGet,
+			is404:  true,
+		},
+		fmt.Sprintf("/users/%s", testNamespaceID): {
+			method: http.MethodGet,
+			is404:  true,
+		},
+		fmt.Sprintf("/users/%s", testNamespaceID): {
+			method: http.MethodPost,
+			is404:  true,
+		},
+		"/users/": {
+			method: http.MethodGet,
+			is404:  true,
+		},
+		"/users": {
+			method: http.MethodPost,
+		},
+		"/users?": {
+			method: http.MethodGet,
+		},
+		"/entities": {
+			method: http.MethodGet,
+		},
+		"/entities/": {
+			method: http.MethodGet,
+		},
+		fmt.Sprintf("/entities/fake-%s", testEntityID): {
+			method: http.MethodGet,
+			is404:  true,
+		},
+		fmt.Sprintf("/entities/%s", testEntityID): {
+			method: http.MethodGet,
+		},
+		fmt.Sprintf("/entities/%s", testEntityID): {
+			method: http.MethodPatch,
+		},
+		fmt.Sprintf("/entities/%s", testEntityID): {
+			method: http.MethodPost,
+		},
+		fmt.Sprintf("/entities/%s", testEntityID): {
+			method: http.MethodPut,
+		},
+		fmt.Sprintf("/entities/%s/connections/uuid", testEntityID): {
+			method: http.MethodPatch,
+		},
+		fmt.Sprintf("/entities/%s/connections/uuid", testEntityID): {
+			method: http.MethodPut,
+		},
+		fmt.Sprintf("/entities/%s/connections/uuid", testEntityID): {
+			method: http.MethodPost,
+			is404:  true,
+		},
+		fmt.Sprintf("/entities/%s/connections/", testEntityID): {
+			method: http.MethodPost,
+			is404:  true,
+		},
+		fmt.Sprintf("/entities/%s/connections/", testEntityID): {
+			method: http.MethodPut,
+			is404:  true,
+		},
+		fmt.Sprintf("/entities/%s/connections/", testEntityID): {
+			method: http.MethodPatch,
+			is404:  true,
+		},
+		fmt.Sprintf("/entities/%s/connections", testEntityID): {
+			method: http.MethodPost,
+			is404:  true,
+		},
+		fmt.Sprintf("/entities/%s/connections", testEntityID): {
+			method: http.MethodPut,
+		},
+		fmt.Sprintf("/entities/%s/connections", testEntityID): {
+			method: http.MethodPatch,
+		},
+		fmt.Sprintf("/entities/%s/connections/uuid", testEntityID): {
+			method: http.MethodDelete,
+		},
+		fmt.Sprintf("/entities/%s/connections/", testEntityID): {
+			method: http.MethodDelete,
+			is404:  true,
+		},
+		fmt.Sprintf("/entities/%s/connections", testEntityID): {
+			method: http.MethodDelete,
+		},
+		fmt.Sprintf("/entities/%s/", testEntityID): {
+			method: http.MethodDelete,
+			is404:  true,
+		},
+		fmt.Sprintf("/entities/%s", testEntityID): {
+			method: http.MethodDelete,
+		},
+	}
+
+	for name, d := range cases {
+		c := d
+		path := name
+		t.Run(name, func(t *testing.T) {
+			r := httptest.NewRequest(c.method, path, strings.NewReader(``))
+			w := httptest.NewRecorder()
+			rest.router.ServeHTTP(w, r)
+			if (w.Code == http.StatusNotFound) != c.is404 {
+				t.Fatalf("want 404==%v, got %d: %+v: %s", c.is404, w.Code, c, w.Body.Bytes())
+			}
+		})
+	}
+}
+
+func testREST(t *testing.T) (*REST, func()) {
+	d, err := ioutil.TempDir(os.TempDir(), "tempdir.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(path.Join(d, testFilename), []byte(testContent), os.ModePerm); err != nil {
+		t.Fatal(err)
+	}
+	os.Setenv("FILEROOT", d)
+	os.Setenv("DRIVER_TYPE", "map")
+	os.Setenv("AUTH", "false")
+	os.Args = os.Args[:1]
+	os.Args = os.Args[:1]
+	rest, err := NewREST(storage.NewRateLimitedGraph(""))
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, can := context.WithTimeout(context.Background(), time.Second*5)
+	defer can()
+	one := randomOne()
+	one.Name = testEntityName
+	one.ID = testEntityID
+	if err := rest.g.Insert(ctx, testNamespaceID, one); err != nil {
+		t.Fatal(err)
+	}
+	if err := rest.g.Insert(ctx, testNamespaceID+"."+AuthKey, entity.One{
+		Name:  testNamespaceName,
+		ID:    testNamespaceID,
+		Title: "title",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := rest.g.Insert(ctx, testNamespaceID+"."+AuthKey, entity.One{
+		Name:  UserKey,
+		ID:    UserKey,
+		Title: "",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	return rest, func() {
+		os.RemoveAll(d)
+	}
+}
+
+func randomOne() entity.One {
+	return entity.One{
+		ID:          "iddd-" + uuid.New().String()[:5],
+		Name:        "name-" + uuid.New().String()[:5],
+		Type:        "type-" + uuid.New().String()[:5],
+		Title:       "titl-" + uuid.New().String()[:5],
+		Text:        "text-" + uuid.New().String()[:5],
+		Modified:    time.Now().UnixNano(),
+		Connections: map[string]entity.One{},
+		Attachments: map[string]string{
+			uuid.New().String()[:5]: uuid.New().String()[:5],
+		},
+	}
+}

server/version.go

@@ -0,0 +1,12 @@
+package server
+
+import (
+	"encoding/json"
+	"net/http"
+)
+
+func (rest *REST) version(w http.ResponseWriter, _ *http.Request) {
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "   ")
+	enc.Encode(map[string]string{"version": GitCommit})
+}

storage/driver/boltdb.go

@@ -289,7 +289,7 @@ func applyUnset(doc, operator bson.M) (bson.M, error) {
 				ok = pmok
 			}
 			if !ok {
-				return nil, fmt.Errorf("subpath cannot be followed for non object: %s (%s): %+v (%T)", k, nesting[0], mInterface, mInterface)
+				return nil, fmt.Errorf("subpath of %v (%v) cannot be followed for non object: %s (%s): %+v (%T)", doc, doc[nesting[0]], k, nesting[0], mInterface, mInterface)
 			}
 			subdoc, err := applyUnset(bson.M(m), bson.M{strings.Join(nesting[1:], "."): ""})
 			if err != nil {
@@ -322,7 +322,7 @@ func applySet(doc, operator bson.M) (bson.M, error) {
 				ok = pmok
 			}
 			if !ok {
-				return nil, fmt.Errorf("subpath cannot be followed for non object: %s (%s): %+v (%T)", k, nesting[0], mInterface, mInterface)
+				return nil, fmt.Errorf("subpath of %v (%v) cannot be followed for non object: %s (%s): %+v (%T)", doc, doc[nesting[0]], k, nesting[0], mInterface, mInterface)
 			}
 			subdoc, err := applySet(bson.M(m), bson.M{strings.Join(nesting[1:], "."): v})
 			if err != nil {

storage/driver/boltdb_test.go

@@ -143,9 +143,6 @@ func TestBoltDBFind(t *testing.T) {
 				if o.Text == "" {
 					t.Error(o.Text)
 				}
-				if o.Relationship != "" {
-					t.Error(o.Relationship)
-				}
 				if o.Modified == 0 {
 					t.Error(o.Modified)
 				}
@@ -156,18 +153,9 @@ func TestBoltDBFind(t *testing.T) {
 					t.Error(o.Connections)
 				}
 				for k := range o.Connections {
-					if o.Connections[k].Name == "" {
-						t.Error(o.Connections[k])
-					}
-					if o.Connections[k].Title == "" {
-						t.Error(o.Connections[k])
-					}
 					if o.Connections[k].Relationship == "" {
 						t.Error(o.Connections[k])
 					}
-					if o.Connections[k].Type == "" {
-						t.Error(o.Connections[k])
-					}
 				}
 			}
 			if n != testN {
@@ -317,13 +305,6 @@ func TestBoltDBInsert(t *testing.T) {
 					if _, ok := ones[0].Connections[k]; !ok {
 						t.Fatalf("db had more connections than real: %s", k)
 					}
-					c := o.Connections[k]
-					c.Modified = 0
-					o.Connections[k] = c
-
-					c = ones[0].Connections[k]
-					c.Modified = 0
-					ones[0].Connections[k] = c
 				}
 				o.Modified = 0
 				ones[0].Modified = 0
@@ -372,7 +353,7 @@ func TestBoltDBDelete(t *testing.T) {
 					t.Fatal(err)
 				}
 				if n != wantN {
-					t.Error(n, filter)
+					t.Error(wantN, n, filter)
 				}
 			}
 		})
@@ -404,7 +385,6 @@ func fillBoltDB(t *testing.T, bdb *BoltDB) {
 				ID:    "iddd-" + uuid.New().String()[:5],
 				Name:  "name-" + uuid.New().String()[:5],
 				Type:  "type-" + uuid.New().String()[:5],
-				Relationship: "rshp-" + uuid.New().String()[:5],
 				Title: "titl-" + uuid.New().String()[:5],
 			}
 			o := entity.One{
@@ -414,8 +394,8 @@ func fillBoltDB(t *testing.T, bdb *BoltDB) {
 				Title:       "titl-" + uuid.New().String()[:5],
 				Text:        "text-" + uuid.New().String()[:5],
 				Modified:    time.Now().UnixNano(),
-				Connections: map[string]entity.One{p.ID: p},
+				Connections: map[string]entity.Connection{p.ID: entity.Connection{p.Name}},
-				Attachments: map[string]string{"filename": "/path/to/file"},
+				Attachments: map[string]entity.Attachment{"filename": {"/path/to/file"}},
 			}
 			b, err := bson.Marshal(o)
 			if err != nil {

storage/driver/map_test.go

@@ -17,7 +17,6 @@ func tempMap(t *testing.T) *Map {
 			ID:    "iddd-" + uuid.New().String()[:5],
 			Name:  "name-" + uuid.New().String()[:5],
 			Type:  "type-" + uuid.New().String()[:5],
-			Relationship: "rshp-" + uuid.New().String()[:5],
 			Title: "titl-" + uuid.New().String()[:5],
 		}
 		o := entity.One{
@@ -27,8 +26,8 @@ func tempMap(t *testing.T) *Map {
 			Title:       "titl-" + uuid.New().String()[:5],
 			Text:        "text-" + uuid.New().String()[:5],
 			Modified:    time.Now().UnixNano(),
-			Connections: map[string]entity.One{p.ID: p},
+			Connections: map[string]entity.Connection{p.ID: entity.Connection{p.Name}},
-			Attachments: map[string]string{"filename": "/path/to/file"},
+			Attachments: map[string]entity.Attachment{"filename": {"/path/to/file"}},
 		}
 		b, err := bson.Marshal(o)
 		if err != nil {

storage/entity/one.go

@@ -19,42 +19,40 @@ const (
 	Modified     = "modified"
 	Connections  = "connections"
 	Attachments  = "attachments"
+	Location     = "location"
 )
 
 type One struct {
-	ID           string            `bson:"_id,omitempty" json:"_id,omitempty"`
+	ID          string                `bson:"_id,omitempty" json:"_id"`
-	Name         string            `bson:"name,omitempty" json:"name,omitempty"`
+	Name        string                `bson:"name,omitempty" json:"name"`
-	Type         string            `bson:"type,omitempty" json:"type,omitempty"`
+	Type        string                `bson:"type,omitempty" json:"type"`
-	Title        string            `bson:"title,omitempty" json:"title,omitempty"`
+	Title       string                `bson:"title,omitempty" json:"title"`
-	Text         string            `bson:"text,omitempty" json:"text,omitempty"`
+	Text        string                `bson:"text,omitempty" json:"text"`
-	Relationship string            `bson:"relationship,omitempty" json:"relationship,omitempty"`
+	Modified    int64                 `bson:"modified,omitempty" json:"modified"`
-	Modified     int64             `bson:"modified,omitempty" json:"modified,omitempty"`
+	Connections map[string]Connection `bson:"connections" json:"connections"`
-	Connections  map[string]One    `bson:"connections" json:"connections,omitempty"`
+	Attachments map[string]Attachment `bson:"attachments" json:"attachments"`
-	Attachments  map[string]string `bson:"attachments" json:"attachments,omitempty"`
 }
 
-func (o One) Query() One {
+type Connection struct {
-	return One{Name: o.Name}
+	Relationship string `bson:"relationship,omitempty" json:"relationship"`
 }
 
-func (o One) Peer() One {
+type Attachment struct {
-	return One{
+	Location string `bson:"location,omitempty" json:"location"`
-		Name:         o.Name,
+}
-		Type:         o.Type,
+
-		Title:        o.Title,
+func (o One) Query() bson.M {
-		Relationship: o.Relationship,
+	return bson.M{ID: o.ID}
-		Modified:     o.Modified,
-	}
 }
 
 func (o One) Peers() []string {
-	names := make([]string, len(o.Connections))
+	ids := make([]string, len(o.Connections))
 	i := 0
 	for k := range o.Connections {
-		names[i] = o.Connections[k].Name
+		ids[i] = k
 		i += 1
 	}
-	return names
+	return ids
 }
 
 func (o One) MarshalBSON() ([]byte, error) {
@@ -62,6 +60,12 @@ func (o One) MarshalBSON() ([]byte, error) {
 	if !isMin {
 		o.Modified = time.Now().UnixNano()
 	}
+	if o.Connections == nil {
+		o.Connections = make(map[string]Connection)
+	}
+	if o.Attachments == nil {
+		o.Attachments = make(map[string]Attachment)
+	}
 	b, err := json.Marshal(o)
 	if err != nil {
 		return nil, err
@@ -76,36 +80,5 @@ func (o One) MarshalBSON() ([]byte, error) {
 			m[k] = strings.TrimSpace(v.(string))
 		}
 	}
-	if !isMin {
-		connections := map[string]interface{}{}
-		switch m[Connections].(type) {
-		case nil:
-		case map[string]interface{}:
-			connections = m[Connections].(map[string]interface{})
-		default:
-			return nil, fmt.Errorf("bad connections type %T", m[Connections])
-		}
-		delete(connections, "")
-		for k := range connections {
-			if k == "" {
-				continue
-			}
-			if o.Connections[k].Name == "" {
-				p := o.Connections[k]
-				p.Name = k
-				o.Connections[k] = p
-			}
-			b, err := bson.Marshal(o.Connections[k])
-			if err != nil {
-				return nil, err
-			}
-			m := bson.M{}
-			if err := bson.Unmarshal(b, &m); err != nil {
-				return nil, err
-			}
-			connections[k] = m
-		}
-		m[Connections] = connections
-	}
 	return bson.Marshal(m)
 }

storage/entity/one_test.go

@@ -9,11 +9,11 @@ import (
 
 func TestOne(t *testing.T) {
 	one := One{
-		Name: "myname",
+		ID:   "myname",
 		Type: "mytype",
 	}
 	q := one.Query()
-	if want := fmt.Sprint(One{Name: one.Name}); want != fmt.Sprint(q) {
+	if want := fmt.Sprint(bson.M{ID: one.ID}); want != fmt.Sprint(q) {
 		t.Error(want, q)
 	}
 }
@@ -21,7 +21,7 @@ func TestOne(t *testing.T) {
 func TestOneMarshalBSON(t *testing.T) {
 	cases := map[string]struct {
 		sameAsQuery bool
-		one         One
+		one         interface{}
 	}{
 		"query no modified change": {
 			sameAsQuery: true,
@@ -31,18 +31,16 @@ func TestOneMarshalBSON(t *testing.T) {
 			one: One{Name: "hello", Type: "world", Modified: 1},
 		},
 		"w/ connections": {
-			one: One{Name: "hello", Type: "world", Modified: 1, Connections: map[string]One{"hi": One{Name: "hi", Relationship: "mom"}}},
+			one: One{Name: "hello", Type: "world", Modified: 1, Connections: map[string]Connection{"hi": Connection{Relationship: "mom"}}},
 		},
 		"w/ attachments": {
-			one: One{Name: "hello", Type: "world", Modified: 1, Attachments: map[string]string{"hello": "/world"}},
+			one: One{Name: "hello", Type: "world", Modified: 1, Attachments: map[string]Attachment{"hello": Attachment{"/world"}}},
 		},
 	}
 
 	for name, d := range cases {
 		c := d
 		t.Run(name, func(t *testing.T) {
-			var bm bson.Marshaler = c.one
-			t.Log(bm)
 			b, err := bson.Marshal(c.one)
 			if err != nil {
 				t.Fatal(err)
@@ -51,85 +49,11 @@ func TestOneMarshalBSON(t *testing.T) {
 			if err := bson.Unmarshal(b, &one); err != nil {
 				t.Fatal(err)
 			}
-			if c.sameAsQuery && (fmt.Sprint(one) != fmt.Sprint(one.Query()) || fmt.Sprint(one) != fmt.Sprint(c.one)) {
+			if !c.sameAsQuery {
-				t.Error(c.sameAsQuery, c.one, one)
+				if one.Modified < 2 {
-			} else if !c.sameAsQuery {
+					t.Error(one.Modified)
-				if c.one.Modified == one.Modified {
-					t.Error(c.one.Modified, one.Modified)
-				}
-				c.one.Modified = 0
-				one.Modified = 0
-				for k := range one.Connections {
-					temp := one.Connections[k]
-					temp.Modified = 0
-					one.Connections[k] = temp
-				}
-				if fmt.Sprint(c.one) != fmt.Sprint(one) {
-					t.Error(c.one, one)
 				}
 			}
 		})
 	}
 }
-
-func TestOneMarshalBSONBadConnections(t *testing.T) {
-	t.Run("connections has an empty string for a key that should die", func(t *testing.T) {
-		input := One{Name: "hello", Connections: map[string]One{"": One{Name: "teehee"}}}
-
-		b, err := bson.Marshal(input)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		output := One{}
-		if err := bson.Unmarshal(b, &output); err != nil {
-			t.Fatal(err)
-		}
-
-		if len(output.Connections) != 0 {
-			t.Fatal(output.Connections)
-		}
-
-		input.Connections = nil
-		output.Connections = nil
-		input.Modified = 0
-		output.Modified = 0
-
-		if fmt.Sprint(input) != fmt.Sprint(output) {
-			t.Fatal(input, output)
-		}
-	})
-
-	t.Run("connections has a key but empty name that should correct", func(t *testing.T) {
-		input := One{Name: "hello", Connections: map[string]One{"teehee": One{Name: ""}}}
-
-		b, err := bson.Marshal(input)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		output := One{}
-		if err := bson.Unmarshal(b, &output); err != nil {
-			t.Fatal(err)
-		}
-
-		if len(output.Connections) != 1 {
-			t.Fatal(output.Connections)
-		} else {
-			for k := range output.Connections {
-				if k != output.Connections[k].Name {
-					t.Fatal(k, output.Connections)
-				}
-			}
-		}
-
-		input.Connections = nil
-		output.Connections = nil
-		input.Modified = 0
-		output.Modified = 0
-
-		if fmt.Sprint(input) != fmt.Sprint(output) {
-			t.Fatal(input, output)
-		}
-	})
-}

storage/graph.go

@@ -31,6 +31,20 @@ func (g Graph) ListCaseInsensitive(ctx context.Context, namespace string, from .
 	return g.find(ctx, namespace, filter)
 }
 
+func (g Graph) Get(ctx context.Context, namespace, id string) (entity.One, error) {
+	ones, err := g.find(ctx, namespace, bson.M{entity.ID: id})
+	if err != nil {
+		return entity.One{}, err
+	}
+	if len(ones) == 0 {
+		return entity.One{}, errors.New("not found")
+	}
+	if len(ones) > 1 {
+		return entity.One{}, errors.New("primary key collision detected")
+	}
+	return ones[0], nil
+}
+
 func (g Graph) List(ctx context.Context, namespace string, from ...string) ([]entity.One, error) {
 	filter := operator.NewFilterIn(entity.Name, from)
 	return g.find(ctx, namespace, filter)
@@ -68,7 +82,7 @@ func (g Graph) Insert(ctx context.Context, namespace string, one entity.One) err
 	return g.driver.Insert(ctx, namespace, one)
 }
 
-func (g Graph) Update(ctx context.Context, namespace string, one entity.One, modify interface{}) error {
+func (g Graph) Update(ctx context.Context, namespace string, one, modify interface{}) error {
 	return g.driver.Update(ctx, namespace, one, modify)
 }
 

storage/graph_test.go

@@ -29,7 +29,7 @@ func TestIntegration(t *testing.T) {
 		randomOne(),
 		randomOne(),
 	}
-	ones[0].Connections = map[string]entity.One{ones[2].Name: entity.One{Name: ones[2].Name, Relationship: ":("}}
+	ones[0].Connections = map[string]entity.Connection{ones[2].Name: entity.Connection{Relationship: ":("}}
 	ones[1].Name = ones[0].Name[1 : len(ones[0].Name)-1]
 	cleanFill := func() {
 		clean()
@@ -62,6 +62,30 @@ func TestIntegration(t *testing.T) {
 		}
 	})
 
+	t.Run("graph.Get 404", func(t *testing.T) {
+		cleanFill()
+		_, err := graph.Get(ctx, "col", "fake_here")
+		if err == nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("graph.Get", func(t *testing.T) {
+		cleanFill()
+		all, err := graph.List(ctx, "col")
+		if err != nil {
+			t.Fatal(err)
+		}
+		want := all[0]
+		got, err := graph.Get(ctx, "col", want.ID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got.ID != want.ID {
+			t.Fatal(got)
+		}
+	})
+
 	t.Run("graph.ListCaseInsensitive", func(t *testing.T) {
 		cleanFill()
 		all, err := graph.ListCaseInsensitive(ctx, "col")
@@ -201,9 +225,9 @@ func TestIntegration(t *testing.T) {
 
 	t.Run("graph.Update(foo, +=2); graph.Update(foo, -=1)", func(t *testing.T) {
 		cleanFill()
-		err := graph.Update(ctx, "col", ones[0].Query(), operator.Set{entity.Connections, map[string]entity.One{
+		err := graph.Update(ctx, "col", ones[0].Query(), operator.Set{entity.Connections, map[string]entity.Connection{
-			"hello": entity.One{Name: "hello", Relationship: ":("},
+			"hello": entity.Connection{Relationship: ":("},
-			"world": entity.One{Name: "world", Relationship: ":("},
+			"world": entity.Connection{Relationship: ":("},
 		}})
 		if err != nil {
 			t.Fatal(err)
@@ -240,7 +264,7 @@ func TestIntegration(t *testing.T) {
 
 	t.Run("graph.Update(new attachment), Update(--new attachment)", func(t *testing.T) {
 		cleanFill()
-		err := graph.Update(ctx, "col", ones[0].Query(), operator.Set{Key: fmt.Sprintf("%s.new attachment", entity.Attachments), Value: "my new attachment"})
+		err := graph.Update(ctx, "col", ones[0].Query(), operator.Set{Key: fmt.Sprintf("%s.new attachment", entity.Attachments), Value: entity.Attachment{Location: "my new attachment"}})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -255,8 +279,8 @@ func TestIntegration(t *testing.T) {
 		}
 		if v, ok := some1[0].Attachments["new attachment"]; !ok {
 			t.Fatal(ok, some1[0].Attachments)
-		} else if v != "my new attachment" {
+		} else if v.Location != "my new attachment" {
-			t.Fatal(v, some1[0].Attachments)
+			t.Fatalf("when listing from DB, did not find updated attachment: got %+v from %+v", v, some1[0].Attachments)
 		}
 
 		err = graph.Update(ctx, "col", ones[0].Query(), operator.Unset(fmt.Sprintf("%s.new attachment", entity.Attachments)))
@@ -372,10 +396,10 @@ func randomOne() entity.One {
 		Title:       "Biggus",
 		Text:        "tee hee xd",
 		Modified:    time.Now().UnixNano(),
-		Connections: map[string]entity.One{},
+		Connections: map[string]entity.Connection{},
-		Attachments: map[string]string{
+		Attachments: map[string]entity.Attachment{
-			"pdf file": "/path/to.pdf",
+			"pdf file": entity.Attachment{Location: "/path/to.pdf"},
-			"png file": "/path/to.png",
+			"png file": entity.Attachment{Location: "/path/to.png"},
 		},
 	}
 }

storage/ratelimitedgraph.go

@@ -46,6 +46,10 @@ func (rlg RateLimitedGraph) Insert(ctx context.Context, namespace string, one en
 	return rlg.g.Insert(ctx, namespace, one)
 }
 
+func (rlg RateLimitedGraph) Get(ctx context.Context, namespace, id string) (entity.One, error) {
+	return rlg.g.Get(ctx, namespace, id)
+}
+
 func (rlg RateLimitedGraph) List(ctx context.Context, namespace string, from ...string) ([]entity.One, error) {
 	return rlg.g.List(ctx, namespace, from...)
 }
@@ -58,6 +62,6 @@ func (rlg RateLimitedGraph) Search(ctx context.Context, namespace string, nameCo
 	return rlg.g.Search(ctx, namespace, nameContains)
 }
 
-func (rlg RateLimitedGraph) Update(ctx context.Context, namespace string, one entity.One, modify interface{}) error {
+func (rlg RateLimitedGraph) Update(ctx context.Context, namespace string, one, modify interface{}) error {
 	return rlg.g.Update(ctx, namespace, one, modify)
 }