From 67160f506054bfdab941516c237e6875e4d90ee9 Mon Sep 17 00:00:00 2001 From: bel Date: Fri, 27 Dec 2019 19:44:48 -0700 Subject: [PATCH] cleaner testing, leveldb can sort and limit --- db.go | 43 +++++++++--- db_test.go | 174 ++++++++++++++++++++++++++++++++++++++---------- leveldb.go | 81 +++++++++++++++------- leveldb_test.go | 70 +++++++++++++++++++ 4 files changed, 297 insertions(+), 71 deletions(-) create mode 100644 leveldb_test.go diff --git a/db.go b/db.go index 9571c38..d777239 100755 --- a/db.go +++ b/db.go @@ -2,6 +2,7 @@ package storage import ( "fmt" + "strconv" "strings" ) @@ -62,16 +63,36 @@ func resolveNamespace(ns []string) string { } func resolveLimits(input []string) []string { - output := []string{"", ""} - if len(input) > 0 { - output[0] = input[0] - } else { - output[0] = " " + return []string{ + resolveLimitsStart(input), + resolveLimitsStop(input), + resolveLimitsLimit(input), + resolveLimitsAscending(input), } - if len(input) > 1 { - output[1] = input[1] - } else { - output[1] = "}}}}}}}}}}}}}" - } - return output[:] +} + +func resolveLimitsStart(input []string) string { + if len(input) > 0 { + return input[0] + } + return " " +} + +func resolveLimitsStop(input []string) string { + if len(input) > 1 { + return input[1] + } + return "}}}}}}" +} + +func resolveLimitsLimit(input []string) string { + if len(input) > 2 { + v, _ := strconv.Atoi(input[2]) + return strconv.Itoa(v) + } + return "0" +} + +func resolveLimitsAscending(input []string) string { + return strconv.FormatBool(len(input) < 4 || input[3] != "-") } diff --git a/db_test.go b/db_test.go index 597b90c..6f1a49d 100755 --- a/db_test.go +++ b/db_test.go @@ -70,16 +70,24 @@ func TestImplementations(t *testing.T) { cases = append(cases, cacheFile) } - if dynomite, err := NewDynomite("localhost:8102", "", ""); err != nil { - t.Logf("cannot make dynomite: %v", err) + if _, ok := os.LookupEnv("DYNOMITE"); ok { + if dynomite, err := NewDynomite("localhost:8102", "", ""); err != nil { + t.Logf("cannot make dynomite: %v", err) + } else { + cases = append(cases, dynomite) + } } else { - cases = append(cases, dynomite) + t.Log("$DYNOMITE not set. Skipping") } - if redis, err := NewRedis("localhost:8103", "", ""); err != nil { - t.Logf("cannot make redis: %v", err) + if _, ok := os.LookupEnv("REDIS"); ok { + if redis, err := NewRedis("localhost:8103", "", ""); err != nil { + t.Logf("cannot make redis: %v", err) + } else { + cases = append(cases, redis) + } } else { - cases = append(cases, redis) + t.Log("$REDIS not set. Skipping") } if bolt, err := NewBolt(path.Join(dir, "bolt")); err != nil { @@ -94,66 +102,88 @@ func TestImplementations(t *testing.T) { cases = append(cases, leveldb) } - mongoLN, err := net.Listen("tcp", "localhost:27017") - if err == nil { - defer mongoLN.Close() - go func() { - for { - conn, err := mongoLN.Accept() - if err == nil { - conn.Close() + if _, ok := os.LookupEnv("MONGO"); ok { + mongoLN, err := net.Listen("tcp", "localhost:27017") + if err == nil { + defer mongoLN.Close() + go func() { + for { + conn, err := mongoLN.Accept() + if err == nil { + conn.Close() + } } - } - }() - } - if mongo1, err := NewMongo("localhost:27017"); err == nil { - cases = append(cases, mongo1) - } else if mongo2, err := NewMongo("localhost:27017", "root", "pass"); err == nil { - cases = append(cases, mongo2) + }() + } + if mongo1, err := NewMongo("localhost:27017"); err == nil { + cases = append(cases, mongo1) + } else if mongo2, err := NewMongo("localhost:27017", "root", "pass"); err == nil { + cases = append(cases, mongo2) + } else { + t.Logf("cannot make mongo: %v", err) + } } else { - t.Logf("cannot make mongo: %v", err) + t.Log("$MONGO not set. Skipping") } - if memcache, err := NewMemcache("localhost:11211"); err != nil { - t.Logf("cannot make memcache: %v", err) + if _, ok := os.LookupEnv("MEMCACHED"); ok { + if memcache, err := NewMemcache("localhost:11211"); err != nil { + t.Logf("cannot make memcache: %v", err) + } else { + cases = append(cases, memcache) + } } else { - cases = append(cases, memcache) + t.Log("$MEMCACHED not set. Skipping") } - if memcacheCluster, err := NewMemcacheCluster("localhost:11211"); err != nil { - t.Logf("cannot make memcacheCluster: %v", err) + if _, ok := os.LookupEnv("MEMCACHEDCLUSTER"); ok { + if memcacheCluster, err := NewMemcacheCluster("localhost:11211"); err != nil { + t.Logf("cannot make memcacheCluster: %v", err) + } else { + cases = append(cases, memcacheCluster) + } } else { - cases = append(cases, memcacheCluster) + t.Log("$MEMCACHEDCLUSTER not set. Skipping") } - if minio, err := NewMinio("localhost:9000", "accesskey", "secretkey"); err != nil { - t.Logf("cannot make minio: %v", err) + if _, ok := os.LookupEnv("MINIO"); ok { + if minio, err := NewMinio("localhost:9000", "accesskey", "secretkey"); err != nil { + t.Logf("cannot make minio: %v", err) + } else { + cases = append(cases, minio) + } } else { - cases = append(cases, minio) + t.Log("$MINIO not set. Skipping") } validKey := "key" validValue := []byte("value") for _, db := range cases { + log.Printf("Trying %T", db) + t.Logf(" %T: set", db) if err := db.Set(validKey, validValue, "ns1", "ns2"); err != nil { t.Errorf("%T) cannot set: %v", db, err) } + t.Logf(" %T: get", db) if v, err := db.Get(validKey, "ns1", "ns2"); err != nil { t.Errorf("%T) cannot get: %v", db, err) } else if !bytes.Equal(v, validValue) { t.Errorf("%T) wrong get: %q vs %q", db, v, validValue) - } else if keys, err := db.List([]string{"ns1", "ns2"}); err != nil || len(keys) < 1 { + } + t.Logf(" %T: list1", db) + if keys, err := db.List([]string{"ns1", "ns2"}); err != nil || len(keys) < 1 { t.Errorf("%T) cannot List(): %v", db, err) } else if !strings.Contains(keys[0], validKey) { t.Errorf("%T) List()[0] != %s: %s", db, validKey, keys[0]) - } else if keys, err := db.List([]string{"ns1", "ns2"}, validKey[:1]); err != nil || len(keys) < 1 { + } + t.Logf(" %T: list2", db) + if keys, err := db.List([]string{"ns1", "ns2"}, validKey[:1]); err != nil || len(keys) < 1 { t.Errorf("%T) cannot List(prefix): %v", db, err) } else if !strings.Contains(keys[0], validKey) { t.Errorf("%T) List(prefix)[0] != %s: %s", db, validKey, keys[0]) - } else { - t.Logf("%25T GET: %s", db, v) } + t.Logf(" %T: close", db) if err := db.Close(); err != nil { t.Errorf("cannot close %T: %v", db, err) } @@ -242,3 +272,77 @@ func recoverDeferred(c Type, t *testing.T, wg *sync.WaitGroup) { } defer wg.Done() } + +func TestResolveLimitsStart(t *testing.T) { + cases := map[string]struct { + in string + out string + }{ + "explicit": {in: "no", out: "no"}, + } + + for name, c := range cases { + input := strings.Split(c.in, ",") + out := resolveLimitsStart(input) + if out != c.out { + t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in) + } + } +} + +func TestResolveLimitsStop(t *testing.T) { + cases := map[string]struct { + in string + out string + }{ + "short arr": {in: "", out: "}}}}}}"}, + "explicit": {in: ",no", out: "no"}, + } + + for name, c := range cases { + input := strings.Split(c.in, ",") + out := resolveLimitsStop(input) + if out != c.out { + t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in) + } + } +} + +func TestResolveLimitsLimit(t *testing.T) { + cases := map[string]struct { + in string + out string + }{ + "15": {in: ",,15", out: "15"}, + "0 set": {in: ",,0", out: "0"}, + "0 default": {in: ",,", out: "0"}, + } + + for name, c := range cases { + input := strings.Split(c.in, ",") + out := resolveLimitsLimit(input) + if out != c.out { + t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in) + } + } +} + +func TestResolveLimitsAscending(t *testing.T) { + cases := map[string]struct { + in string + out string + }{ + "desc": {in: "a,b,c,-", out: "false"}, + "asc": {in: "a,b,c,+", out: "true"}, + "default asc prev": {in: "a,b,c,", out: "true"}, + "default asc empty": {in: ",,,", out: "true"}, + } + + for name, c := range cases { + input := strings.Split(c.in, ",") + out := resolveLimitsAscending(input) + if out != c.out { + t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in) + } + } +} diff --git a/leveldb.go b/leveldb.go index 875d54d..8a59987 100755 --- a/leveldb.go +++ b/leveldb.go @@ -2,10 +2,12 @@ package storage import ( "path" + "strconv" "strings" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/filter" + "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/util" ) @@ -33,31 +35,6 @@ func NewLevelDB(path string) (*LevelDB, error) { }, err } -func (ldb *LevelDB) List(ns []string, limits ...string) ([]string, error) { - namespace := resolveNamespace(ns) - namespace = path.Join(namespace) - limits = resolveLimits(limits) - limits[0] = path.Join(namespace, limits[0]) - limits[1] = path.Join(namespace, limits[1]) - - keys := []string{} - r := util.BytesPrefix([]byte(namespace)) - it := ldb.db.NewIterator(r, nil) - defer it.Release() - for it.Next() { - k := string(it.Key()) - if k < limits[0] { - continue - } else if k > limits[1] { - break - } - k = strings.TrimPrefix(k, namespace+"/") - keys = append(keys, k) - } - err := it.Error() - return keys, err -} - func (ldb *LevelDB) Get(key string, ns ...string) ([]byte, error) { namespace := resolveNamespace(ns) snapshot, err := ldb.db.GetSnapshot() @@ -91,3 +68,57 @@ func (ldb *LevelDB) Set(key string, value []byte, ns ...string) error { func (ldb *LevelDB) Close() error { return ldb.db.Close() } + +func (ldb *LevelDB) List(ns []string, limits ...string) ([]string, error) { + namespace := path.Join(resolveNamespace(ns)) + limits = resolveLimits(limits) + it, next := ldb.getIterator(namespace, limits) + defer it.Release() + keys := ldb.useIterator(it, next, namespace, limits) + err := it.Error() + return keys, err +} + +func (ldb *LevelDB) getIterator(namespace string, limits []string) (iterator.Iterator, func() bool) { + limits[0] = path.Join(namespace, limits[0]) + limits[1] = path.Join(namespace, limits[1]) + asc := limits[3] == "true" + + r := util.BytesPrefix([]byte(namespace)) + it := ldb.db.NewIterator(r, nil) + next := it.Next + it.First() + if !asc { + next = it.Prev + it.Last() + } + return it, next +} + +func (ldb *LevelDB) useIterator(it iterator.Iterator, next func() bool, namespace string, limits []string) []string { + limits = resolveLimits(limits) + n, _ := strconv.Atoi(limits[2]) + m := 0 + + for it.Error() == nil && !ldb.inRange(string(it.Key()), limits[0], limits[1]) && next() { + } + + keys := []string{} + for it.Error() == nil && ldb.inRange(string(it.Key()), limits[0], limits[1]) { + k := string(it.Key()) + k = strings.TrimPrefix(k, namespace+"/") + keys = append(keys, k) + m += 1 + if n > 0 && m >= n { + break + } + if !next() { + break + } + } + return keys +} + +func (ldb *LevelDB) inRange(k, start, stop string) bool { + return k != "" && k >= start && k <= stop +} diff --git a/leveldb_test.go b/leveldb_test.go new file mode 100644 index 0000000..361698e --- /dev/null +++ b/leveldb_test.go @@ -0,0 +1,70 @@ +package storage + +import ( + "fmt" + "io/ioutil" + "os" + "strconv" + "testing" +) + +func TestLevelDBListLimitedAscending(t *testing.T) { + d, err := ioutil.TempDir(os.TempDir(), "leveldb.list.test.*") + if err != nil { + t.Fatal(err) + } + db, err := NewLevelDB(d) + if err != nil { + t.Fatal(err) + } + + db.Set("key", []byte("hi"), "bad") + n := 20 + for i := 0; i < n; i++ { + db.Set(fmt.Sprintf("%02d", i), []byte(strconv.Itoa(i)), "good") + } + + if list, err := db.List([]string{}); err != nil { + t.Error(err) + } else if len(list) != 0 { + t.Error(len(list), list) + } + + if list, err := db.List([]string{""}); err != nil { + t.Error(err) + } else if len(list) != n+1 { + t.Error(len(list), list) + } + + if list, err := db.List([]string{"good"}); err != nil { + t.Error(err) + } else if len(list) != n { + t.Error(len(list), list) + } + + if list, err := db.List([]string{"good"}, "10"); err != nil { + t.Error(err) + } else if len(list) != n-10 { + t.Error(len(list), list) + } + + if list, err := db.List([]string{"good"}, "10", "15"); err != nil { + t.Error(err) + } else if len(list) != 15-10+1 { + t.Error(len(list), list) + } + + if list, err := db.List([]string{"good"}, "10", "15", "2"); err != nil { + t.Error(err) + } else if len(list) != 2 { + t.Error(len(list), list) + } + + if list, err := db.List([]string{"good"}, "10", "15", "2", "-"); err != nil { + t.Error(err) + } else if len(list) != 2 { + t.Error(len(list), list) + } else if list[1] > list[0] { + t.Errorf("not desc: %v", list) + } +}