cleaner testing, leveldb can sort and limit

db.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 )
 
@@ -62,16 +63,36 @@ func resolveNamespace(ns []string) string {
 }
 
 func resolveLimits(input []string) []string {
-	output := []string{"", ""}
+	return []string{
+		resolveLimitsStart(input),
+		resolveLimitsStop(input),
+		resolveLimitsLimit(input),
+		resolveLimitsAscending(input),
+	}
+}
+
+func resolveLimitsStart(input []string) string {
 	if len(input) > 0 {
-		output[0] = input[0]
+		return input[0]
-	} else {
-		output[0] = " "
 	}
+	return " "
+}
+
+func resolveLimitsStop(input []string) string {
 	if len(input) > 1 {
-		output[1] = input[1]
+		return input[1]
-	} else {
-		output[1] = "}}}}}}}}}}}}}"
 	}
-	return output[:]
+	return "}}}}}}"
+}
+
+func resolveLimitsLimit(input []string) string {
+	if len(input) > 2 {
+		v, _ := strconv.Atoi(input[2])
+		return strconv.Itoa(v)
+	}
+	return "0"
+}
+
+func resolveLimitsAscending(input []string) string {
+	return strconv.FormatBool(len(input) < 4 || input[3] != "-")
 }

db_test.go

@@ -70,17 +70,25 @@ func TestImplementations(t *testing.T) {
 		cases = append(cases, cacheFile)
 	}
 
+	if _, ok := os.LookupEnv("DYNOMITE"); ok {
 		if dynomite, err := NewDynomite("localhost:8102", "", ""); err != nil {
 			t.Logf("cannot make dynomite: %v", err)
 		} else {
 			cases = append(cases, dynomite)
 		}
+	} else {
+		t.Log("$DYNOMITE not set. Skipping")
+	}
 
+	if _, ok := os.LookupEnv("REDIS"); ok {
 		if redis, err := NewRedis("localhost:8103", "", ""); err != nil {
 			t.Logf("cannot make redis: %v", err)
 		} else {
 			cases = append(cases, redis)
 		}
+	} else {
+		t.Log("$REDIS not set. Skipping")
+	}
 
 	if bolt, err := NewBolt(path.Join(dir, "bolt")); err != nil {
 		t.Errorf("cannot make bolt: %v", err)
@@ -94,6 +102,7 @@ func TestImplementations(t *testing.T) {
 		cases = append(cases, leveldb)
 	}
 
+	if _, ok := os.LookupEnv("MONGO"); ok {
 		mongoLN, err := net.Listen("tcp", "localhost:27017")
 		if err == nil {
 			defer mongoLN.Close()
@@ -113,47 +122,68 @@ func TestImplementations(t *testing.T) {
 		} else {
 			t.Logf("cannot make mongo: %v", err)
 		}
+	} else {
+		t.Log("$MONGO not set. Skipping")
+	}
 
+	if _, ok := os.LookupEnv("MEMCACHED"); ok {
 		if memcache, err := NewMemcache("localhost:11211"); err != nil {
 			t.Logf("cannot make memcache: %v", err)
 		} else {
 			cases = append(cases, memcache)
 		}
+	} else {
+		t.Log("$MEMCACHED not set. Skipping")
+	}
 
+	if _, ok := os.LookupEnv("MEMCACHEDCLUSTER"); ok {
 		if memcacheCluster, err := NewMemcacheCluster("localhost:11211"); err != nil {
 			t.Logf("cannot make memcacheCluster: %v", err)
 		} else {
 			cases = append(cases, memcacheCluster)
 		}
+	} else {
+		t.Log("$MEMCACHEDCLUSTER not set. Skipping")
+	}
 
+	if _, ok := os.LookupEnv("MINIO"); ok {
 		if minio, err := NewMinio("localhost:9000", "accesskey", "secretkey"); err != nil {
 			t.Logf("cannot make minio: %v", err)
 		} else {
 			cases = append(cases, minio)
 		}
+	} else {
+		t.Log("$MINIO not set. Skipping")
+	}
 
 	validKey := "key"
 	validValue := []byte("value")
 
 	for _, db := range cases {
+		log.Printf("Trying %T", db)
+		t.Logf("   %T: set", db)
 		if err := db.Set(validKey, validValue, "ns1", "ns2"); err != nil {
 			t.Errorf("%T) cannot set: %v", db, err)
 		}
+		t.Logf("   %T: get", db)
 		if v, err := db.Get(validKey, "ns1", "ns2"); err != nil {
 			t.Errorf("%T) cannot get: %v", db, err)
 		} else if !bytes.Equal(v, validValue) {
 			t.Errorf("%T) wrong get: %q vs %q", db, v, validValue)
-		} else if keys, err := db.List([]string{"ns1", "ns2"}); err != nil || len(keys) < 1 {
+		}
+		t.Logf("   %T: list1", db)
+		if keys, err := db.List([]string{"ns1", "ns2"}); err != nil || len(keys) < 1 {
 			t.Errorf("%T) cannot List(): %v", db, err)
 		} else if !strings.Contains(keys[0], validKey) {
 			t.Errorf("%T) List()[0] != %s: %s", db, validKey, keys[0])
-		} else if keys, err := db.List([]string{"ns1", "ns2"}, validKey[:1]); err != nil || len(keys) < 1 {
+		}
+		t.Logf("   %T: list2", db)
+		if keys, err := db.List([]string{"ns1", "ns2"}, validKey[:1]); err != nil || len(keys) < 1 {
 			t.Errorf("%T) cannot List(prefix): %v", db, err)
 		} else if !strings.Contains(keys[0], validKey) {
 			t.Errorf("%T) List(prefix)[0] != %s: %s", db, validKey, keys[0])
-		} else {
-			t.Logf("%25T GET: %s", db, v)
 		}
+		t.Logf("   %T: close", db)
 		if err := db.Close(); err != nil {
 			t.Errorf("cannot close %T: %v", db, err)
 		}
@@ -242,3 +272,77 @@ func recoverDeferred(c Type, t *testing.T, wg *sync.WaitGroup) {
 	}
 	defer wg.Done()
 }
+
+func TestResolveLimitsStart(t *testing.T) {
+	cases := map[string]struct {
+		in  string
+		out string
+	}{
+		"explicit": {in: "no", out: "no"},
+	}
+
+	for name, c := range cases {
+		input := strings.Split(c.in, ",")
+		out := resolveLimitsStart(input)
+		if out != c.out {
+			t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in)
+		}
+	}
+}
+
+func TestResolveLimitsStop(t *testing.T) {
+	cases := map[string]struct {
+		in  string
+		out string
+	}{
+		"short arr": {in: "", out: "}}}}}}"},
+		"explicit":  {in: ",no", out: "no"},
+	}
+
+	for name, c := range cases {
+		input := strings.Split(c.in, ",")
+		out := resolveLimitsStop(input)
+		if out != c.out {
+			t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in)
+		}
+	}
+}
+
+func TestResolveLimitsLimit(t *testing.T) {
+	cases := map[string]struct {
+		in  string
+		out string
+	}{
+		"15":        {in: ",,15", out: "15"},
+		"0 set":     {in: ",,0", out: "0"},
+		"0 default": {in: ",,", out: "0"},
+	}
+
+	for name, c := range cases {
+		input := strings.Split(c.in, ",")
+		out := resolveLimitsLimit(input)
+		if out != c.out {
+			t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in)
+		}
+	}
+}
+
+func TestResolveLimitsAscending(t *testing.T) {
+	cases := map[string]struct {
+		in  string
+		out string
+	}{
+		"desc":              {in: "a,b,c,-", out: "false"},
+		"asc":               {in: "a,b,c,+", out: "true"},
+		"default asc prev":  {in: "a,b,c,", out: "true"},
+		"default asc empty": {in: ",,,", out: "true"},
+	}
+
+	for name, c := range cases {
+		input := strings.Split(c.in, ",")
+		out := resolveLimitsAscending(input)
+		if out != c.out {
+			t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in)
+		}
+	}
+}

leveldb.go

@@ -2,10 +2,12 @@ package storage
 
 import (
 	"path"
+	"strconv"
 	"strings"
 
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
@@ -33,31 +35,6 @@ func NewLevelDB(path string) (*LevelDB, error) {
 	}, err
 }
 
-func (ldb *LevelDB) List(ns []string, limits ...string) ([]string, error) {
-	namespace := resolveNamespace(ns)
-	namespace = path.Join(namespace)
-	limits = resolveLimits(limits)
-	limits[0] = path.Join(namespace, limits[0])
-	limits[1] = path.Join(namespace, limits[1])
-
-	keys := []string{}
-	r := util.BytesPrefix([]byte(namespace))
-	it := ldb.db.NewIterator(r, nil)
-	defer it.Release()
-	for it.Next() {
-		k := string(it.Key())
-		if k < limits[0] {
-			continue
-		} else if k > limits[1] {
-			break
-		}
-		k = strings.TrimPrefix(k, namespace+"/")
-		keys = append(keys, k)
-	}
-	err := it.Error()
-	return keys, err
-}
-
 func (ldb *LevelDB) Get(key string, ns ...string) ([]byte, error) {
 	namespace := resolveNamespace(ns)
 	snapshot, err := ldb.db.GetSnapshot()
@@ -91,3 +68,57 @@ func (ldb *LevelDB) Set(key string, value []byte, ns ...string) error {
 func (ldb *LevelDB) Close() error {
 	return ldb.db.Close()
 }
+
+func (ldb *LevelDB) List(ns []string, limits ...string) ([]string, error) {
+	namespace := path.Join(resolveNamespace(ns))
+	limits = resolveLimits(limits)
+	it, next := ldb.getIterator(namespace, limits)
+	defer it.Release()
+	keys := ldb.useIterator(it, next, namespace, limits)
+	err := it.Error()
+	return keys, err
+}
+
+func (ldb *LevelDB) getIterator(namespace string, limits []string) (iterator.Iterator, func() bool) {
+	limits[0] = path.Join(namespace, limits[0])
+	limits[1] = path.Join(namespace, limits[1])
+	asc := limits[3] == "true"
+
+	r := util.BytesPrefix([]byte(namespace))
+	it := ldb.db.NewIterator(r, nil)
+	next := it.Next
+	it.First()
+	if !asc {
+		next = it.Prev
+		it.Last()
+	}
+	return it, next
+}
+
+func (ldb *LevelDB) useIterator(it iterator.Iterator, next func() bool, namespace string, limits []string) []string {
+	limits = resolveLimits(limits)
+	n, _ := strconv.Atoi(limits[2])
+	m := 0
+
+	for it.Error() == nil && !ldb.inRange(string(it.Key()), limits[0], limits[1]) && next() {
+	}
+
+	keys := []string{}
+	for it.Error() == nil && ldb.inRange(string(it.Key()), limits[0], limits[1]) {
+		k := string(it.Key())
+		k = strings.TrimPrefix(k, namespace+"/")
+		keys = append(keys, k)
+		m += 1
+		if n > 0 && m >= n {
+			break
+		}
+		if !next() {
+			break
+		}
+	}
+	return keys
+}
+
+func (ldb *LevelDB) inRange(k, start, stop string) bool {
+	return k != "" && k >= start && k <= stop
+}

leveldb_test.go

@@ -0,0 +1,70 @@
+package storage
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"testing"
+)
+
+func TestLevelDBListLimitedAscending(t *testing.T) {
+	d, err := ioutil.TempDir(os.TempDir(), "leveldb.list.test.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+	db, err := NewLevelDB(d)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db.Set("key", []byte("hi"), "bad")
+	n := 20
+	for i := 0; i < n; i++ {
+		db.Set(fmt.Sprintf("%02d", i), []byte(strconv.Itoa(i)), "good")
+	}
+
+	if list, err := db.List([]string{}); err != nil {
+		t.Error(err)
+	} else if len(list) != 0 {
+		t.Error(len(list), list)
+	}
+
+	if list, err := db.List([]string{""}); err != nil {
+		t.Error(err)
+	} else if len(list) != n+1 {
+		t.Error(len(list), list)
+	}
+
+	if list, err := db.List([]string{"good"}); err != nil {
+		t.Error(err)
+	} else if len(list) != n {
+		t.Error(len(list), list)
+	}
+
+	if list, err := db.List([]string{"good"}, "10"); err != nil {
+		t.Error(err)
+	} else if len(list) != n-10 {
+		t.Error(len(list), list)
+	}
+
+	if list, err := db.List([]string{"good"}, "10", "15"); err != nil {
+		t.Error(err)
+	} else if len(list) != 15-10+1 {
+		t.Error(len(list), list)
+	}
+
+	if list, err := db.List([]string{"good"}, "10", "15", "2"); err != nil {
+		t.Error(err)
+	} else if len(list) != 2 {
+		t.Error(len(list), list)
+	}
+
+	if list, err := db.List([]string{"good"}, "10", "15", "2", "-"); err != nil {
+		t.Error(err)
+	} else if len(list) != 2 {
+		t.Error(len(list), list)
+	} else if list[1] > list[0] {
+		t.Errorf("not desc: %v", list)
+	}
+}