local
/
storage
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cleaner testing, leveldb can sort and limit

master
bel 2019-12-27 19:44:48 -07:00
parent 08b2e8461a
commit 67160f5060
4 changed files with 297 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
db.go
View File

@ -2,6 +2,7 @@ package storage
import (
"fmt"
"strconv"
"strings"
)
@ -62,16 +63,36 @@ func resolveNamespace(ns []string) string {
}
func resolveLimits(input []string) []string {
output := []string{"", ""}
if len(input) > 0 {
output[0] = input[0]
} else {
output[0] = " "
return []string{
resolveLimitsStart(input),
resolveLimitsStop(input),
resolveLimitsLimit(input),
resolveLimitsAscending(input),
}
if len(input) > 1 {
output[1] = input[1]
} else {
output[1] = "}}}}}}}}}}}}}"
}
return output[:]
}
func resolveLimitsStart(input []string) string {
if len(input) > 0 {
return input[0]
}
return " "
}
func resolveLimitsStop(input []string) string {
if len(input) > 1 {
return input[1]
}
return "}}}}}}"
}
func resolveLimitsLimit(input []string) string {
if len(input) > 2 {
v, _ := strconv.Atoi(input[2])
return strconv.Itoa(v)
}
return "0"
}
func resolveLimitsAscending(input []string) string {
return strconv.FormatBool(len(input) < 4 || input[3] != "-")
}

174
db_test.go
View File

@ -70,16 +70,24 @@ func TestImplementations(t *testing.T) {
cases = append(cases, cacheFile)
}
if dynomite, err := NewDynomite("localhost:8102", "", ""); err != nil {
t.Logf("cannot make dynomite: %v", err)
if _, ok := os.LookupEnv("DYNOMITE"); ok {
if dynomite, err := NewDynomite("localhost:8102", "", ""); err != nil {
t.Logf("cannot make dynomite: %v", err)
} else {
cases = append(cases, dynomite)
}
} else {
cases = append(cases, dynomite)
t.Log("$DYNOMITE not set. Skipping")
}
if redis, err := NewRedis("localhost:8103", "", ""); err != nil {
t.Logf("cannot make redis: %v", err)
if _, ok := os.LookupEnv("REDIS"); ok {
if redis, err := NewRedis("localhost:8103", "", ""); err != nil {
t.Logf("cannot make redis: %v", err)
} else {
cases = append(cases, redis)
}
} else {
cases = append(cases, redis)
t.Log("$REDIS not set. Skipping")
}
if bolt, err := NewBolt(path.Join(dir, "bolt")); err != nil {
@ -94,66 +102,88 @@ func TestImplementations(t *testing.T) {
cases = append(cases, leveldb)
}
mongoLN, err := net.Listen("tcp", "localhost:27017")
if err == nil {
defer mongoLN.Close()
go func() {
for {
conn, err := mongoLN.Accept()
if err == nil {
conn.Close()
if _, ok := os.LookupEnv("MONGO"); ok {
mongoLN, err := net.Listen("tcp", "localhost:27017")
if err == nil {
defer mongoLN.Close()
go func() {
for {
conn, err := mongoLN.Accept()
if err == nil {
conn.Close()
}
}
}
}()
}
if mongo1, err := NewMongo("localhost:27017"); err == nil {
cases = append(cases, mongo1)
} else if mongo2, err := NewMongo("localhost:27017", "root", "pass"); err == nil {
cases = append(cases, mongo2)
}()
}
if mongo1, err := NewMongo("localhost:27017"); err == nil {
cases = append(cases, mongo1)
} else if mongo2, err := NewMongo("localhost:27017", "root", "pass"); err == nil {
cases = append(cases, mongo2)
} else {
t.Logf("cannot make mongo: %v", err)
}
} else {
t.Logf("cannot make mongo: %v", err)
t.Log("$MONGO not set. Skipping")
}
if memcache, err := NewMemcache("localhost:11211"); err != nil {
t.Logf("cannot make memcache: %v", err)
if _, ok := os.LookupEnv("MEMCACHED"); ok {
if memcache, err := NewMemcache("localhost:11211"); err != nil {
t.Logf("cannot make memcache: %v", err)
} else {
cases = append(cases, memcache)
}
} else {
cases = append(cases, memcache)
t.Log("$MEMCACHED not set. Skipping")
}
if memcacheCluster, err := NewMemcacheCluster("localhost:11211"); err != nil {
t.Logf("cannot make memcacheCluster: %v", err)
if _, ok := os.LookupEnv("MEMCACHEDCLUSTER"); ok {
if memcacheCluster, err := NewMemcacheCluster("localhost:11211"); err != nil {
t.Logf("cannot make memcacheCluster: %v", err)
} else {
cases = append(cases, memcacheCluster)
}
} else {
cases = append(cases, memcacheCluster)
t.Log("$MEMCACHEDCLUSTER not set. Skipping")
}
if minio, err := NewMinio("localhost:9000", "accesskey", "secretkey"); err != nil {
t.Logf("cannot make minio: %v", err)
if _, ok := os.LookupEnv("MINIO"); ok {
if minio, err := NewMinio("localhost:9000", "accesskey", "secretkey"); err != nil {
t.Logf("cannot make minio: %v", err)
} else {
cases = append(cases, minio)
}
} else {
cases = append(cases, minio)
t.Log("$MINIO not set. Skipping")
}
validKey := "key"
validValue := []byte("value")
for _, db := range cases {
log.Printf("Trying %T", db)
t.Logf(" %T: set", db)
if err := db.Set(validKey, validValue, "ns1", "ns2"); err != nil {
t.Errorf("%T) cannot set: %v", db, err)
}
t.Logf(" %T: get", db)
if v, err := db.Get(validKey, "ns1", "ns2"); err != nil {
t.Errorf("%T) cannot get: %v", db, err)
} else if !bytes.Equal(v, validValue) {
t.Errorf("%T) wrong get: %q vs %q", db, v, validValue)
} else if keys, err := db.List([]string{"ns1", "ns2"}); err != nil || len(keys) < 1 {
}
t.Logf(" %T: list1", db)
if keys, err := db.List([]string{"ns1", "ns2"}); err != nil || len(keys) < 1 {
t.Errorf("%T) cannot List(): %v", db, err)
} else if !strings.Contains(keys[0], validKey) {
t.Errorf("%T) List()[0] != %s: %s", db, validKey, keys[0])
} else if keys, err := db.List([]string{"ns1", "ns2"}, validKey[:1]); err != nil || len(keys) < 1 {
}
t.Logf(" %T: list2", db)
if keys, err := db.List([]string{"ns1", "ns2"}, validKey[:1]); err != nil || len(keys) < 1 {
t.Errorf("%T) cannot List(prefix): %v", db, err)
} else if !strings.Contains(keys[0], validKey) {
t.Errorf("%T) List(prefix)[0] != %s: %s", db, validKey, keys[0])
} else {
t.Logf("%25T GET: %s", db, v)
}
t.Logf(" %T: close", db)
if err := db.Close(); err != nil {
t.Errorf("cannot close %T: %v", db, err)
}
@ -242,3 +272,77 @@ func recoverDeferred(c Type, t *testing.T, wg *sync.WaitGroup) {
}
defer wg.Done()
}
func TestResolveLimitsStart(t *testing.T) {
cases := map[string]struct {
in string
out string
}{
"explicit": {in: "no", out: "no"},
}
for name, c := range cases {
input := strings.Split(c.in, ",")
out := resolveLimitsStart(input)
if out != c.out {
t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in)
}
}
}
func TestResolveLimitsStop(t *testing.T) {
cases := map[string]struct {
in string
out string
}{
"short arr": {in: "", out: "}}}}}}"},
"explicit": {in: ",no", out: "no"},
}
for name, c := range cases {
input := strings.Split(c.in, ",")
out := resolveLimitsStop(input)
if out != c.out {
t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in)
}
}
}
func TestResolveLimitsLimit(t *testing.T) {
cases := map[string]struct {
in string
out string
}{
"15": {in: ",,15", out: "15"},
"0 set": {in: ",,0", out: "0"},
"0 default": {in: ",,", out: "0"},
}
for name, c := range cases {
input := strings.Split(c.in, ",")
out := resolveLimitsLimit(input)
if out != c.out {
t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in)
}
}
}
func TestResolveLimitsAscending(t *testing.T) {
cases := map[string]struct {
in string
out string
}{
"desc": {in: "a,b,c,-", out: "false"},
"asc": {in: "a,b,c,+", out: "true"},
"default asc prev": {in: "a,b,c,", out: "true"},
"default asc empty": {in: ",,,", out: "true"},
}
for name, c := range cases {
input := strings.Split(c.in, ",")
out := resolveLimitsAscending(input)
if out != c.out {
t.Errorf("%v: got %v, want %v from %v", name, out, c.out, c.in)
}
}
}

81
leveldb.go
View File

@ -2,10 +2,12 @@ package storage
import (
"path"
"strconv"
"strings"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/util"
)
@ -33,31 +35,6 @@ func NewLevelDB(path string) (*LevelDB, error) {
}, err
}
func (ldb *LevelDB) List(ns []string, limits ...string) ([]string, error) {
namespace := resolveNamespace(ns)
namespace = path.Join(namespace)
limits = resolveLimits(limits)
limits[0] = path.Join(namespace, limits[0])
limits[1] = path.Join(namespace, limits[1])
keys := []string{}
r := util.BytesPrefix([]byte(namespace))
it := ldb.db.NewIterator(r, nil)
defer it.Release()
for it.Next() {
k := string(it.Key())
if k < limits[0] {
continue
} else if k > limits[1] {
break
}
k = strings.TrimPrefix(k, namespace+"/")
keys = append(keys, k)
}
err := it.Error()
return keys, err
}
func (ldb *LevelDB) Get(key string, ns ...string) ([]byte, error) {
namespace := resolveNamespace(ns)
snapshot, err := ldb.db.GetSnapshot()
@ -91,3 +68,57 @@ func (ldb *LevelDB) Set(key string, value []byte, ns ...string) error {
func (ldb *LevelDB) Close() error {
return ldb.db.Close()
}
func (ldb *LevelDB) List(ns []string, limits ...string) ([]string, error) {
namespace := path.Join(resolveNamespace(ns))
limits = resolveLimits(limits)
it, next := ldb.getIterator(namespace, limits)
defer it.Release()
keys := ldb.useIterator(it, next, namespace, limits)
err := it.Error()
return keys, err
}
func (ldb *LevelDB) getIterator(namespace string, limits []string) (iterator.Iterator, func() bool) {
limits[0] = path.Join(namespace, limits[0])
limits[1] = path.Join(namespace, limits[1])
asc := limits[3] == "true"
r := util.BytesPrefix([]byte(namespace))
it := ldb.db.NewIterator(r, nil)
next := it.Next
it.First()
if !asc {
next = it.Prev
it.Last()
}
return it, next
}
func (ldb *LevelDB) useIterator(it iterator.Iterator, next func() bool, namespace string, limits []string) []string {
limits = resolveLimits(limits)
n, _ := strconv.Atoi(limits[2])
m := 0
for it.Error() == nil && !ldb.inRange(string(it.Key()), limits[0], limits[1]) && next() {
}
keys := []string{}
for it.Error() == nil && ldb.inRange(string(it.Key()), limits[0], limits[1]) {
k := string(it.Key())
k = strings.TrimPrefix(k, namespace+"/")
keys = append(keys, k)
m += 1
if n > 0 && m >= n {
break
}
if !next() {
break
}
}
return keys
}
func (ldb *LevelDB) inRange(k, start, stop string) bool {
return k != "" && k >= start && k <= stop
}

70
leveldb_test.go Normal file
View File

@ -0,0 +1,70 @@
package storage
import (
"fmt"
"io/ioutil"
"os"
"strconv"
"testing"
)
func TestLevelDBListLimitedAscending(t *testing.T) {
d, err := ioutil.TempDir(os.TempDir(), "leveldb.list.test.*")
if err != nil {
t.Fatal(err)
}
db, err := NewLevelDB(d)
if err != nil {
t.Fatal(err)
}
db.Set("key", []byte("hi"), "bad")
n := 20
for i := 0; i < n; i++ {
db.Set(fmt.Sprintf("%02d", i), []byte(strconv.Itoa(i)), "good")
}
if list, err := db.List([]string{}); err != nil {
t.Error(err)
} else if len(list) != 0 {
t.Error(len(list), list)
}
if list, err := db.List([]string{""}); err != nil {
t.Error(err)
} else if len(list) != n+1 {
t.Error(len(list), list)
}
if list, err := db.List([]string{"good"}); err != nil {
t.Error(err)
} else if len(list) != n {
t.Error(len(list), list)
}
if list, err := db.List([]string{"good"}, "10"); err != nil {
t.Error(err)
} else if len(list) != n-10 {
t.Error(len(list), list)
}
if list, err := db.List([]string{"good"}, "10", "15"); err != nil {
t.Error(err)
} else if len(list) != 15-10+1 {
t.Error(len(list), list)
}
if list, err := db.List([]string{"good"}, "10", "15", "2"); err != nil {
t.Error(err)
} else if len(list) != 2 {
t.Error(len(list), list)
}
if list, err := db.List([]string{"good"}, "10", "15", "2", "-"); err != nil {
t.Error(err)
} else if len(list) != 2 {
t.Error(len(list), list)
} else if list[1] > list[0] {
t.Errorf("not desc: %v", list)
}
}