local
/
storage
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement list and fix test

master
bel 2019-06-21 17:57:00 -06:00
parent ade973d19d
commit 52479ed8a0
12 changed files with 161 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cache.go
View File

@ -1,7 +1,6 @@
package storage
import (
"errors"
"os"
"path"
"time"
@ -27,7 +26,18 @@ func NewCache(path ...string) (*Cache, error) {
}
func (c *Cache) List(ns []string, limits ...string) ([]string, error) {
return nil, errors.New("not impl")
namespace := resolveNamespace(ns)
limits = resolveLimits(limits)
limits[0] = path.Join(namespace, limits[0])
limits[1] = path.Join(namespace, limits[1])
m := c.db.Items()
keys := []string{}
for k := range m {
if k >= limits[0] && k <= limits[1] {
keys = append(keys, k)
}
}
return keys, nil
}
func (c *Cache) Get(key string, ns ...string) ([]byte, error) {

8
cli/main.go
View File

@ -13,8 +13,8 @@ import (
func main() {
as := args.NewArgSet()
as.Append(args.STRING, "addr", "addr of store", "")
as.Append(args.STRING, "db", "type of store", "map")
as.Append(args.STRING, "addr", "addr of store", "localhost:8102")
as.Append(args.STRING, "db", "type of store", "dynomite")
as.Append(args.STRING, "user", "user of store", "")
as.Append(args.STRING, "pass", "pass of store", "")
as.Append(args.STRING, "do", "[get set]", "get")
@ -37,6 +37,10 @@ func main() {
start := time.Now()
var b []byte
switch strings.ToLower(as.Get("do").GetString()) {
case "list":
var c []string
c, err = db.List(nil, as.Get("k").GetString())
b = []byte(fmt.Sprintf("%v", c))
case "get":
b, err = db.Get(as.Get("k").GetString())
case "set":

2
db.go
View File

@ -22,6 +22,8 @@ func New(key Type, params ...string) (db DB, err error) {
}()
err = ErrNotImpl
switch key {
case DYNOMITE:
db, err = NewDynomite(params[0], params[1], params[2])
case REDIS:
db, err = NewRedis(params[0], params[1], params[2])
case MAP:

44
db_test.go
View File

@ -7,6 +7,7 @@ import (
"net"
"os"
"path"
"strings"
"sync"
"testing"
)
@ -16,9 +17,12 @@ type mock struct {
}
func (mock *mock) List(ns []string, limits ...string) ([]string, error) {
limits = resolveLimits(limits)
keys := []string{}
for k := range mock.m {
keys = append(keys, k)
if k >= limits[0] && k <= limits[1] {
keys = append(keys, k)
}
}
return keys, nil
}
@ -66,8 +70,14 @@ func TestImplementations(t *testing.T) {
cases = append(cases, cacheFile)
}
if redis, err := NewRedis("localhost:6379", "", ""); err != nil {
t.Errorf("cannot make redis: %v", err)
if dynomite, err := NewDynomite("localhost:8102", "", ""); err != nil {
t.Logf("cannot make dynomite: %v", err)
} else {
cases = append(cases, dynomite)
}
if redis, err := NewRedis("localhost:8103", "", ""); err != nil {
t.Logf("cannot make redis: %v", err)
} else {
cases = append(cases, redis)
}
@ -97,7 +107,7 @@ func TestImplementations(t *testing.T) {
}()
}
if riak, err := NewRiak("localhost:8087"); err != nil {
t.Errorf("cannot make riak: %v", err)
t.Logf("cannot make riak: %v", err)
} else {
cases = append(cases, riak)
}
@ -119,23 +129,23 @@ func TestImplementations(t *testing.T) {
} else if mongo2, err := NewMongo("localhost:27017", "root", "pass"); err == nil {
cases = append(cases, mongo2)
} else {
t.Errorf("cannot make mongo: %v", err)
t.Logf("cannot make mongo: %v", err)
}
if memcache, err := NewMemcache("localhost:11211"); err != nil {
t.Errorf("cannot make memcache: %v", err)
t.Logf("cannot make memcache: %v", err)
} else {
cases = append(cases, memcache)
}
if memcacheCluster, err := NewMemcacheCluster("localhost:11211"); err != nil {
t.Errorf("cannot make memcacheCluster: %v", err)
t.Logf("cannot make memcacheCluster: %v", err)
} else {
cases = append(cases, memcacheCluster)
}
if minio, err := NewMinio("localhost:9000", "accesskey", "secretkey"); err != nil {
t.Errorf("cannot make minio: %v", err)
t.Logf("cannot make minio: %v", err)
} else {
cases = append(cases, minio)
}
@ -145,20 +155,20 @@ func TestImplementations(t *testing.T) {
for _, db := range cases {
if err := db.Set(validKey, validValue); err != nil {
t.Errorf("cannot set %T: %v", db, err)
t.Errorf("%T) cannot set: %v", db, err)
}
if v, err := db.Get(validKey); err != nil {
t.Errorf("cannot get %T: %v", db, err)
t.Errorf("%T) cannot get: %v", db, err)
} else if !bytes.Equal(v, validValue) {
t.Errorf("wrong get %T: %q vs %q", db, v, validValue)
t.Errorf("%T) wrong get: %q vs %q", db, v, validValue)
} else if keys, err := db.List(nil); err != nil || len(keys) < 1 {
t.Errorf("cannot List(): %v", err)
} else if keys[0] != validKey {
t.Errorf("List()[0] != %s: %s", validKey, keys[0])
t.Errorf("%T) cannot List(): %v", db, err)
} else if !strings.Contains(keys[0], validKey) {
t.Errorf("%T) List()[0] != %s: %s", db, validKey, keys[0])
} else if keys, err := db.List(nil, validKey[:1]); err != nil || len(keys) < 1 {
t.Errorf("cannot List(prefix): %v", err)
} else if keys[0] != validKey {
t.Errorf("List(prefix)[0] != %s: %s", validKey, keys[0])
t.Errorf("%T) cannot List(prefix): %v", db, err)
} else if !strings.Contains(keys[0], validKey) {
t.Errorf("%T) List(prefix)[0] != %s: %s", db, validKey, keys[0])
} else {
t.Logf("%25T GET: %s", db, v)
}

10
dynomite.go Normal file
View File

@ -0,0 +1,10 @@
package storage
type Dynomite struct {
*Redis
}
func NewDynomite(addr, user, pass string) (*Dynomite, error) {
r, err := NewRedis(addr, user, pass)
return &Dynomite{Redis: r}, err
}

15
leveldb.go
View File

@ -23,12 +23,23 @@ func NewLevelDB(path string) (*LevelDB, error) {
}
func (ldb *LevelDB) List(ns []string, limits ...string) ([]string, error) {
namespace := resolveNamespace(ns)
limits = resolveLimits(limits)
limits[0] = path.Join(namespace, limits[0])
limits[1] = path.Join(namespace, limits[1])
keys := []string{}
r := util.BytesPrefix([]byte{})
r := util.BytesPrefix([]byte(namespace))
it := ldb.db.NewIterator(r, nil)
defer it.Release()
for it.Next() {
keys = append(keys, string(it.Key()))
k := string(it.Key())
if k < limits[0] {
continue
} else if k > limits[1] {
break
}
keys = append(keys, k)
}
err := it.Error()
return keys, err

16
map.go
View File

@ -1,7 +1,6 @@
package storage
import (
"errors"
"fmt"
)
@ -30,7 +29,20 @@ func (m *Map) Close() error {
}
func (m *Map) List(ns []string, limits ...string) ([]string, error) {
return nil, errors.New("not impl")
namespace := resolveNamespace(ns)
limits = resolveLimits(limits)
keys := []string{}
if _, ok := (*m)[namespace]; !ok {
return nil, nil
}
for k := range (*m)[namespace] {
if k >= limits[0] && k <= limits[1] {
keys = append(keys, k)
}
}
return keys, nil
}
func (m *Map) Get(key string, ns ...string) ([]byte, error) {

16
minio.go
View File

@ -2,7 +2,6 @@ package storage
import (
"bytes"
"errors"
"io/ioutil"
"strings"
@ -19,7 +18,20 @@ func NewMinio(addr, user, pass string) (*Minio, error) {
}
func (m *Minio) List(ns []string, limits ...string) ([]string, error) {
return nil, errors.New("not impl")
namespace := resolveNamespace(ns)
limits = resolveLimits(limits)
done := make(chan struct{})
defer close(done)
keys := []string{}
for resp := range m.db.ListObjects(namespace, "", true, done) {
if resp.Key < limits[0] {
continue
} else if resp.Key > limits[1] {
break
}
keys = append(keys, resp.Key)
}
return keys, nil
}
func (m *Minio) Get(key string, ns ...string) ([]byte, error) {

31
mongo.go
View File

@ -55,7 +55,36 @@ func NewMongo(addr string, auth ...string) (*Mongo, error) {
}
func (mg *Mongo) List(ns []string, limits ...string) ([]string, error) {
return nil, errors.New("not impl")
namespace := resolveNamespace(ns)
limits = resolveLimits(limits)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
collection := mg.db.Database(DefaultNamespace).Collection(namespace)
filter := bson.M{"_id": bson.M{
"$gte": limits[0],
"$lte": limits[1],
}}
projection := bson.M{"_id": 1}
cursor, err := collection.Find(ctx, filter, options.Find().SetProjection(projection))
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
keys := []string{}
for cursor.Next(ctx) {
var elem bson.Raw
if err := cursor.Decode(&elem); err != nil {
return nil, err
}
if raw, err := elem.LookupErr("_id"); err != nil {
return nil, err
} else if s, ok := raw.StringValueOK(); !ok {
return nil, errors.New("_id is not a string")
} else {
keys = append(keys, s)
}
}
return keys, nil
}
func (mg *Mongo) Get(key string, ns ...string) ([]byte, error) {

30
redis.go
View File

@ -33,7 +33,35 @@ func (m *Redis) Close() error {
}
func (m *Redis) List(ns []string, limits ...string) ([]string, error) {
return nil, errors.New("not impl")
limits = resolveLimits(limits)
limits[0] = resolveNamespace(append(ns, limits[0]))
limits[1] = resolveNamespace(append(ns, limits[1]))
resp, err := m.client.Do("KEYS", "*")
if err != nil {
return nil, err
}
keys := []string{}
if results, ok := resp.([]interface{}); !ok {
return nil, ErrNotFound
} else {
for i := range results {
_, ok := results[i].([]uint8)
if !ok {
return nil, errors.New("not a []byte key")
}
k := fmt.Sprintf("%s", results[i])
if k < limits[0] {
continue
}
if k > limits[1] {
break
}
keys = append(keys, k)
}
}
return keys, nil
}
func (m *Redis) Get(key string, ns ...string) ([]byte, error) {

1
riak.go
View File

@ -11,6 +11,7 @@ type Riak struct {
}
func NewRiak(addr string, addrs ...string) (*Riak, error) {
return nil, ErrNotImpl
clientOpts := &riak.NewClientOptions{
RemoteAddresses: append(addrs, addr),
}

3
type.go
View File

@ -9,6 +9,7 @@ type Type int
const (
MAP = Type(iota)
REDIS = Type(iota)
DYNOMITE = Type(iota)
BOLT = Type(iota)
COCKROACH = Type(iota)
CACHE = Type(iota)
@ -21,6 +22,8 @@ const (
func (t Type) String() string {
switch t {
case DYNOMITE:
return "dynomite"
case REDIS:
return "redis"
case MAP: