From ade973d19decc5c9128ec04e16868f23e6ba2f20 Mon Sep 17 00:00:00 2001 From: bel Date: Fri, 21 Jun 2019 17:13:18 -0600 Subject: [PATCH] working on List --- bolt.go | 19 ++++++++++++ cache.go | 5 ++++ cli/main.go | 74 ++++++++++++++++++++++++++++++++++++++++++++++ db.go | 16 ++++++++++ db_test.go | 16 ++++++++++ leveldb.go | 13 ++++++++ map.go | 5 ++++ memcache.go | 5 ++++ memcachecluster.go | 5 ++++ minio.go | 5 ++++ mongo.go | 5 ++++ redis.go | 6 ++-- riak.go | 6 ++++ 13 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 cli/main.go diff --git a/bolt.go b/bolt.go index f23e5a6..461ec3f 100644 --- a/bolt.go +++ b/bolt.go @@ -13,6 +13,25 @@ func NewBolt(path string) (*Bolt, error) { }, err } +func (b *Bolt) List(ns []string, limits ...string) ([]string, error) { + namespace := resolveNamespace(ns) + limits = resolveLimits(limits) + found := []string{} + err := b.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(namespace)) + if bucket == nil { + return nil + } + + c := bucket.Cursor() + for k, _ := c.Seek([]byte(limits[0])); k != nil; k, _ = c.Next() { + found = append(found, string(k)) + } + return nil + }) + return found, err +} + func (b *Bolt) Get(key string, ns ...string) ([]byte, error) { namespace := resolveNamespace(ns) var result []byte diff --git a/cache.go b/cache.go index 0bf958a..0c478f0 100644 --- a/cache.go +++ b/cache.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "os" "path" "time" @@ -25,6 +26,10 @@ func NewCache(path ...string) (*Cache, error) { return c, err } +func (c *Cache) List(ns []string, limits ...string) ([]string, error) { + return nil, errors.New("not impl") +} + func (c *Cache) Get(key string, ns ...string) ([]byte, error) { namespace := resolveNamespace(ns) v, ok := c.db.Get(path.Join(namespace, key)) diff --git a/cli/main.go b/cli/main.go new file mode 100644 index 0000000..20b6ed4 --- /dev/null +++ b/cli/main.go @@ -0,0 +1,74 @@ +package main + +import ( + "bufio" + "fmt" + "local/args" + "local/storage" + "log" + "os" + "strings" + "time" +) + +func main() { + as := args.NewArgSet() + as.Append(args.STRING, "addr", "addr of store", "") + as.Append(args.STRING, "db", "type of store", "map") + as.Append(args.STRING, "user", "user of store", "") + as.Append(args.STRING, "pass", "pass of store", "") + as.Append(args.STRING, "do", "[get set]", "get") + as.Append(args.STRING, "k", "key", "key") + as.Append(args.STRING, "v", "value", "value") + if err := as.Parse(); err != nil { + panic(err) + } + + db, err := storage.New( + storage.TypeFromString(as.Get("db").GetString()), + as.Get("addr").GetString(), + as.Get("user").GetString(), + as.Get("pass").GetString(), + ) + if err != nil { + panic(err) + } + + start := time.Now() + var b []byte + switch strings.ToLower(as.Get("do").GetString()) { + case "get": + b, err = db.Get(as.Get("k").GetString()) + case "set": + err = db.Set(as.Get("k").GetString(), []byte(as.Get("v").GetString())) + case "cli": + reader := bufio.NewReader(os.Stdin) + for { + fmt.Print("> ") + line, err := reader.ReadString('\n') + if err != nil { + panic(err) + } + s := strings.Split(line, " ") + s[len(s)-1] = strings.TrimSpace(s[len(s)-1]) + for len(s) < 3 { + s = append(s, "") + } + start := time.Now() + switch strings.ToLower(s[0]) { + case "get": + b, err := db.Get(s[1]) + stop := time.Now() + log.Printf("(%v) %v: %s => %s", stop.Sub(start), err, s[1], b) + case "set": + err := db.Set(s[1], []byte(s[2])) + stop := time.Now() + log.Printf("(%v) %v: %s", stop.Sub(start), err, s[1]) + } + } + default: + log.Fatal("-do not in [get set]") + } + stop := time.Now() + log.Printf("(%v) %v: %s", stop.Sub(start), err, b) +} diff --git a/db.go b/db.go index 303f504..9711b54 100644 --- a/db.go +++ b/db.go @@ -8,6 +8,7 @@ import ( type DB interface { Get(string, ...string) ([]byte, error) Set(string, []byte, ...string) error + List([]string, ...string) ([]string, error) Close() error } @@ -57,3 +58,18 @@ func resolveNamespace(ns []string) string { } return namespace } + +func resolveLimits(input []string) []string { + output := []string{"", ""} + if len(input) > 0 { + output[0] = input[0] + } else { + output[0] = " " + } + if len(input) > 1 { + output[1] = input[1] + } else { + output[1] = "}}}}}}}}}}}}}" + } + return output[:] +} diff --git a/db_test.go b/db_test.go index d068f17..c162225 100644 --- a/db_test.go +++ b/db_test.go @@ -15,6 +15,14 @@ type mock struct { m map[string][]byte } +func (mock *mock) List(ns []string, limits ...string) ([]string, error) { + keys := []string{} + for k := range mock.m { + keys = append(keys, k) + } + return keys, nil +} + func (mock *mock) Get(key string, ns ...string) ([]byte, error) { namespace := resolveNamespace(ns) v, ok := mock.m[path.Join(namespace, key)] @@ -143,6 +151,14 @@ func TestImplementations(t *testing.T) { t.Errorf("cannot get %T: %v", db, err) } else if !bytes.Equal(v, validValue) { t.Errorf("wrong get %T: %q vs %q", db, v, validValue) + } else if keys, err := db.List(nil); err != nil || len(keys) < 1 { + t.Errorf("cannot List(): %v", err) + } else if keys[0] != validKey { + t.Errorf("List()[0] != %s: %s", validKey, keys[0]) + } else if keys, err := db.List(nil, validKey[:1]); err != nil || len(keys) < 1 { + t.Errorf("cannot List(prefix): %v", err) + } else if keys[0] != validKey { + t.Errorf("List(prefix)[0] != %s: %s", validKey, keys[0]) } else { t.Logf("%25T GET: %s", db, v) } diff --git a/leveldb.go b/leveldb.go index b985567..0855d0c 100644 --- a/leveldb.go +++ b/leveldb.go @@ -6,6 +6,7 @@ import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/util" ) type LevelDB struct { @@ -21,6 +22,18 @@ func NewLevelDB(path string) (*LevelDB, error) { }, err } +func (ldb *LevelDB) List(ns []string, limits ...string) ([]string, error) { + keys := []string{} + r := util.BytesPrefix([]byte{}) + it := ldb.db.NewIterator(r, nil) + defer it.Release() + for it.Next() { + keys = append(keys, string(it.Key())) + } + err := it.Error() + return keys, err +} + func (ldb *LevelDB) Get(key string, ns ...string) ([]byte, error) { namespace := resolveNamespace(ns) snapshot, err := ldb.db.GetSnapshot() diff --git a/map.go b/map.go index c33ee22..db739d7 100644 --- a/map.go +++ b/map.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "fmt" ) @@ -28,6 +29,10 @@ func (m *Map) Close() error { return nil } +func (m *Map) List(ns []string, limits ...string) ([]string, error) { + return nil, errors.New("not impl") +} + func (m *Map) Get(key string, ns ...string) ([]byte, error) { namespace := resolveNamespace(ns) if _, ok := (*m)[namespace]; !ok { diff --git a/memcache.go b/memcache.go index 17392b7..18542ca 100644 --- a/memcache.go +++ b/memcache.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "net" "path" @@ -47,6 +48,10 @@ func NewMemcache(addr string, addrs ...string) (*Memcache, error) { return &Memcache{db: db}, nil } +func (mc *Memcache) List(ns []string, limits ...string) ([]string, error) { + return nil, errors.New("not impl") +} + func (mc *Memcache) Get(key string, ns ...string) ([]byte, error) { namespace := resolveNamespace(ns) v, err := mc.db.Get(path.Join(namespace, key)) diff --git a/memcachecluster.go b/memcachecluster.go index 260473d..ca21b59 100644 --- a/memcachecluster.go +++ b/memcachecluster.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "net" "path" @@ -69,6 +70,10 @@ func NewMemcacheCluster(addr string, addrs ...string) (*MemcacheCluster, error) return &MemcacheCluster{db: db}, nil } +func (mc *MemcacheCluster) List(ns []string, limits ...string) ([]string, error) { + return nil, errors.New("not impl") +} + func (mc *MemcacheCluster) Get(key string, ns ...string) ([]byte, error) { namespace := resolveNamespace(ns) v, err := mc.db.Get(path.Join(namespace, key)) diff --git a/minio.go b/minio.go index c1e0a02..ce4f170 100644 --- a/minio.go +++ b/minio.go @@ -2,6 +2,7 @@ package storage import ( "bytes" + "errors" "io/ioutil" "strings" @@ -17,6 +18,10 @@ func NewMinio(addr, user, pass string) (*Minio, error) { return &Minio{db: db}, err } +func (m *Minio) List(ns []string, limits ...string) ([]string, error) { + return nil, errors.New("not impl") +} + func (m *Minio) Get(key string, ns ...string) ([]byte, error) { namespace := resolveNamespace(ns) obj, err := m.db.GetObject(namespace, key, minio.GetObjectOptions{}) diff --git a/mongo.go b/mongo.go index 332c857..be0b007 100644 --- a/mongo.go +++ b/mongo.go @@ -2,6 +2,7 @@ package storage import ( "context" + "errors" "strings" "time" @@ -53,6 +54,10 @@ func NewMongo(addr string, auth ...string) (*Mongo, error) { return &Mongo{db: db}, nil } +func (mg *Mongo) List(ns []string, limits ...string) ([]string, error) { + return nil, errors.New("not impl") +} + func (mg *Mongo) Get(key string, ns ...string) ([]byte, error) { namespace := resolveNamespace(ns) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) diff --git a/redis.go b/redis.go index 205c9a7..b7b361b 100644 --- a/redis.go +++ b/redis.go @@ -3,7 +3,6 @@ package storage import ( "errors" "fmt" - "log" "time" "github.com/gomodule/redigo/redis" @@ -33,9 +32,12 @@ func (m *Redis) Close() error { return m.client.Close() } +func (m *Redis) List(ns []string, limits ...string) ([]string, error) { + return nil, errors.New("not impl") +} + func (m *Redis) Get(key string, ns ...string) ([]byte, error) { key = resolveNamespace(append(ns, key)) - log.Println(key) resp, err := m.client.Do("GET", key) if err != nil { return nil, err diff --git a/riak.go b/riak.go index 67b67b3..1dfbd93 100644 --- a/riak.go +++ b/riak.go @@ -1,6 +1,8 @@ package storage import ( + "errors" + riak "github.com/basho/riak-go-client" ) @@ -28,6 +30,10 @@ func NewRiak(addr string, addrs ...string) (*Riak, error) { }, err } +func (r *Riak) List(ns []string, limits ...string) ([]string, error) { + return nil, errors.New("not impl") +} + func (r *Riak) Get(key string, ns ...string) ([]byte, error) { namespace := resolveNamespace(ns) obj := &riak.Object{}