335 lines
8.1 KiB
Go
Executable File
335 lines
8.1 KiB
Go
Executable File
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"local/storage/minio"
|
|
"local/storage/rclone"
|
|
"local/storage/resolve"
|
|
"log"
|
|
"net"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type mock struct {
|
|
m map[string][]byte
|
|
}
|
|
|
|
func (mock *mock) List(ns []string, limits ...string) ([]string, error) {
|
|
namespace := resolve.Namespace(ns)
|
|
limits = resolve.Limits(limits)
|
|
keys := []string{}
|
|
for k := range mock.m {
|
|
if k >= limits[0] && k <= limits[1] {
|
|
keys = append(keys, strings.TrimPrefix(k, namespace+"/"))
|
|
}
|
|
}
|
|
return keys, nil
|
|
}
|
|
|
|
func (mock *mock) Get(key string, ns ...string) ([]byte, error) {
|
|
namespace := resolve.Namespace(ns)
|
|
v, ok := mock.m[path.Join(namespace, key)]
|
|
if ok {
|
|
return v, nil
|
|
}
|
|
return nil, ErrNotFound
|
|
}
|
|
|
|
func (mock *mock) Set(key string, value []byte, ns ...string) error {
|
|
namespace := resolve.Namespace(ns)
|
|
mock.m[path.Join(namespace, key)] = value
|
|
return nil
|
|
}
|
|
|
|
func (mock *mock) Close() error {
|
|
return nil
|
|
}
|
|
|
|
func TestImplementations(t *testing.T) {
|
|
dir := path.Join(t.TempDir(), "storage_tests")
|
|
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer os.RemoveAll(dir)
|
|
|
|
cases := make([]DB, 0)
|
|
cases = append(cases, &mock{m: make(map[string][]byte)})
|
|
cases = append(cases, NewMap())
|
|
cases = append(cases, NewMapStream())
|
|
|
|
if cacheMem, err := NewCache(); err != nil {
|
|
t.Errorf("cannot make cache/mem: %v", err)
|
|
} else {
|
|
cases = append(cases, cacheMem)
|
|
}
|
|
|
|
if cacheFile, err := NewCache(path.Join(dir, "cache")); err != nil {
|
|
t.Errorf("cannot make cache/file: %v", err)
|
|
} else {
|
|
cases = append(cases, cacheFile)
|
|
}
|
|
|
|
if _, ok := os.LookupEnv("DYNOMITE"); ok {
|
|
if dynomite, err := NewDynomite("localhost:8102", "", ""); err != nil {
|
|
t.Logf("cannot make dynomite: %v", err)
|
|
} else {
|
|
cases = append(cases, dynomite)
|
|
}
|
|
} else {
|
|
t.Log("$DYNOMITE not set. Skipping")
|
|
}
|
|
|
|
if _, ok := os.LookupEnv("REDIS"); ok {
|
|
if redis, err := NewRedis("localhost:8103", "", ""); err != nil {
|
|
t.Logf("cannot make redis: %v", err)
|
|
} else {
|
|
cases = append(cases, redis)
|
|
}
|
|
} else {
|
|
t.Log("$REDIS not set. Skipping")
|
|
}
|
|
|
|
if bolt, err := NewBolt(path.Join(dir, "bolt")); err != nil {
|
|
t.Errorf("cannot make bolt: %v", err)
|
|
} else {
|
|
cases = append(cases, bolt)
|
|
}
|
|
|
|
if yamls, err := NewYaml(path.Join(dir, "yaml")); err != nil {
|
|
t.Errorf("cannot make yaml: %v", err)
|
|
} else {
|
|
cases = append(cases, yamls)
|
|
}
|
|
|
|
if files, err := NewFiles(path.Join(dir, "files")); err != nil {
|
|
t.Errorf("cannot make files: %v", err)
|
|
} else {
|
|
cases = append(cases, files)
|
|
}
|
|
|
|
if leveldb, err := NewLevelDB(path.Join(dir, "leveldb")); err != nil {
|
|
t.Errorf("cannot make leveldb: %v", err)
|
|
} else {
|
|
cases = append(cases, leveldb)
|
|
}
|
|
|
|
if _, ok := os.LookupEnv("MONGO"); ok {
|
|
mongoLN, err := net.Listen("tcp", "localhost:27017")
|
|
if err == nil {
|
|
defer mongoLN.Close()
|
|
go func() {
|
|
for {
|
|
conn, err := mongoLN.Accept()
|
|
if err == nil {
|
|
conn.Close()
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
if mongo1, err := NewMongo("localhost:27017"); err == nil {
|
|
cases = append(cases, mongo1)
|
|
} else if mongo2, err := NewMongo("localhost:27017", "root", "pass"); err == nil {
|
|
cases = append(cases, mongo2)
|
|
} else {
|
|
t.Logf("cannot make mongo: %v", err)
|
|
}
|
|
} else {
|
|
t.Log("$MONGO not set. Skipping")
|
|
}
|
|
|
|
if _, ok := os.LookupEnv("MEMCACHED"); ok {
|
|
if memcache, err := NewMemcache("localhost:11211"); err != nil {
|
|
t.Logf("cannot make memcache: %v", err)
|
|
} else {
|
|
cases = append(cases, memcache)
|
|
}
|
|
} else {
|
|
t.Log("$MEMCACHED not set. Skipping")
|
|
}
|
|
|
|
if _, ok := os.LookupEnv("MEMCACHEDCLUSTER"); ok {
|
|
if memcacheCluster, err := NewMemcacheCluster("localhost:11211"); err != nil {
|
|
t.Logf("cannot make memcacheCluster: %v", err)
|
|
} else {
|
|
cases = append(cases, memcacheCluster)
|
|
}
|
|
} else {
|
|
t.Log("$MEMCACHEDCLUSTER not set. Skipping")
|
|
}
|
|
|
|
if _, ok := os.LookupEnv("MINIO"); ok {
|
|
if minio, err := minio.NewMinio("localhost:9000", "accesskey", "secretkey"); err != nil {
|
|
t.Logf("cannot make minio: %v", err)
|
|
} else {
|
|
cases = append(cases, minio)
|
|
}
|
|
} else {
|
|
t.Log("$MINIO not set. Skipping")
|
|
}
|
|
|
|
rcloneConfigPath := path.Join(t.TempDir(), "rclone.conf."+uuid.New().String())
|
|
if err := ioutil.WriteFile(rcloneConfigPath, []byte(`
|
|
[local]
|
|
type = local
|
|
`), os.ModePerm); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.MkdirAll(path.Join(dir, "rclone"), os.ModePerm); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rclone, err := rclone.NewRClone(rcloneConfigPath, "local:"+path.Join(dir, "rclone"))
|
|
if err != nil {
|
|
t.Errorf("cannot make rclone: %v", err)
|
|
} else {
|
|
cases = append(cases, rclone)
|
|
}
|
|
|
|
validKey := "key"
|
|
validValue := []byte("value")
|
|
|
|
for _, db := range cases {
|
|
t.Run(fmt.Sprintf("%T", db), func(t *testing.T) {
|
|
log.Printf("Trying %T", db)
|
|
t.Logf(" %T: set", db)
|
|
if keys, err := db.List([]string{"ns1", "ns2"}); err != nil || len(keys) > 0 {
|
|
t.Errorf("%T) cannot List() empty: (%T) %+v: %v", db, err, err, keys)
|
|
}
|
|
if keys, err := db.List([]string{path.Join("ns1", "ns2")}); err != nil || len(keys) > 0 {
|
|
t.Errorf("%T) cannot List() empty w/ /: (%T) %+v: %v", db, err, err, keys)
|
|
}
|
|
if err := db.Set(validKey, validValue, "ns1", "ns2"); err != nil {
|
|
t.Errorf("%T) cannot set: %v", db, err)
|
|
}
|
|
t.Logf(" %T: get", db)
|
|
if v, err := db.Get(validKey, "ns1", "ns2"); err != nil {
|
|
t.Errorf("%T) cannot get: %v", db, err)
|
|
} else if !bytes.Equal(v, validValue) {
|
|
t.Errorf("%T) wrong get: %q vs %q", db, v, validValue)
|
|
}
|
|
t.Logf(" %T: list1", db)
|
|
if keys, err := db.List([]string{"ns1", "ns2"}); err != nil || len(keys) < 1 {
|
|
t.Errorf("%T) cannot List(): %v", db, err)
|
|
} else if keys[0] != validKey {
|
|
t.Errorf("%T) List()[0] != %s: %s: want %q, got %q", db, validKey, keys[0], validKey, keys[0])
|
|
}
|
|
t.Logf(" %T: list2", db)
|
|
if keys, err := db.List([]string{"ns1", "ns2"}, validKey[:1]); err != nil || len(keys) < 1 {
|
|
t.Errorf("%T) cannot List(prefix): %v", db, err)
|
|
} else if keys[0] != validKey {
|
|
t.Errorf("%T) List(prefix)[0] != %s: %s", db, validKey, keys[0])
|
|
}
|
|
|
|
if dbstream, ok := db.(DBStream); ok {
|
|
log.Printf("trying %T as DBStream", dbstream)
|
|
raw := "raw"
|
|
if err := dbstream.SetStream("k", strings.NewReader(raw), "ns1", "ns2"); err != nil {
|
|
t.Errorf("%T) cannot setstream: %v", dbstream, err)
|
|
}
|
|
if r, err := dbstream.GetStream("k", "ns1", "ns2"); err != nil {
|
|
t.Errorf("%T) cannot getstream: %v", dbstream, err)
|
|
} else if b, err := ioutil.ReadAll(r); err != nil {
|
|
t.Errorf("%T) cannot readall getstream: %v", dbstream, err)
|
|
} else if string(b) != raw {
|
|
t.Errorf("%T) wrong getstream: %v", dbstream, string(b))
|
|
}
|
|
}
|
|
|
|
t.Logf(" %T: close", db)
|
|
if err := db.Close(); err != nil {
|
|
t.Errorf("cannot close %T: %v", db, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestToFromString(t *testing.T) {
|
|
cases := []struct {
|
|
key string
|
|
t Type
|
|
}{
|
|
{
|
|
key: "map",
|
|
t: MAP,
|
|
},
|
|
{
|
|
key: "redis",
|
|
t: REDIS,
|
|
},
|
|
{
|
|
key: "minio",
|
|
t: MINIO,
|
|
},
|
|
{
|
|
key: "bolt",
|
|
t: BOLT,
|
|
},
|
|
{
|
|
key: "cache",
|
|
t: CACHE,
|
|
},
|
|
{
|
|
key: "leveldb",
|
|
t: LEVELDB,
|
|
},
|
|
{
|
|
key: "memcache",
|
|
t: MEMCACHE,
|
|
},
|
|
{
|
|
key: "memcachecluster",
|
|
t: MEMCACHECLUSTER,
|
|
},
|
|
{
|
|
key: "mongo",
|
|
t: MONGO,
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
if TypeFromString(c.key) != c.t {
|
|
t.Errorf("wrong type for %v: got %v", c.key, TypeFromString(c.key))
|
|
}
|
|
if c.key != c.t.String() {
|
|
t.Errorf("wrong string for %v (%v): got %v", int(c.t), c.key, c.t.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNew(t *testing.T) {
|
|
dir, err := ioutil.TempDir("", "storage_tests_")
|
|
if err != nil {
|
|
t.Fatalf("cannot create temp dir: %v", err)
|
|
return
|
|
}
|
|
defer os.RemoveAll(dir)
|
|
|
|
if b, err := New(BOLT); err == nil {
|
|
t.Errorf("can create bolt without path")
|
|
b.Close()
|
|
}
|
|
|
|
if b, err := New(BOLT, path.Join(dir, "bolt")); err != nil {
|
|
t.Errorf("cannot create bolt with path")
|
|
} else {
|
|
b.Close()
|
|
}
|
|
}
|
|
|
|
func recoverDeferred(c Type, t *testing.T, wg *sync.WaitGroup) {
|
|
if err := recover(); err != nil {
|
|
log.Printf("recover deferre fail: %s", c)
|
|
t.Errorf("[%s] panic: %v", c, err)
|
|
} else {
|
|
log.Printf("recover deferre ok: %s", c)
|
|
}
|
|
defer wg.Done()
|
|
}
|