local/storage
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: local:82d9519e72db27dc844b5f238c692f38455f85b1
Branches Tags
local:master
..
compare: local:2d5cd62d650e55bc067897ef16cfbf63786bcc9a
Branches Tags
local:master

10 Commits
82d9519e72 ... 2d5cd62d65

Author SHA1 Message Date
bel
2d5cd62d65 bolt put nil is del 2023-04-12 15:26:11 -06:00
bel
01db198192 add thing 2023-04-12 15:23:44 -06:00
Bel LaPointe
db39d7b02e to gogs.inhome 2023-04-10 10:21:02 -06:00
Bel LaPointe
cb219e269d no more memcache, go broke for mac 18.1 2022-05-10 06:25:59 -06:00
Bel LaPointe
9914252040 add install.sh 2022-05-10 06:22:05 -06:00
Bel LaPointe
87b4d28c71 fix yaml human not friendly 2021-12-15 07:48:46 -07:00
Bel LaPointe
aef66625c0 human readable yaml 2021-12-15 07:40:36 -07:00
Bel LaPointe
44ce475100 add yaml 2021-12-15 07:33:50 -07:00
Bel LaPointe
bc8a0d82df yaml namespaces 2021-12-15 06:53:28 -07:00
Bel LaPointe
0121c426f7 whoops double ext not backwards compatible 2021-09-28 13:53:15 -06:00
25 changed files with 412 additions and 286 deletions
Expand all files Collapse all files

5
bolt.go
View File

@@ -1,7 +1,7 @@
package storage
import (
"local/storage/resolve"
"gogs.inhome.blapointe.com/local/storage/resolve"
"github.com/boltdb/bolt"
)
@@ -60,6 +60,9 @@ func (b *Bolt) Set(key string, value []byte, ns ...string) error {
if err != nil {
return err
}
if value == nil {
return bkt.Delete(key)
}
return bkt.Put([]byte(key), value)
})
}

2
cache.go
View File

@@ -1,7 +1,7 @@
package storage
import (
"local/storage/resolve"
"gogs.inhome.blapointe.com/local/storage/resolve"
"os"
"path"
"strings"

10
cli/cli.go
View File

@@ -3,12 +3,13 @@ package main
import (
"bufio"
"fmt"
"local/args"
"local/storage"
"log"
"os"
"strings"
"time"
"gogs.inhome.blapointe.com/local/args"
"gogs.inhome.blapointe.com/local/storage"
)
func main() {
@@ -17,7 +18,7 @@ func main() {
as.Append(args.STRING, "db", "type of store", "dynomite")
as.Append(args.STRING, "user", "user of store", "")
as.Append(args.STRING, "pass", "pass of store", "")
as.Append(args.STRING, "do", "[get set]", "get")
as.Append(args.STRING, "do", "[get set del list]", "get")
as.Append(args.STRING, "k", "key", "key")
as.Append(args.STRING, "v", "value", "value")
as.Append(args.STRING, "ns", "namespace", "")
@@ -52,6 +53,9 @@ func main() {
case "set":
ns := strings.Split(as.Get("ns").GetString(), " ")
err = db.Set(as.Get("k").GetString(), []byte(as.Get("v").GetString()), ns...)
case "del":
ns := strings.Split(as.Get("ns").GetString(), " ")
err = db.Set(as.Get("k").GetString(), nil, ns...)
case "cli":
reader := bufio.NewReader(os.Stdin)
for {

10
cli/go.mod
View File

@@ -1,12 +1,10 @@
module local/storage/cli
module gogs.inhome.blapointe.com/local/storage/cli
go 1.16
replace local/storage => ../
replace local/args => ../../args
replace gogs.inhome.blapointe.com/local/storage => ../
require (
local/args v0.0.0-00010101000000-000000000000
local/storage v0.0.0-00010101000000-000000000000
gogs.inhome.blapointe.com/local/args v0.0.0-20230410154220-44370f257b34
gogs.inhome.blapointe.com/local/storage v0.0.0-20230410162102-db39d7b02e29
)

17
cli/go.sum
View File

@@ -5,8 +5,6 @@ github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9a
github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 h1:6X8iB881g299aNEv6KXrcjL31iLOH7yA6NXoQX+MbDg=
github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
github.com/a8m/tree v0.0.0-20180321023834-3cf936ce15d6/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=
@@ -18,12 +16,6 @@ github.com/aws/aws-sdk-go v1.15.81/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3A
github.com/billziss-gh/cgofuse v1.1.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0=
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
github.com/buraksezer/consistent v0.9.0 h1:Zfs6bX62wbP3QlbPGKUhqDw7SmNkOzY5bHZIYXYpR5g=
github.com/buraksezer/consistent v0.9.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/coreos/bbolt v0.0.0-20180318001526-af9db2027c98/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -164,8 +156,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -193,6 +183,10 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul
github.com/yunify/qingstor-sdk-go v2.2.15+incompatible/go.mod h1:w6wqLDQ5bBTzxGJ55581UrSwLrsTAsdo9N6yX/8d9RY=
go.mongodb.org/mongo-driver v1.7.2 h1:pFttQyIiJUHEn50YfZgC9ECjITMT44oiN36uArf/OFg=
go.mongodb.org/mongo-driver v1.7.2/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
gogs.inhome.blapointe.com/local/args v0.0.0-20230410154220-44370f257b34 h1:0tuX5dfOksiOQD1vbJjVNVTVxTTIng7UrUdSLF5T+Ao=
gogs.inhome.blapointe.com/local/args v0.0.0-20230410154220-44370f257b34/go.mod h1:YG9n3Clg7683ohkVnJK2hdX8bBS9EojIsd1qPZumX0Y=
gogs.inhome.blapointe.com/local/logb v0.0.0-20230410154319-880efa39d871 h1:cMGPiwvK/QGg4TfW8VasO6SsS/O7UQmwyKDErV/ozoA=
gogs.inhome.blapointe.com/local/logb v0.0.0-20230410154319-880efa39d871/go.mod h1:E0pLNvMLzY0Kth1W078y+06z1AUyVMWnChMpRFf4w2Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -224,8 +218,9 @@ golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2 h1:T5DasATyLQfmbTpfEXx/IOL9vfjzW6up+ZDkmHvIf2s=
golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

2
cli/install_cli.sh Normal file
View File

@@ -0,0 +1,2 @@
#! /bin/bash
exec go build -o $HOME/Go/bin/local-storage-cli cli.go

6
cli/migr8.go
View File

@@ -2,9 +2,9 @@ package main
import (
"fmt"
"local/args"
"local/storage"
"local/storage/resolve"
"gogs.inhome.blapointe.com/local/args"
"gogs.inhome.blapointe.com/local/storage"
"gogs.inhome.blapointe.com/local/storage/resolve"
"log"
)

10
db.go
View File

@@ -3,8 +3,8 @@ package storage
import (
"fmt"
"io"
"local/storage/minio"
"local/storage/rclone"
"gogs.inhome.blapointe.com/local/storage/minio"
"gogs.inhome.blapointe.com/local/storage/rclone"
)
type DBStream interface {
@@ -42,6 +42,8 @@ func New(key Type, params ...string) (db DB, err error) {
db, err = rclone.NewRClone(params[0], params[1])
case FILES:
db, err = NewFiles(params[0])
case YAML:
db, err = NewYaml(params[0])
case BOLT:
db, err = NewBolt(params[0])
case MINIO:
@@ -50,10 +52,6 @@ func New(key Type, params ...string) (db DB, err error) {
db, err = NewCache(params...)
case LEVELDB:
db, err = NewLevelDB(params[0])
case MEMCACHE:
db, err = NewMemcache(params[0], params[1:]...)
case MEMCACHECLUSTER:
db, err = NewMemcacheCluster(params[0], params[1:]...)
case MONGO:
db, err = NewMongo(params[0], params[1:]...)
}

52
db_test.go
View File

@@ -4,9 +4,9 @@ import (
"bytes"
"fmt"
"io/ioutil"
"local/storage/minio"
"local/storage/rclone"
"local/storage/resolve"
"gogs.inhome.blapointe.com/local/storage/minio"
"gogs.inhome.blapointe.com/local/storage/rclone"
"gogs.inhome.blapointe.com/local/storage/resolve"
"log"
"net"
"os"
@@ -14,6 +14,7 @@ import (
"strings"
"sync"
"testing"
"time"
"github.com/google/uuid"
)
@@ -103,6 +104,12 @@ func TestImplementations(t *testing.T) {
cases = append(cases, bolt)
}
if yamls, err := NewYaml(path.Join(dir, "yaml")); err != nil {
t.Errorf("cannot make yaml: %v", err)
} else {
cases = append(cases, yamls)
}
if files, err := NewFiles(path.Join(dir, "files")); err != nil {
t.Errorf("cannot make files: %v", err)
} else {
@@ -139,26 +146,6 @@ func TestImplementations(t *testing.T) {
t.Log("$MONGO not set. Skipping")
}
if _, ok := os.LookupEnv("MEMCACHED"); ok {
if memcache, err := NewMemcache("localhost:11211"); err != nil {
t.Logf("cannot make memcache: %v", err)
} else {
cases = append(cases, memcache)
}
} else {
t.Log("$MEMCACHED not set. Skipping")
}
if _, ok := os.LookupEnv("MEMCACHEDCLUSTER"); ok {
if memcacheCluster, err := NewMemcacheCluster("localhost:11211"); err != nil {
t.Logf("cannot make memcacheCluster: %v", err)
} else {
cases = append(cases, memcacheCluster)
}
} else {
t.Log("$MEMCACHEDCLUSTER not set. Skipping")
}
if _, ok := os.LookupEnv("MINIO"); ok {
if minio, err := minio.NewMinio("localhost:9000", "accesskey", "secretkey"); err != nil {
t.Logf("cannot make minio: %v", err)
@@ -192,19 +179,20 @@ type = local
for _, db := range cases {
t.Run(fmt.Sprintf("%T", db), func(t *testing.T) {
log.Printf("Trying %T", db)
t.Logf(" %T: set", db)
t.Logf(" %T: list @[ns1, ns2] against empty", db)
if keys, err := db.List([]string{"ns1", "ns2"}); err != nil || len(keys) > 0 {
t.Errorf("%T) cannot List() empty: (%T) %+v: %v", db, err, err, keys)
}
if keys, err := db.List([]string{path.Join("ns1", "ns2")}); err != nil || len(keys) > 0 {
t.Errorf("%T) cannot List() empty w/ /: (%T) %+v: %v", db, err, err, keys)
}
t.Logf(" %T: set %s @[ns1, ns2]", db, validKey)
if err := db.Set(validKey, validValue, "ns1", "ns2"); err != nil {
t.Errorf("%T) cannot set: %v", db, err)
}
t.Logf(" %T: get", db)
t.Logf(" %T: db: %+v", db, db)
t.Logf(" %T: get %s @[ns1, ns2]", db, validKey)
if v, err := db.Get(validKey, "ns1", "ns2"); err != nil {
t.Errorf("%T) cannot get: %v", db, err)
log.Printf("%T) cannot get: %v (%+v)", db, err, db)
time.Sleep(time.Second * 10)
} else if !bytes.Equal(v, validValue) {
t.Errorf("%T) wrong get: %q vs %q", db, v, validValue)
}
@@ -273,14 +261,6 @@ func TestToFromString(t *testing.T) {
key: "leveldb",
t: LEVELDB,
},
{
key: "memcache",
t: MEMCACHE,
},
{
key: "memcachecluster",
t: MEMCACHECLUSTER,
},
{
key: "mongo",
t: MONGO,

2
errors.go
View File

@@ -1,6 +1,6 @@
package storage
import "local/storage/resolve"
import "gogs.inhome.blapointe.com/local/storage/resolve"
var ErrNotFound = resolve.ErrNotFound
var ErrNotImpl = resolve.ErrNotImpl

12
files.go
View File

@@ -5,8 +5,8 @@ import (
"errors"
"io"
"io/ioutil"
"local/storage/resolve"
"log"
"gogs.inhome.blapointe.com/local/logb"
"gogs.inhome.blapointe.com/local/storage/resolve"
"os"
"path"
"path/filepath"
@@ -76,10 +76,10 @@ func (b *Files) Get(key string, ns ...string) ([]byte, error) {
func (b *Files) GetStream(key string, ns ...string) (io.Reader, error) {
namespace := resolve.Namespace(ns)
key += fileExt
path := path.Join(b.root, namespace, key)
r, err := os.Open(path + fileExt)
if os.IsNotExist(err) {
logb.Warnf("does not exist: %q + %q from key=%q ns=%v namespace=%q, trying %q instead", path, fileExt, key, ns, namespace, path)
r, err = os.Open(path)
}
if os.IsNotExist(err) {
@@ -121,18 +121,18 @@ func (b *Files) SetStream(key string, r io.Reader, ns ...string) error {
return err
}
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
log.Printf("failed mkdir: %v", err)
logb.Warnf("failed mkdir: %v", err)
return err
}
f, err := os.Create(path)
if err != nil {
log.Printf("failed create: %v: path=%q, dir=%q, ns=%q", err, path, dir, namespace)
logb.Warnf("failed create: %v: path=%q, dir=%q, ns=%q", err, path, dir, namespace)
return err
}
defer f.Close()
_, err = io.Copy(f, r)
if err != nil {
log.Printf("failed copy: %v", err)
logb.Warnf("failed copy: %v", err)
return err
}
return nil

8
go.mod
View File

@@ -1,12 +1,9 @@
module local/storage
module gogs.inhome.blapointe.com/local/storage
go 1.16
require (
github.com/boltdb/bolt v1.3.1
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
github.com/buraksezer/consistent v0.9.0
github.com/cespare/xxhash v1.1.0
github.com/gomodule/redigo v1.8.5
github.com/google/uuid v1.3.0
github.com/minio/minio-go/v6 v6.0.57
@@ -14,4 +11,7 @@ require (
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/syndtr/goleveldb v1.0.0
go.mongodb.org/mongo-driver v1.7.2
gogs.inhome.blapointe.com/local/logb v0.0.0-20230410154319-880efa39d871
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
gopkg.in/yaml.v2 v2.2.8
)

18
go.sum
View File

@@ -5,8 +5,6 @@ github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9a
github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 h1:6X8iB881g299aNEv6KXrcjL31iLOH7yA6NXoQX+MbDg=
github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
github.com/a8m/tree v0.0.0-20180321023834-3cf936ce15d6/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=
@@ -18,12 +16,6 @@ github.com/aws/aws-sdk-go v1.15.81/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3A
github.com/billziss-gh/cgofuse v1.1.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0=
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
github.com/buraksezer/consistent v0.9.0 h1:Zfs6bX62wbP3QlbPGKUhqDw7SmNkOzY5bHZIYXYpR5g=
github.com/buraksezer/consistent v0.9.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/coreos/bbolt v0.0.0-20180318001526-af9db2027c98/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -100,8 +92,10 @@ github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
@@ -162,8 +156,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -191,6 +183,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul
github.com/yunify/qingstor-sdk-go v2.2.15+incompatible/go.mod h1:w6wqLDQ5bBTzxGJ55581UrSwLrsTAsdo9N6yX/8d9RY=
go.mongodb.org/mongo-driver v1.7.2 h1:pFttQyIiJUHEn50YfZgC9ECjITMT44oiN36uArf/OFg=
go.mongodb.org/mongo-driver v1.7.2/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
gogs.inhome.blapointe.com/local/logb v0.0.0-20230410154319-880efa39d871 h1:cMGPiwvK/QGg4TfW8VasO6SsS/O7UQmwyKDErV/ozoA=
gogs.inhome.blapointe.com/local/logb v0.0.0-20230410154319-880efa39d871/go.mod h1:E0pLNvMLzY0Kth1W078y+06z1AUyVMWnChMpRFf4w2Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -222,8 +216,9 @@ golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2 h1:T5DasATyLQfmbTpfEXx/IOL9vfjzW6up+ZDkmHvIf2s=
golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -243,6 +238,7 @@ google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQ
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=

2
leveldb.go
View File

@@ -1,7 +1,7 @@
package storage
import (
"local/storage/resolve"
"gogs.inhome.blapointe.com/local/storage/resolve"
"path"
"strconv"
"strings"

2
map.go
View File

@@ -2,7 +2,7 @@ package storage
import (
"fmt"
"local/storage/resolve"
"gogs.inhome.blapointe.com/local/storage/resolve"
"sync"
)

78
memcache.go
View File

@@ -1,78 +0,0 @@
package storage
import (
"errors"
"local/storage/resolve"
"net"
"path"
"github.com/bradfitz/gomemcache/memcache"
)
type Memcache struct {
db *memcache.Client
}
type netAddr struct {
network string
addr string
}
func (a *netAddr) Network() string {
return a.network
}
func (a *netAddr) String() string {
return a.addr
}
func NewMemcache(addr string, addrs ...string) (*Memcache, error) {
for i := len(addrs) - 1; i >= 0; i-- {
if len(addrs[i]) == 0 {
addrs = append(addrs[:i], addrs[i+1:]...)
}
}
ss := &memcache.ServerList{}
if err := ss.SetServers(append([]string{addr}, addrs...)...); err != nil {
return nil, err
}
if err := ss.Each(func(addr net.Addr) error {
conn, err := net.Dial("tcp", addr.String())
if err != nil {
return err
}
return conn.Close()
}); err != nil {
return nil, err
}
db := memcache.NewFromSelector(ss)
return &Memcache{db: db}, nil
}
func (mc *Memcache) List(ns []string, limits ...string) ([]string, error) {
return nil, errors.New("not impl")
}
func (mc *Memcache) Get(key string, ns ...string) ([]byte, error) {
namespace := resolve.Namespace(ns)
v, err := mc.db.Get(path.Join(namespace, key))
if err != nil {
return nil, err
}
if v == nil {
return nil, ErrNotFound
}
return v.Value, err
}
func (mc *Memcache) Set(key string, value []byte, ns ...string) error {
namespace := resolve.Namespace(ns)
return mc.db.Set(&memcache.Item{
Key: path.Join(namespace, key),
Value: value,
})
}
func (mc *Memcache) Close() error {
return mc.db.FlushAll()
}

94
memcachecluster.go
View File

@@ -1,94 +0,0 @@
package storage
import (
"errors"
"local/storage/resolve"
"net"
"path"
"github.com/bradfitz/gomemcache/memcache"
"github.com/buraksezer/consistent"
"github.com/cespare/xxhash"
)
type MemcacheCluster struct {
db *memcache.Client
}
type serverSelector struct {
hash *consistent.Consistent
}
func (ss *serverSelector) PickServer(key string) (net.Addr, error) {
return &netAddr{
network: "tcp",
addr: ss.hash.LocateKey([]byte(key)).String(),
}, nil
}
func (ss *serverSelector) Each(each func(net.Addr) error) error {
for _, member := range ss.hash.GetMembers() {
if err := each(&netAddr{
network: "tcp",
addr: member.String(),
}); err != nil {
return err
}
}
return nil
}
type hasher struct{}
func (h hasher) Sum64(data []byte) uint64 {
return xxhash.Sum64(data)
}
func NewMemcacheCluster(addr string, addrs ...string) (*MemcacheCluster, error) {
cfg := consistent.Config{
PartitionCount: 71,
ReplicationFactor: 20,
Load: 1.25,
Hasher: hasher{},
}
hash := consistent.New(nil, cfg)
for _, addr := range append(addrs, addr) {
hash.Add(&netAddr{addr: addr})
}
ss := &serverSelector{
hash: hash,
}
if err := ss.Each(func(addr net.Addr) error {
conn, err := net.Dial("tcp", addr.String())
if err != nil {
return err
}
return conn.Close()
}); err != nil {
return nil, err
}
db := memcache.NewFromSelector(ss)
return &MemcacheCluster{db: db}, nil
}
func (mc *MemcacheCluster) List(ns []string, limits ...string) ([]string, error) {
return nil, errors.New("not impl")
}
func (mc *MemcacheCluster) Get(key string, ns ...string) ([]byte, error) {
namespace := resolve.Namespace(ns)
v, err := mc.db.Get(path.Join(namespace, key))
return v.Value, err
}
func (mc *MemcacheCluster) Set(key string, value []byte, ns ...string) error {
namespace := resolve.Namespace(ns)
return mc.db.Set(&memcache.Item{
Key: path.Join(namespace, key),
Value: value,
})
}
func (mc *MemcacheCluster) Close() error {
return mc.db.FlushAll()
}

2
minio.go
View File

@@ -1,5 +1,5 @@
package storage
import "local/storage/minio"
import "gogs.inhome.blapointe.com/local/storage/minio"
type Minio minio.Minio

2
minio/minio.go
View File

@@ -4,7 +4,7 @@ import (
"bytes"
"fmt"
"io/ioutil"
"local/storage/resolve"
"gogs.inhome.blapointe.com/local/storage/resolve"
"net/url"
"sort"
"strings"

2
mongo.go
View File

@@ -3,7 +3,7 @@ package storage
import (
"context"
"errors"
"local/storage/resolve"
"gogs.inhome.blapointe.com/local/storage/resolve"
"strings"
"time"

2
rclone/rclone.go
View File

@@ -5,7 +5,7 @@ import (
"errors"
"io"
"io/ioutil"
"local/storage/resolve"
"gogs.inhome.blapointe.com/local/storage/resolve"
"os"
"path"
"time"

2
redis.go
View File

@@ -3,7 +3,7 @@ package storage
import (
"errors"
"fmt"
"local/storage/resolve"
"gogs.inhome.blapointe.com/local/storage/resolve"
"time"
"github.com/gomodule/redigo/redis"

33
type.go
View File

@@ -7,20 +7,19 @@ import (
type Type int
const (
MAP = Type(iota)
REDIS = Type(iota)
DYNOMITE = Type(iota)
BOLT = Type(iota)
FILES = Type(iota)
COCKROACH = Type(iota)
CACHE = Type(iota)
LEVELDB = Type(iota)
MEMCACHE = Type(iota)
MEMCACHECLUSTER = Type(iota)
MONGO = Type(iota)
MINIO = Type(iota)
RCLONE = Type(iota)
MAPSTREAM = Type(iota)
MAP = Type(iota)
REDIS = Type(iota)
DYNOMITE = Type(iota)
BOLT = Type(iota)
FILES = Type(iota)
COCKROACH = Type(iota)
CACHE = Type(iota)
LEVELDB = Type(iota)
MONGO = Type(iota)
MINIO = Type(iota)
RCLONE = Type(iota)
MAPSTREAM = Type(iota)
YAML = Type(iota)
)
func (t Type) String() string {
@@ -37,6 +36,8 @@ func (t Type) String() string {
return "rclone"
case COCKROACH:
return "cockroach"
case YAML:
return "yaml"
case FILES:
return "files"
case BOLT:
@@ -47,10 +48,6 @@ func (t Type) String() string {
return "cache"
case LEVELDB:
return "leveldb"
case MEMCACHE:
return "memcache"
case MEMCACHECLUSTER:
return "memcachecluster"
case MONGO:
return "mongo"
}

263
yaml.go Executable file
View File

@@ -0,0 +1,263 @@
package storage
import (
"bytes"
"encoding/base64"
"errors"
"fmt"
"io"
"io/ioutil"
"gogs.inhome.blapointe.com/local/storage/resolve"
"os"
"path"
"path/filepath"
"strings"
"unicode"
yaml "gopkg.in/yaml.v2"
)
const (
yamlExt = ".yaml"
)
type Yaml struct {
path string
}
func NewYaml(p string) (*Yaml, error) {
_, err := os.Stat(path.Dir(p))
if err != nil {
return nil, err
}
p, err = filepath.Abs(p)
if err != nil {
return nil, err
}
return &Yaml{
path: p,
}, os.MkdirAll(path.Dir(p), os.ModePerm)
}
func (y *Yaml) Namespaces() ([][]string, error) {
m, err := y.getMap()
if err != nil {
return nil, err
}
return keysDFS(m)
}
func (y *Yaml) List(ns []string, limits ...string) ([]string, error) {
namespace := resolve.Namespace(ns)
m, err := y.getMap(namespace)
if err != nil {
return nil, err
}
limits = resolve.Limits(limits)
ks := make([]string, 0, len(m))
for k := range m {
if k >= limits[0] && k <= limits[1] {
ks = append(ks, k)
}
}
return ks, nil
}
func (y *Yaml) Get(key string, ns ...string) ([]byte, error) {
r, err := y.GetStream(key, ns...)
if err != nil {
return nil, err
}
return ioutil.ReadAll(r)
}
func (y *Yaml) GetStream(key string, ns ...string) (io.Reader, error) {
namespace := resolve.Namespace(ns)
m, err := y.getMap(namespace)
if err != nil {
return nil, err
}
v, ok := m[key]
if !ok {
return nil, ErrNotFound
}
s, ok := v.(string)
if !ok {
return nil, ErrNotFound
}
if strings.HasPrefix(s, "b64://") {
b, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(s, "b64://"))
return bytes.NewReader(b), err
}
return strings.NewReader(s), nil
}
func (y *Yaml) Set(key string, value []byte, ns ...string) error {
r := bytes.NewReader(value)
if value == nil {
return y.Del(key, ns...)
}
return y.SetStream(key, r, ns...)
}
func (y *Yaml) Del(key string, ns ...string) error {
return y.SetStream(key, nil, ns...)
}
func isASCII(s string) bool {
for _, c := range s {
if c > unicode.MaxASCII {
return false
}
}
return true
}
func (y *Yaml) SetStream(key string, r io.Reader, ns ...string) error {
namespace := resolve.Namespace(ns)
var v interface{} = nil
if r != nil {
b, err := ioutil.ReadAll(r)
if err != nil {
return err
}
if isASCII(string(b)) {
v = string(b)
} else {
v = "b64://" + base64.StdEncoding.EncodeToString(b)
}
}
m, err := y.getMap()
if err != nil {
return err
}
if err := setInMap(m, []string{namespace}, key, v); err != nil {
return err
}
return y.setMap(m)
}
func (y *Yaml) Close() error {
return nil
}
func (y *Yaml) getMap(keys ...string) (map[string]interface{}, error) {
b, err := y.get()
if err != nil {
return nil, err
}
var mBad map[interface{}]interface{}
if err := yaml.Unmarshal(b, &mBad); err != nil {
return nil, err
}
m, err := mbadToM(mBad)
if err != nil {
return nil, err
}
if m == nil {
m = map[string]interface{}{}
}
for _, k := range keys {
subv, ok := m[k]
if !ok {
subv = map[string]interface{}{}
m[k] = subv
}
subm, ok := subv.(map[string]interface{})
if !ok {
return nil, ErrNotFound
}
m = subm
}
return m, err
}
func (y *Yaml) setMap(m map[string]interface{}) error {
b, err := yaml.Marshal(m)
if err != nil {
return err
}
return y.set(b)
}
func setInMap(m map[string]interface{}, keys []string, key string, v interface{}) error {
if len(keys) == 0 {
m[key] = v
if v == nil {
delete(m, key)
}
return nil
}
subv, ok := m[keys[0]]
if !ok {
subv = map[string]interface{}{}
}
subm, ok := subv.(map[string]interface{})
if !ok {
return errors.New("clobber")
}
if err := setInMap(subm, keys[1:], key, v); err != nil {
return err
}
m[keys[0]] = subm
return nil
}
func (y *Yaml) get() ([]byte, error) {
b, err := ioutil.ReadFile(y.path)
if os.IsNotExist(err) {
return []byte{}, nil
}
return b, err
}
func (y *Yaml) set(b []byte) error {
return ioutil.WriteFile(y.path, b, os.ModePerm)
}
func keysDFS(m map[string]interface{}) ([][]string, error) {
keys, _, err := _keysDFS(m)
return keys, err
}
func _keysDFS(m map[string]interface{}) ([][]string, bool, error) {
keys := make([][]string, 0)
hasNonMaps := false
for k, v := range m {
if subm, ok := v.(map[string]interface{}); ok {
subkeys, hasElements, err := _keysDFS(subm)
if err != nil {
return nil, false, err
}
if hasElements {
subkeys = append(subkeys, []string{})
}
for i := range subkeys {
subkeys[i] = append([]string{k}, subkeys[i]...)
keys = append(keys, subkeys[i])
}
} else {
hasNonMaps = true
}
}
return keys, hasNonMaps, nil
}
func mbadToM(mBad map[interface{}]interface{}) (map[string]interface{}, error) {
m := map[string]interface{}{}
for k, v := range mBad {
s, ok := k.(string)
if !ok {
s = fmt.Sprint(k)
}
m[s] = v
if m2, ok := v.(map[interface{}]interface{}); ok {
v2, err := mbadToM(m2)
if err != nil {
return nil, err
}
m[s] = v2
}
}
return m, nil
}

62
yaml_test.go Normal file
View File

@@ -0,0 +1,62 @@
package storage
import (
"fmt"
"testing"
)
func TestKeysDFS(t *testing.T) {
cases := map[string]struct {
input map[string]interface{}
want [][]string
}{
"empty": {
input: map[string]interface{}{},
want: [][]string{},
},
"top level non map keys": {
input: map[string]interface{}{"a": "b", "c": "d"},
want: [][]string{},
},
"top level non map keys and map key": {
input: map[string]interface{}{"a": "b", "c": "d", "e": map[string]interface{}{"f": "g"}},
want: [][]string{[]string{"e"}},
},
"top level non map keys and map key and nested, ignore empty nested": {
input: map[string]interface{}{
"a": "b",
"c": "d",
"e": map[string]interface{}{
"f": map[string]interface{}{"g": "h"},
},
"i": map[string]interface{}{},
"j": map[string]interface{}{"k": "l"},
},
want: [][]string{[]string{"e", "f"}, []string{"j"}},
},
}
for name, d := range cases {
c := d
t.Run(name, func(t *testing.T) {
got, err := keysDFS(c.input)
if err != nil {
t.Fatal(err)
}
for j := range c.want {
found := false
for i := range got {
if fmt.Sprint(got[i]) == fmt.Sprint(c.want[j]) {
found = true
}
}
if !found {
t.Errorf("want %+v among %+v", c.want[j], got)
}
}
if fmt.Sprintf("%+v", got) != fmt.Sprintf("%+v", c.want) {
t.Fatalf("want: %+v\ngot: %+v", c.want, got)
}
})
}
}