From a2b66a03db599c7bdd08bcd931578ccbfacbd641 Mon Sep 17 00:00:00 2001 From: bel Date: Sun, 5 Nov 2023 09:29:46 -0700 Subject: [PATCH] impl filetree driver --- replicator/driver.go | 2 +- replicator/driver_test.go | 32 +++++++++ replicator/filetree.go | 139 ++++++++++++++++++++++++++++++++++-- replicator/filetree_test.go | 42 +++++++++++ 4 files changed, 209 insertions(+), 6 deletions(-) diff --git a/replicator/driver.go b/replicator/driver.go index 85ed8a3..1c6e3e2 100644 --- a/replicator/driver.go +++ b/replicator/driver.go @@ -8,7 +8,7 @@ import ( type ( Driver interface { - Keys(context.Context) (chan Key, error) + Keys(context.Context) (chan Key, *error) Get(context.Context, Key) (Value, error) Set(context.Context, Key, Value) error Del(context.Context, Key) error diff --git a/replicator/driver_test.go b/replicator/driver_test.go index 4479015..b91dada 100644 --- a/replicator/driver_test.go +++ b/replicator/driver_test.go @@ -48,4 +48,36 @@ func testDriver(t *testing.T, d Driver) { t.Errorf("failed to clean up: %v", err) } }) + + t.Run("keys of nothing", func(t *testing.T) { + ch, err := d.Keys(ctx) + for got := range ch { + t.Error("expected nothing but got", got) + } + if *err != nil { + t.Error(*err) + } + }) + + t.Run("keys of one key", func(t *testing.T) { + if err := d.Set(ctx, key, value); err != nil { + t.Fatal(err) + } + defer d.Del(ctx, key) + + ch, err := d.Keys(ctx) + n := 0 + for got := range ch { + n += 1 + if got != key { + t.Error(got) + } + } + if n == 0 { + t.Error("expected to find at least 1 key") + } + if *err != nil { + t.Error(*err) + } + }) } diff --git a/replicator/filetree.go b/replicator/filetree.go index 5a71cda..4fa34b2 100644 --- a/replicator/filetree.go +++ b/replicator/filetree.go @@ -2,9 +2,17 @@ package replicator import ( "context" + "fmt" "io" + "io/fs" + "io/ioutil" "net/url" + "os" "path" + "path/filepath" + "strconv" + "strings" + "time" ) type FileTree string @@ -17,18 +25,139 @@ func NewFileTree(spec url.URL) (FileTree, error) { return FileTree(p), nil } -func (tree FileTree) Keys(ctx context.Context) (chan Key, error) { - return nil, io.EOF +func (tree FileTree) Keys(ctx context.Context) (chan Key, *error) { + result := make(chan Key) + var final error + go func() { + defer close(result) + if err := filepath.Walk(path.Dir(tree.realpath(Key{})), func(p string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return ctx.Err() + } + key := tree.toKey(p) + select { + case result <- key: + case <-ctx.Done(): + } + return ctx.Err() + }); err != nil && !os.IsNotExist(err) { + final = err + } + }() + return result, &final } func (tree FileTree) Get(ctx context.Context, key Key) (Value, error) { - return Value{}, io.EOF + version, err := tree.getVersion(key) + if version == nil { + return Value{}, err + } + + f, err := os.Open(tree.realpath(key)) + if err != nil { + return Value{}, err + } + defer f.Close() + b, _ := io.ReadAll(f) + + return Value{ + Value: b, + Version: version, + }, nil } func (tree FileTree) Set(ctx context.Context, key Key, value Value) error { - return io.EOF + version, err := tree.fromVersion(value.Version) + if err != nil { + return err + } + + f2, err := ioutil.TempFile(os.TempDir(), "*.*") + if err != nil { + return err + } + defer os.Remove(f2.Name()) + + if n, err := f2.Write(value.Value); err != nil { + return err + } else if n != len(value.Value) { + return fmt.Errorf("truncated write") + } + + if err := f2.Close(); err != nil { + return err + } + + if err := os.Chtimes(f2.Name(), time.Time{}, version); err != nil { + return err + } + + os.MkdirAll(path.Dir(tree.realpath(key)), os.ModePerm) + return os.Rename(f2.Name(), tree.realpath(key)) } func (tree FileTree) Del(ctx context.Context, key Key) error { - return io.EOF + if _, err := os.Stat(tree.realpath(key)); os.IsNotExist(err) { + return nil + } else if err != nil { + return err + } + return os.Remove(tree.realpath(key)) +} + +func (tree FileTree) realpath(key Key) string { + namespace := "" + tmp := path.Clean("/" + key.Namespace) + for path.Dir(tmp) != tmp { + _, base := path.Split(tmp) + namespace = path.Join(base+".d", namespace) + tmp = path.Dir(tmp) + } + return path.Join(string(tree), namespace, key.Key+".bin") +} + +func (tree FileTree) getVersion(key Key) ([]byte, error) { + info, err := os.Stat(tree.realpath(key)) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(info.ModTime().UnixNano(), 10)), nil +} + +func (tree FileTree) fromVersion(v []byte) (time.Time, error) { + if len(v) == 0 { + return time.Time{}, nil + } + n, err := strconv.ParseInt(string(v), 10, 64) + if err != nil { + return time.Time{}, err + } + return time.Unix(0, n), nil +} + +func (tree FileTree) toKey(p string) Key { + p = path.Clean(p) + p = strings.TrimPrefix(p, path.Clean(string(tree))) + p = strings.TrimPrefix(p, "/") + p = strings.TrimSuffix(p, ".bin") + p = strings.ReplaceAll(p, ".d/", "/") + namespace := path.Dir(p) + if namespace == "/" || namespace == "." { + namespace = "" + } + key := path.Base(p) + if key == "" { + key = namespace + namespace = "" + } + return Key{ + Namespace: namespace, + Key: key, + } } diff --git a/replicator/filetree_test.go b/replicator/filetree_test.go index b3cd02c..a8bb9f8 100644 --- a/replicator/filetree_test.go +++ b/replicator/filetree_test.go @@ -18,3 +18,45 @@ func TestFileTree(t *testing.T) { testDriver(t, tree) } + +func TestFileTreeRealpath(t *testing.T) { + root := "/root" + base := "base" + cases := map[string]string{ + "a/b": "/root/a.d/b.d/base.bin", + "a": "/root/a.d/base.bin", + "": "/root/base.bin", + } + + for given, wantd := range cases { + want := wantd + t.Run(given, func(t *testing.T) { + key := Key{Key: base, Namespace: given} + got := FileTree(root).realpath(key) + if got != want { + t.Error(got) + } + }) + } +} + +func TestFileTreeToKey(t *testing.T) { + root := "/root" + cases := map[string]Key{ + "/root/a.d/b.bin": {Namespace: "a", Key: "b"}, + "//root/a.d/b.bin": {Namespace: "a", Key: "b"}, + "/root/b.bin": {Namespace: "", Key: "b"}, + "/root/c.d/a.d/b.bin": {Namespace: "c/a", Key: "b"}, + "/root/c/a.d/b.bin": {Namespace: "c/a", Key: "b"}, + } + + for given, wantd := range cases { + want := wantd + t.Run(given, func(t *testing.T) { + got := FileTree(root).toKey(given) + if got != want { + t.Error(got) + } + }) + } +}