From 6baa938b22c1a9c55dd610a0a8b7c11e1f2ade59 Mon Sep 17 00:00:00 2001 From: Bel LaPointe Date: Mon, 18 May 2020 14:56:33 -0600 Subject: [PATCH] whoops --- .gitignore | 1 - lastn/lastn.go | 164 ++++++++++++++++++++++++++++++++++++++++++++ lastn/lastn_test.go | 92 +++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 1 deletion(-) create mode 100755 lastn/lastn.go create mode 100755 lastn/lastn_test.go diff --git a/.gitignore b/.gitignore index cb81dcf..ef64b27 100755 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ **.sw* -lastn **/testdata **/._* **/exec-* diff --git a/lastn/lastn.go b/lastn/lastn.go new file mode 100755 index 0000000..4e1308e --- /dev/null +++ b/lastn/lastn.go @@ -0,0 +1,164 @@ +package lastn + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "local/storage" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "time" + + "github.com/google/uuid" +) + +type Config struct { + N int + RcloneConf string + RcloneAlias string + Ns string + Root string + Store string + StoreAddr string + StoreUser string + StorePass string + Cmd string +} + +type LastN struct { + store storage.DB + conf Config +} + +func New(conf Config) (*LastN, error) { + var store storage.DB + var err error + switch conf.Store { + case "rclone": + store, err = storage.New(storage.TypeFromString(conf.Store), conf.RcloneConf, path.Join(conf.RcloneAlias+":")) + default: + store, err = storage.New(storage.TypeFromString(conf.Store), conf.StoreAddr, conf.StoreUser, conf.StorePass) + } + if err != nil { + return nil, err + } + root, err := filepath.Abs(conf.Root) + conf.Root = root + return &LastN{ + store: store, + conf: conf, + }, err +} + +func (lastN *LastN) Push() error { + root := lastN.conf.Root + store := lastN.store + root, err := filepath.Abs(root) + if err != nil { + return err + } + archive := path.Join( + os.TempDir(), + fmt.Sprintf( + "%s.%s.tar", + time.Now().Format("2006.01.02.15.04.05"), + uuid.New().String(), + ), + ) + cmd := exec.Command( + "tar", + "-czf", + archive, + "-C", + path.Dir(root), + path.Base(root), + ) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%v: %s", err, out) + } + b, err := ioutil.ReadFile(archive) + if err != nil { + return err + } + log.Println("Created backup", path.Base(archive)) + return store.Set(path.Base(archive), b, lastN.conf.Ns) +} + +func (lastN *LastN) Clean() error { + n := lastN.conf.N + store := lastN.store + backups, err := store.List([]string{lastN.conf.Ns}) + if err != nil { + return err + } + sort.Strings(backups) + for i := 0; i < len(backups)-n; i++ { + log.Println("Pruning old backup", backups[i]) + err := store.Set(backups[i], nil, lastN.conf.Ns) + if err != nil { + return err + } + } + return nil +} + +func (lastN *LastN) List() error { + store := lastN.store + backups, err := store.List([]string{lastN.conf.Ns}) + if err != nil { + return err + } + sort.Strings(backups) + for _, backup := range backups { + log.Println(backup) + } + return nil +} + +func (lastN *LastN) Restore() error { + root := lastN.conf.Root + "-restore" + os.RemoveAll(root) + if _, err := os.Stat(root); os.IsNotExist(err) { + if err := os.MkdirAll(root, os.ModePerm); err != nil { + return err + } + } + store := lastN.store + backups, err := store.List([]string{lastN.conf.Ns}) + if err != nil { + return fmt.Errorf("cannot list: %v", err) + } + sort.Strings(backups) + backup := backups[len(backups)-1] + b, err := store.Get(backup, lastN.conf.Ns) + if err != nil { + return fmt.Errorf("cannot get %s: %v", backup, err) + } + log.Printf("restoring %s (%v) in %s", backup, len(b), root) + cmd := exec.Command( + "tar", + "-C", + root, + "-xzf", + "-", + ) + stdin, err := cmd.StdinPipe() + if err != nil { + return fmt.Errorf("cannot get stdin: %v", err) + } + go func() { + defer stdin.Close() + io.Copy(stdin, bytes.NewReader(b)) + }() + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed tar -xf: %v: %s", err, out) + } + return nil +} diff --git a/lastn/lastn_test.go b/lastn/lastn_test.go new file mode 100755 index 0000000..ba1fcd5 --- /dev/null +++ b/lastn/lastn_test.go @@ -0,0 +1,92 @@ +package lastn + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "path" + "testing" + "time" +) + +func TestLastN(t *testing.T) { + d, clean := makeJunk(t) + defer clean() + + conf := Config{ + N: 3, + Ns: "b", + Root: d, + Store: "map", + } + lastn, err := New(conf) + if err != nil { + t.Fatal(err) + } + + if err := lastn.Push(); err != nil { + t.Fatal(err) + } + + was := log.Writer() + buff := bytes.NewBuffer(nil) + log.SetOutput(buff) + defer log.SetOutput(was) + if err := lastn.List(); err != nil { + t.Fatal(err) + } else if !bytes.Contains(buff.Bytes(), []byte(time.Now().Format("2006.01.02"))) { + t.Fatal(string(buff.Bytes())) + } + log.SetOutput(was) + + restored := path.Join(d+"-restore", path.Base(d)) + + if err := lastn.Restore(); err != nil { + t.Fatal(err) + } else if _, err := os.Stat(restored); os.IsNotExist(err) { + t.Fatal(err) + } else if err != nil { + t.Fatal(err) + } else if files, err := ioutil.ReadDir(restored); err != nil { + t.Fatal(err) + } else if len(files) != 3 { + t.Fatal(len(files)) + } else { + for _, file := range files { + if b, err := ioutil.ReadFile(path.Join(restored, file.Name())); err != nil { + t.Fatal(err) + } else if v := string(b); v != "hi" { + t.Fatal(v) + } + } + } + + for i := 0; i < 10; i++ { + if err := lastn.Push(); err != nil { + t.Fatal(err) + } + } + if saved, err := lastn.store.List([]string{lastn.conf.Ns}); err != nil { + t.Fatal(err) + } else if len(saved) != 11 { + t.Fatal(len(saved)) + } +} + +func makeJunk(t *testing.T) (string, func()) { + d, err := ioutil.TempDir(os.TempDir(), "lastNtest*") + if err != nil { + t.Fatal(err) + } + for i := 0; i < 3; i++ { + if err := ioutil.WriteFile(path.Join(d, fmt.Sprint(i)), []byte("hi"), os.ModePerm); err != nil { + os.RemoveAll(d) + t.Fatal(err) + } + } + return d, func() { + os.RemoveAll(d) + } +}