initial
This commit is contained in:
Bel LaPointe
59
rclone/copy.go
Executable file
59
rclone/copy.go
Executable file
@@ -0,0 +1,59 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"local/logb"
|
||||
"local/sandbox/arlo-cleaner/config"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
_ "github.com/ncw/rclone/backend/drive"
|
||||
_ "github.com/ncw/rclone/backend/local"
|
||||
_ "github.com/ncw/rclone/backend/s3"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/object"
|
||||
)
|
||||
|
||||
func (rc *RClone) CopyTo(r io.Reader, destination string) error {
|
||||
logb.Verbosef("copy to: %v", destination)
|
||||
f, err := fs.NewFs(path.Dir(destination))
|
||||
logb.Verbosef("got new fs: %v", err)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info, err := f.NewObject(path.Base(destination)); err == nil {
|
||||
logb.Warnf("not re-uploading file to: %v: %v", destination, info)
|
||||
fmt.Println("copyTo", 3)
|
||||
return nil
|
||||
} else {
|
||||
logb.Verbosef("error listing existing object as expected: %v, %v", err, info)
|
||||
}
|
||||
logb.Verbosef("new obj err: %v", nil)
|
||||
info := object.NewStaticObjectInfo(path.Base(destination), time.Now(), 0, true, nil, f)
|
||||
logb.Infof("uploading %v", destination)
|
||||
if config.DryRun {
|
||||
logb.Debugf("copy to", info)
|
||||
//return nil TODO
|
||||
}
|
||||
_, err = f.Put(r, info)
|
||||
return err
|
||||
}
|
||||
|
||||
func (rc *RClone) CopyFrom(w io.Writer, source string) error {
|
||||
logb.Infof("copying from %v", source)
|
||||
f, err := fs.NewFs(path.Dir(source))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
obj, err := f.NewObject(path.Base(source))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r, err := obj.Open()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
_, err = io.Copy(w, r)
|
||||
return err
|
||||
}
|
||||
106
rclone/copy_test.go
Executable file
106
rclone/copy_test.go
Executable file
@@ -0,0 +1,106 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"local/logb"
|
||||
"local/sandbox/arlo-cleaner/config"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCopyFrom(t *testing.T) {
|
||||
logb.Set(logb.ERROR)
|
||||
rc, def := makeRC()
|
||||
defer def()
|
||||
|
||||
t.Log("making f")
|
||||
f, err := ioutil.TempFile(os.TempDir(), "remote*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Write([]byte(`hello`))
|
||||
f.Close()
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
t.Log("reading f")
|
||||
b := bytes.NewBuffer(nil)
|
||||
if err := rc.CopyFrom(b, "local:"+f.Name()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Log("result")
|
||||
if v := string(b.Bytes()); v != "hello" {
|
||||
t.Fatal(v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyTo(t *testing.T) {
|
||||
rc, def := makeRC()
|
||||
defer def()
|
||||
logb.Set(logb.VERBOSE)
|
||||
|
||||
t.Log("making f")
|
||||
f, err := ioutil.TempFile(os.TempDir(), "remote*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
t.Logf("writing %s", f.Name())
|
||||
s := strings.NewReader("hello")
|
||||
|
||||
was := logb.Writer()
|
||||
b := bytes.NewBuffer(nil)
|
||||
logb.SetWriter(b)
|
||||
if err := rc.CopyTo(s, "local:"+f.Name()); err != nil {
|
||||
logb.SetWriter(was)
|
||||
t.Fatal(err)
|
||||
}
|
||||
logb.SetWriter(was)
|
||||
if !bytes.Contains(b.Bytes(), []byte("not re-uploading")) {
|
||||
t.Fatalf("%q", string(b.Bytes()))
|
||||
}
|
||||
if err := os.Remove(f.Name()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := rc.CopyTo(s, "local:"+f.Name()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Log("reading f from", f.Name())
|
||||
if b, err := ioutil.ReadFile(f.Name()); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if string(b) != "hello" {
|
||||
t.Fatal(string(b))
|
||||
}
|
||||
}
|
||||
|
||||
func makeRC() (*RClone, func()) {
|
||||
logb.Set(logb.ERROR)
|
||||
f, err := ioutil.TempFile(os.TempDir(), "copy.config.*")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
f.Write([]byte(`
|
||||
[local]
|
||||
type = local
|
||||
`))
|
||||
f.Close()
|
||||
os.Setenv("RCCONF", f.Name())
|
||||
|
||||
was := os.Args
|
||||
os.Args = []string{"foo"}
|
||||
config.Refresh()
|
||||
rc, err := New()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return rc, func() {
|
||||
os.Args = was
|
||||
os.Remove(f.Name())
|
||||
}
|
||||
}
|
||||
31
rclone/del.go
Executable file
31
rclone/del.go
Executable file
@@ -0,0 +1,31 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
"local/logb"
|
||||
"local/sandbox/arlo-cleaner/config"
|
||||
"path"
|
||||
|
||||
_ "github.com/ncw/rclone/backend/drive"
|
||||
_ "github.com/ncw/rclone/backend/local"
|
||||
_ "github.com/ncw/rclone/backend/s3"
|
||||
"github.com/ncw/rclone/fs"
|
||||
)
|
||||
|
||||
func (rc *RClone) Del(destination string) error {
|
||||
f, err := fs.NewFs(path.Dir(destination))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
obj, err := f.NewObject(path.Base(destination))
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if config.DryRun {
|
||||
logb.Infof("del", obj)
|
||||
//return nil TODO
|
||||
}
|
||||
logb.Infof("removing %v", destination)
|
||||
return obj.Remove()
|
||||
}
|
||||
31
rclone/del_test.go
Executable file
31
rclone/del_test.go
Executable file
@@ -0,0 +1,31 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDel(t *testing.T) {
|
||||
rc, def := makeRC()
|
||||
defer def()
|
||||
|
||||
t.Log("making f")
|
||||
f, err := ioutil.TempFile(os.TempDir(), "remote*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Write([]byte(`hello`))
|
||||
f.Close()
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
t.Log("deleting f")
|
||||
if err := rc.Del("local:" + f.Name()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Log("reading deleted f")
|
||||
if _, err := ioutil.ReadFile(f.Name()); err == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
24
rclone/list.go
Executable file
24
rclone/list.go
Executable file
@@ -0,0 +1,24 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
_ "github.com/ncw/rclone/backend/drive"
|
||||
_ "github.com/ncw/rclone/backend/local"
|
||||
_ "github.com/ncw/rclone/backend/s3"
|
||||
"github.com/ncw/rclone/fs"
|
||||
)
|
||||
|
||||
func (rc *RClone) List(destination string) ([]string, error) {
|
||||
f, err := fs.NewFs(destination)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := f.List("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, len(entries))
|
||||
for i, entry := range entries {
|
||||
names[i] = entry.Remote()
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
41
rclone/list_test.go
Executable file
41
rclone/list_test.go
Executable file
@@ -0,0 +1,41 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestList(t *testing.T) {
|
||||
rc, def := makeRC()
|
||||
defer def()
|
||||
|
||||
d, err := ioutil.TempDir(os.TempDir(), "list.*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := ioutil.WriteFile(path.Join(d, strconv.Itoa(i)), []byte("hi"), os.ModePerm); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
names, err := rc.List("local:" + d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(names) != 3 {
|
||||
t.Fatal(len(names))
|
||||
}
|
||||
|
||||
sort.Strings(names)
|
||||
for i := range names {
|
||||
if names[i] != strconv.Itoa(i) {
|
||||
t.Error(names[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
105
rclone/minio_test.go
Executable file
105
rclone/minio_test.go
Executable file
@@ -0,0 +1,105 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"local/sandbox/arlo-cleaner/config"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMinio(t *testing.T) {
|
||||
if _, ok := os.LookupEnv("INTEGRATION"); !ok {
|
||||
t.Log("$INTEGRATION not set")
|
||||
return
|
||||
}
|
||||
t.Log("docker run --rm -d --name minio -p 9000:9000 -e MINIO_ACCESS_KEY=accesskey -e MINIO_SECRET_KEY=secretkey minio/minio server /data && mc mb local/bucket")
|
||||
f, err := ioutil.TempFile(os.TempDir(), "minio.config.*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
f.Write([]byte(`
|
||||
[minio]
|
||||
type = s3
|
||||
provider = Minio
|
||||
env_auth = false
|
||||
access_key_id = accesskey
|
||||
secret_access_key = secretkey
|
||||
endpoint = http://localhost:9000
|
||||
acl = private
|
||||
`))
|
||||
f.Close()
|
||||
os.Setenv("RCCONF", f.Name())
|
||||
|
||||
was := os.Args
|
||||
defer func() {
|
||||
os.Args = was
|
||||
}()
|
||||
os.Args = []string{"a"}
|
||||
if err := config.Refresh(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rc, err := New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
name := "minio:bucket/key"
|
||||
|
||||
calls := []struct {
|
||||
foo func(*RClone, string) error
|
||||
ok bool
|
||||
}{
|
||||
{minioDel, true},
|
||||
{minioCopyFrom, false},
|
||||
{minioCopyTo, true},
|
||||
{minioList, true},
|
||||
{minioCopyFrom, true},
|
||||
{minioDel, true},
|
||||
{minioCopyFrom, false},
|
||||
}
|
||||
|
||||
for _, call := range calls {
|
||||
if err := call.foo(rc, name); (err == nil) != call.ok {
|
||||
t.Fatal(call)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func minioCopyTo(rc *RClone, name string) error {
|
||||
return rc.CopyTo(strings.NewReader("hello"), name)
|
||||
}
|
||||
|
||||
func minioCopyFrom(rc *RClone, name string) error {
|
||||
w := bytes.NewBuffer(nil)
|
||||
if err := rc.CopyFrom(w, name); err != nil {
|
||||
return err
|
||||
} else if v := string(w.Bytes()); v != "hello" {
|
||||
return errors.New(v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func minioDel(rc *RClone, name string) error {
|
||||
return rc.Del(name)
|
||||
}
|
||||
|
||||
func minioList(rc *RClone, name string) error {
|
||||
names, err := rc.List(path.Dir(name))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(names) != 1 {
|
||||
return fmt.Errorf("%v", len(names))
|
||||
}
|
||||
if names[0] != "key" {
|
||||
return errors.New(names[0])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
13
rclone/rclone.go
Executable file
13
rclone/rclone.go
Executable file
@@ -0,0 +1,13 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
_ "github.com/ncw/rclone/backend/drive"
|
||||
_ "github.com/ncw/rclone/backend/local"
|
||||
_ "github.com/ncw/rclone/backend/s3"
|
||||
)
|
||||
|
||||
type RClone struct{}
|
||||
|
||||
func New() (*RClone, error) {
|
||||
return &RClone{}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user