initial

monitor/arlo.go

@@ -0,0 +1,122 @@
+package monitor
+
+import (
+	"fmt"
+	"local/logb"
+	"local/sandbox/arlo-cleaner/arlo"
+	"local/sandbox/arlo-cleaner/config"
+	"local/sandbox/arlo-cleaner/rclone"
+	"net/http"
+	"path"
+	"sort"
+	"time"
+)
+
+type Arlo struct {
+	arlo   *arlo.Arlo
+	rclone *rclone.RClone
+}
+
+func NewArlo() (*Arlo, error) {
+	rclone, err := rclone.New()
+	if err != nil {
+		return nil, err
+	}
+	arlo, err := arlo.New()
+	return &Arlo{
+		arlo:   arlo,
+		rclone: rclone,
+	}, err
+}
+
+func (a *Arlo) Start() <-chan error {
+	ch := make(chan error)
+	go a.watch(ch)
+	return ch
+}
+
+func (a *Arlo) watch(ch chan<- error) {
+	if err := a.Clean(); err != nil {
+		ch <- err
+	}
+	ticker := time.NewTicker(config.MonitorInterval)
+	for _ = range ticker.C {
+		if err := a.Clean(); err != nil {
+			ch <- err
+		}
+	}
+}
+
+func (a *Arlo) Clean() error {
+	if ok, err := a.arlo.NeedsPruning(); err != nil {
+		return err
+	} else if !ok {
+		logb.Infof("Arlo cleaning not needed")
+		return nil
+	}
+	logb.Infof("Arlo clean commencing...")
+	defer logb.Infof("Arlo clean done")
+	prunes, err := a.pickPrunes()
+	if err != nil {
+		return err
+	}
+	if err := a.archives(prunes); err != nil {
+		return err
+	}
+	return a.arlo.DeleteVideos(prunes)
+}
+
+func (a *Arlo) pickPrunes() ([]*arlo.Video, error) {
+	videos, err := a.list()
+	if err != nil {
+		return nil, err
+	}
+	return a.pickRipe(videos), nil
+}
+
+func (a *Arlo) list() ([]*arlo.Video, error) {
+	return a.arlo.ListSince(time.Now().Add(time.Hour * -1 * 24 * 30))
+}
+
+func (a *Arlo) pickRipe(videos []*arlo.Video) []*arlo.Video {
+	sort.Slice(videos, func(i, j int) bool {
+		return videos[i].Created.Before(videos[j].Created)
+	})
+	allowed := config.ArloCapacity
+	retain := len(videos)
+	for i := len(videos) - 1; i >= 0; i-- {
+		sz := videos[i].Size()
+		allowed -= sz
+		if allowed >= 0 {
+			retain = i
+		}
+	}
+	logb.Debugf("found %v videos to purge of %v", retain, len(videos))
+	return videos[:retain]
+}
+
+func (a *Arlo) archives(videos []*arlo.Video) error {
+	for _, video := range videos {
+		if err := a.archive(video); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (a *Arlo) archive(video *arlo.Video) error {
+	resp, err := http.Get(video.Loc.String())
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	target := path.Join(
+		config.RCloneName,
+		fmt.Sprintf("%02d", video.Created.Year()),
+		fmt.Sprintf("%02d", video.Created.Month()),
+		fmt.Sprintf("%02d", video.Created.Day()),
+		video.Key(),
+	)
+	logb.Infof("archiving %v", target)
+	return a.rclone.CopyTo(resp.Body, target)
+}

monitor/arlo_test.go

@@ -0,0 +1,139 @@
+package monitor
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io/ioutil"
+	"local/sandbox/arlo-cleaner/arlo"
+	"local/sandbox/arlo-cleaner/config"
+	"local/sandbox/arlo-cleaner/rclone"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestArloPickRipe(t *testing.T) {
+	a := &Arlo{}
+
+	config.ArloBPS = 1
+
+	cases := map[string]struct {
+		videoCnt int
+		confCap  int64
+		outLen   int
+	}{
+		"prune none": {
+			videoCnt: 5,
+			confCap:  5,
+			outLen:   0,
+		},
+		"prune one": {
+			videoCnt: 5,
+			confCap:  4,
+			outLen:   1,
+		},
+		"prune all": {
+			videoCnt: 5,
+			confCap:  0,
+			outLen:   5,
+		},
+	}
+
+	for name, c := range cases {
+		videos := make([]*arlo.Video, c.videoCnt)
+		for i := range videos {
+			b := make([]byte, 1)
+			rand.Read(b)
+			videos[i] = &arlo.Video{
+				Duration: time.Second,
+				Created:  time.Now().Add(-24 * 365 * time.Hour * time.Duration(int(b[0]))),
+				Loc:      &url.URL{},
+			}
+		}
+		config.ArloCapacity = int64(c.confCap)
+		videos = a.pickRipe(videos)
+		if len(videos) != c.outLen {
+			t.Errorf("%s: want %v videos, got %v", name, c.outLen, len(videos))
+		}
+		for i := 0; i < len(videos)-1; i++ {
+			if videos[i].Created.After(videos[i+1].Created) {
+				t.Error(videos)
+			}
+		}
+	}
+}
+
+func TestArloArchive(t *testing.T) {
+	a, def := mockArlo(t)
+	defer def()
+
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "Hello\n")
+	}))
+	defer s.Close()
+
+	url, _ := url.Parse(s.URL)
+	url.Path = "my_video.mp4"
+	q := url.Query()
+	q.Set("hello", ".world.")
+	url.RawQuery = q.Encode()
+	video := &arlo.Video{
+		Loc:     url,
+		Created: time.Now(),
+		Device:  "device",
+	}
+
+	if err := a.archive(video); err != nil {
+		t.Fatal(err)
+	}
+
+	path := path.Join(
+		config.RCloneName,
+		fmt.Sprintf("%02d", video.Created.Year()),
+		fmt.Sprintf("%02d", video.Created.Month()),
+		fmt.Sprintf("%02d", video.Created.Day()),
+		video.Key(),
+	)
+	path = strings.TrimPrefix(path, "local:")
+	if b, err := ioutil.ReadFile(path); err != nil {
+		t.Fatal(err)
+	} else if v := string(b); v != "Hello\n" {
+		t.Fatal(v)
+	} else if err := os.Remove(path); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func mockArlo(t *testing.T) (*Arlo, func()) {
+	f, err := ioutil.TempFile(os.TempDir(), "rclone.conf.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Write([]byte(`
+[local]
+type = local
+   `))
+	f.Close()
+	was := os.Args
+	os.Args = []string{"foo", "-rcconf", f.Name(), "-rcname", "local:" + path.Dir(f.Name())}
+	if err := config.Refresh(); err != nil {
+		t.Fatal(err)
+	}
+	rclone, err := rclone.New()
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := &Arlo{
+		arlo:   &arlo.Arlo{},
+		rclone: rclone,
+	}
+	return a, func() {
+		os.Remove(f.Name())
+		os.Args = was
+	}
+}

monitor/arloclean_test.go

@@ -0,0 +1,37 @@
+package monitor
+
+import (
+	"local/sandbox/arlo-cleaner/config"
+	"os"
+	"testing"
+)
+
+func TestArloClean(t *testing.T) {
+	if _, ok := os.LookupEnv("INTEGRATION"); !ok {
+		t.Log("$INTEGRATION not set")
+		return
+	}
+	was := os.Args
+	defer func() {
+		os.Args = was
+	}()
+	os.Args = []string{
+		"foo",
+		"-arlouser",
+		"squeaky2x3@gmail.com",
+		"-arlopass",
+		"LetMe!23In",
+		"-arlomb",
+		"20",
+	}
+	if err := config.Refresh(); err != nil {
+		t.Fatal(err)
+	}
+	arlo, err := NewArlo()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := arlo.Clean(); err != nil {
+		t.Fatal(err)
+	}
+}

monitor/drive.go

@@ -0,0 +1,91 @@
+package monitor
+
+import (
+	"local/logb"
+	"local/sandbox/arlo-cleaner/arlo"
+	"local/sandbox/arlo-cleaner/config"
+	"local/sandbox/arlo-cleaner/rclone"
+	"path"
+	"strings"
+	"time"
+)
+
+type Drive struct {
+	rclone *rclone.RClone
+}
+
+func NewDrive() (*Drive, error) {
+	rclone, err := rclone.New()
+	return &Drive{
+		rclone: rclone,
+	}, err
+}
+
+func (d *Drive) Start() <-chan error {
+	ch := make(chan error)
+	go d.watch(ch)
+	return ch
+}
+
+func (d *Drive) watch(ch chan<- error) {
+	if err := d.Clean(); err != nil {
+		ch <- err
+	}
+	ticker := time.NewTicker(config.MonitorInterval)
+	for _ = range ticker.C {
+		if err := d.Clean(); err != nil {
+			ch <- err
+		}
+	}
+}
+
+func (d *Drive) Clean() error {
+	videos, err := d.rclone.List(config.RCloneName)
+	if err != nil {
+		return err
+	}
+	videos = d.filters(videos)
+	if len(videos) == 0 {
+		logb.Infof("Drive nothing to clean")
+		return nil
+	}
+	logb.Infof("Drive clean commencing...")
+	defer logb.Infof("Drive clean done")
+	return d.deletes(videos)
+}
+
+func (d *Drive) filters(videos []string) []string {
+	filtered := []string{}
+	for _, video := range videos {
+		if d.expired(video) {
+			filtered = append(filtered, video)
+		}
+	}
+	logb.Debugf("drive found %v expired videos from %v", len(filtered), len(videos))
+	return filtered
+}
+
+func (d *Drive) expired(video string) bool {
+	base := path.Base(video)
+	timestamp := strings.Split(base, ".")[0]
+	created, err := time.Parse(arlo.TSFormat, timestamp)
+	if err != nil {
+		logb.Errorf("not deleting from archive: cannot parse", timestamp, ":", err)
+		return false
+	}
+	return time.Since(created) > config.ArchiveLength
+}
+
+func (d *Drive) deletes(videos []string) error {
+	for _, video := range videos {
+		if err := d.delete(video); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (d *Drive) delete(video string) error {
+	logb.Infof("deleting drive video %v", video)
+	return d.rclone.Del(path.Join(config.RCloneName, video))
+}

monitor/drive_test.go

@@ -0,0 +1,87 @@
+package monitor
+
+import (
+	"io/ioutil"
+	"local/sandbox/arlo-cleaner/arlo"
+	"local/sandbox/arlo-cleaner/config"
+	"os"
+	"path"
+	"strings"
+	"testing"
+	"time"
+)
+
+func mockDrive(t *testing.T) (*Drive, func()) {
+	arlo, def := mockArlo(t)
+	d := &Drive{
+		rclone: arlo.rclone,
+	}
+	return d, def
+}
+
+func TestDriveFilters(t *testing.T) {
+	d := &Drive{}
+	videos := []string{expVideo()}
+	videos = d.filters(videos)
+	if len(videos) != 1 {
+		t.Error(videos)
+	}
+}
+
+func TestDriveExpired(t *testing.T) {
+	d := &Drive{}
+	if exp := d.expired(expVideo()); !exp {
+		t.Error(exp)
+	}
+	if exp := d.expired(nonExpVideo()); exp {
+		t.Error(exp)
+	}
+}
+
+func TestDriveDeletes(t *testing.T) {
+	d, def := mockDrive(t)
+	defer def()
+
+	if err := d.deletes(nil); err != nil {
+		t.Error(err)
+	}
+	if err := d.deletes([]string{}); err != nil {
+		t.Error(err)
+	}
+	if err := d.deletes([]string{expVideo()}); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestDriveDelete(t *testing.T) {
+	d, def := mockDrive(t)
+	defer def()
+
+	video := expVideo()
+	abspath := path.Join(strings.TrimPrefix(config.RCloneName, "local:"), video)
+	if err := ioutil.WriteFile(abspath, []byte("hi"), os.ModePerm); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.delete(expVideo()); err != nil {
+		t.Error(err)
+	}
+
+	if _, err := os.Stat(abspath); err == nil {
+		t.Fatal(err)
+	}
+}
+
+func expVideo() string {
+	config.ArchiveLength = time.Duration(0)
+	return videoName()
+}
+
+func nonExpVideo() string {
+	config.ArchiveLength = time.Hour
+	return videoName()
+}
+
+func videoName() string {
+	return time.Now().UTC().Format(arlo.TSFormat)
+}