rss and rss item and encode

Former-commit-id: 333b5635cc0eb22964403cc78f660e02a830f077

config/encodable.go

@@ -1,22 +0,0 @@
-package config
-
-import (
-	"bytes"
-	"encoding/gob"
-)
-
-type Encodable struct{}
-
-func (e *Encodable) Encode() ([]byte, error) {
-	buff := bytes.NewBuffer(nil)
-	enc := gob.NewEncoder(buff)
-	err := enc.Encode(e)
-	return buff.Bytes(), err
-}
-
-func (e *Encodable) Decode(b []byte) error {
-	buff := bytes.NewBuffer(b)
-	dec := gob.NewDecoder(buff)
-	err := dec.Decode(e)
-	return err
-}

config/encode.go

@@ -0,0 +1,20 @@
+package config
+
+import (
+	"bytes"
+	"encoding/gob"
+)
+
+func Encode(v interface{}) ([]byte, error) {
+	buff := bytes.NewBuffer(nil)
+	enc := gob.NewEncoder(buff)
+	err := enc.Encode(v)
+	return buff.Bytes(), err
+}
+
+func Decode(b []byte, v interface{}) error {
+	buff := bytes.NewBuffer(b)
+	enc := gob.NewDecoder(buff)
+	err := enc.Decode(v)
+	return err
+}

config/encode_test.go

@@ -0,0 +1,24 @@
+package config
+
+import (
+	"testing"
+	"time"
+)
+
+func TestEncodeDecode(t *testing.T) {
+	c := time.Now()
+	d := c.Add(0)
+
+	b, err := Encode(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := Decode(b, &c); err != nil {
+		t.Fatal(err)
+	}
+
+	if c.Sub(d) > time.Millisecond*5 {
+		t.Fatalf("%v => %v", d, c)
+	}
+}

monitor/item.go

@@ -12,7 +12,6 @@ import (
 
 type Item struct {
 	Key string
-	*config.Encodable
 }
 
 const nsLast = "nsLast"
@@ -23,8 +22,7 @@ var forever = time.Duration(time.Hour * 99999)
 
 func NewItem(key string, interval time.Duration) (*Item, error) {
 	i := &Item{
-		Key:       key,
+		Key: key,
-		Encodable: &config.Encodable{},
 	}
 
 	if err := i.setInterval(interval); err != nil {
@@ -124,3 +122,11 @@ func (i *Item) getLast() (time.Time, error) {
 	}
 	return t, nil
 }
+
+func (i *Item) Encode() ([]byte, error) {
+	return config.Encode(i)
+}
+
+func (i *Item) Decode(b []byte) error {
+	return config.Decode(b, i)
+}

rss/feed.go

@@ -13,24 +13,30 @@ import (
 
 const NSFeeds = "NSFeeds"
 
-type feed struct {
+type Feed struct {
 	Key           string
 	URL           string
 	Updated       time.Time
 	TitleFilter   string
 	ContentFilter string
 	Tags          []string
-	*config.Encodable
 }
 
-func newFeed(key string) *feed {
+func newFeed(key string) *Feed {
-	return &feed{
+	return &Feed{
-		Key:       key,
+		Key: key,
-		Encodable: &config.Encodable{},
 	}
 }
 
-func (f *feed) load() error {
+func (f *Feed) Encode() ([]byte, error) {
+	return config.Encode(f)
+}
+
+func (f *Feed) Decode(b []byte) error {
+	return config.Decode(b, f)
+}
+
+func (f *Feed) load() error {
 	if f.Key == "" {
 		return errors.New("cannot load nil feed")
 	}
@@ -42,7 +48,7 @@ func (f *feed) load() error {
 	return f.Decode(b)
 }
 
-func (f *feed) pull() error {
+func (f *Feed) pull() error {
 	if f.URL == "" {
 		if err := f.load(); err != nil {
 			return err
@@ -61,21 +67,20 @@ func (f *feed) pull() error {
 
 	itemTSs := []*time.Time{}
 	for _, i := range gofeed.Items {
-		ts := latestTSPtr(i.UpdatedParsed, i.PublishedParsed)
+		item, err := newItem(i, f.ContentFilter, f.Key)
-		itemTSs = append(itemTSs, &ts)
+		if err != nil {
-		if ts.Before(f.Updated) {
+			log.Println(err)
+			continue
+		}
+		itemTSs = append(itemTSs, &item.TS)
+		if item.TS.Before(f.Updated) {
 			log.Println("Skipping old item")
 			continue
 		}
-		if ok := regexp.MustCompile(f.TitleFilter).MatchString(i.Title); !ok {
+		if ok := regexp.MustCompile(f.TitleFilter).MatchString(item.Title); !ok {
 			log.Println("Skipping bad titled item")
 			continue
 		}
-		item, err := newItem(i, f.ContentFilter)
-		if err != nil {
-			log.Println(err)
-			continue
-		}
 		if err := item.save(); err != nil {
 			log.Println(err)
 			continue
@@ -87,7 +92,7 @@ func (f *feed) pull() error {
 	return nil
 }
 
-func (f *feed) save() error {
+func (f *Feed) save() error {
 	b, err := f.Encode()
 	if err != nil {
 		return err

rss/feed_test.go

@@ -1,9 +1,17 @@
 package rss
 
 import (
+	"bytes"
 	"encoding/json"
+	"fmt"
+	"io"
 	"local/rssmon3/config"
+	"log"
+	"net/http"
+	"net/http/httptest"
 	"os"
+	"path"
+	"reflect"
 	"testing"
 	"time"
 )
@@ -15,6 +23,20 @@ func initRSSFeed() {
 	}
 }
 
+type mockStruct struct{}
+
+func mockRSS() *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var v mockStruct
+		base := reflect.TypeOf(v).PkgPath()
+		f, err := os.Open(path.Join(os.Getenv("GOPATH"), "src", base, "./testdata/rss.xml"))
+		if err != nil {
+			panic(err)
+		}
+		io.Copy(w, f)
+	}))
+}
+
 func TestRSSFeedEncodeDecode(t *testing.T) {
 	initRSSFeed()
 
@@ -39,6 +61,97 @@ func TestRSSFeedEncodeDecode(t *testing.T) {
 	if string(fb) != string(gb) {
 		t.Errorf("%v => %v", *f, *g)
 	}
-
+}
-	t.Logf("%s vs. %s", fb, gb)
+
+func TestRSSFeedSaveLoad(t *testing.T) {
+	initRSSFeed()
+
+	f := newFeed("key")
+	f.Updated = time.Now()
+	f.TitleFilter = "a"
+	f.ContentFilter = "b"
+	f.Tags = []string{"c"}
+
+	if err := f.save(); err != nil {
+		t.Fatal(err)
+	}
+
+	g := newFeed("key")
+	if err := g.load(); err != nil {
+		t.Fatal(err)
+	}
+
+	fb, _ := json.Marshal(f)
+	gb, _ := json.Marshal(g)
+	if string(fb) != string(gb) {
+		t.Fatalf("%v != %v", f, g)
+	}
+
+	h := newFeed("key2")
+	if err := h.load(); err == nil {
+		t.Fatal("can load nil feed")
+	}
+}
+
+func TestRSSFeedPull(t *testing.T) {
+	initRSSFeed()
+
+	s := mockRSS()
+	defer s.Close()
+
+	f := newFeed("key")
+	f.TitleFilter = "500"
+	f.ContentFilter = "b"
+	f.Tags = []string{"c"}
+	f.URL = s.URL
+
+	log.SetOutput(bytes.NewBuffer(nil))
+	defer log.SetOutput(os.Stderr)
+	if err := f.pull(); err != nil {
+		t.Fatal(err)
+	}
+	log.SetOutput(os.Stderr)
+
+	if f.Updated.Format("2006-01-02 15:04:05") != "2019-06-18 19:00:00" {
+		t.Errorf("updated is wrong: %v", f)
+	}
+
+	keys, err := config.Values().DB.List([]string{nsItems, f.Key})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(keys) < 1 {
+		t.Fatal(len(keys))
+	}
+	if keys[0] != "https://roosterteeth.com/episode/rooster-teeth-podcast-2018-rooster-teeth-podcast-500" {
+		t.Fatal(keys[0])
+	}
+
+	log.SetOutput(bytes.NewBuffer(nil))
+	defer log.SetOutput(os.Stderr)
+	if err := f.pull(); err != nil {
+		t.Fatal(err)
+	}
+	log.SetOutput(os.Stderr)
+	keysB, err := config.Values().DB.List([]string{nsItems, f.Key})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if fmt.Sprintf("%v", keys) != fmt.Sprintf("%v", keysB) {
+		t.Fatalf("%v vs %v", keys, keysB)
+	}
+
+	i := &Item{
+		Link:    keys[0],
+		FeedKey: f.Key,
+	}
+	if err := i.load(); err != nil {
+		t.Fatal(err)
+	}
+	if *i == (Item{}) {
+		t.Fatal(i)
+	}
+	if i.Link == "" || i.Title == "" || i.FeedKey == "" || i.Content == "" {
+		t.Fatal(i)
+	}
 }

rss/item.go

@@ -2,21 +2,91 @@ package rss
 
 import (
 	"errors"
+	"fmt"
+	"io/ioutil"
 	"local/rssmon3/config"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
 
 	"github.com/mmcdole/gofeed"
 )
 
+const nsItems = "nsItems"
+
 type Item struct {
-	*config.Encodable
+	Title   string
+	Link    string
+	Content string
+	TS      time.Time
+	FeedKey string
 }
 
-func newItem(i *gofeed.Item, contentFilter string) (*Item, error) {
+func newItem(i *gofeed.Item, contentFilter, feedKey string) (*Item, error) {
-	return &Item{
+	item := &Item{
-		Encodable: &config.Encodable{},
+		Title:   i.Title,
-	}, errors.New("not impl")
+		Link:    i.Link,
+		Content: i.Content,
+		TS:      latestTSPtr(i.UpdatedParsed, i.PublishedParsed),
+		FeedKey: feedKey,
+	}
+
+	if item.Content == "" {
+		item.Content = i.Description
+	}
+	if item.Content == "" {
+		resp, err := http.Get(item.Link)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		b, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		item.Content = string(b)
+	}
+	if unescaped, err := url.QueryUnescape(item.Content); err == nil {
+		item.Content = unescaped
+	}
+
+	if contentFilter == "" {
+		contentFilter = ".*"
+	}
+	item.Content = strings.Join(regexp.MustCompile(contentFilter).FindAllString(item.Content, -1), "<br>")
+
+	item.Content = fmt.Sprintf(`<a href="%s">%s</a><br>%s`, item.Link, item.Title, item.Content)
+
+	return item, nil
+}
+
+func (i *Item) Encode() ([]byte, error) {
+	return config.Encode(i)
+}
+
+func (i *Item) Decode(b []byte) error {
+	return config.Decode(b, i)
 }
 
 func (i *Item) save() error {
-	return errors.New("not impl")
+	db := config.Values().DB
+	b, err := i.Encode()
+	if err != nil {
+		return err
+	}
+	return db.Set(i.Link, b, nsItems, i.FeedKey)
+}
+
+func (i *Item) load() error {
+	if i.Link == "" || i.FeedKey == "" {
+		return errors.New("cannot load nil item")
+	}
+	db := config.Values().DB
+	b, err := db.Get(i.Link, nsItems, i.FeedKey)
+	if err != nil {
+		return err
+	}
+	return config.Decode(b, i)
 }

rss/item_test.go

@@ -0,0 +1,84 @@
+package rss
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/mmcdole/gofeed"
+)
+
+func initRSSItem() {
+	initRSSFeed()
+}
+
+func TestRSSItemNewEncodeDecode(t *testing.T) {
+	initRSSItem()
+
+	s := mockRSS()
+	defer s.Close()
+
+	resp, err := http.Get(s.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	gofeed, err := gofeed.NewParser().Parse(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	gofeed.Items[0].Content = ""
+	gofeed.Items[0].Description = ""
+
+	item, err := newItem(gofeed.Items[0], ".*", "key")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	itemB, err := newItem(gofeed.Items[0], "Podcast", "key")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if itemB.Content == item.Content {
+		t.Errorf("content filter did not apply")
+	}
+
+	b, err := item.Encode()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := itemB.Decode(b); err != nil {
+		t.Fatal(err)
+	}
+	if itemB.Content != item.Content {
+		t.Errorf("decode found %v, want %v", itemB.Content, item.Content)
+	}
+
+	if err := item.save(); err != nil {
+		t.Fatal(err)
+	}
+
+	itemC := &Item{
+		Link:    item.Link,
+		FeedKey: item.FeedKey,
+	}
+	if err := itemC.load(); err != nil {
+		t.Fatal(err)
+	}
+	if fmt.Sprintf("%v", item) != fmt.Sprintf("%v", itemC) {
+		t.Fatalf("%v != %v", item, itemC)
+	}
+
+	itemD := &Item{}
+	if err := itemD.load(); err == nil {
+		t.Fatal(err)
+	}
+
+	itemE := &Item{
+		Link:    "nothing",
+		FeedKey: item.FeedKey,
+	}
+	if err := itemE.load(); err == nil {
+		t.Fatal(err)
+	}
+}

rss/testdata/rss.xml.REMOVED.git-id

@@ -0,0 +1 @@
+d36f6c94dfbbaaeac339bfaec9d0dd13b0ff099b

rssmon3.REMOVED.git-id

@@ -1 +1 @@
-bf5140d93694bf40f561841cdee34065d4cc2455
+2ecdcaccf6c3f6135a21544a465ace01b7326ec2