diff --git a/config/encodable.go b/config/encodable.go
deleted file mode 100644
index 4bf3331..0000000
--- a/config/encodable.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package config
-
-import (
- "bytes"
- "encoding/gob"
-)
-
-type Encodable struct{}
-
-func (e *Encodable) Encode() ([]byte, error) {
- buff := bytes.NewBuffer(nil)
- enc := gob.NewEncoder(buff)
- err := enc.Encode(e)
- return buff.Bytes(), err
-}
-
-func (e *Encodable) Decode(b []byte) error {
- buff := bytes.NewBuffer(b)
- dec := gob.NewDecoder(buff)
- err := dec.Decode(e)
- return err
-}
diff --git a/config/encode.go b/config/encode.go
new file mode 100644
index 0000000..d9f4b3d
--- /dev/null
+++ b/config/encode.go
@@ -0,0 +1,20 @@
+package config
+
+import (
+ "bytes"
+ "encoding/gob"
+)
+
+func Encode(v interface{}) ([]byte, error) {
+ buff := bytes.NewBuffer(nil)
+ enc := gob.NewEncoder(buff)
+ err := enc.Encode(v)
+ return buff.Bytes(), err
+}
+
+func Decode(b []byte, v interface{}) error {
+ buff := bytes.NewBuffer(b)
+ enc := gob.NewDecoder(buff)
+ err := enc.Decode(v)
+ return err
+}
diff --git a/config/encode_test.go b/config/encode_test.go
new file mode 100644
index 0000000..8ceed45
--- /dev/null
+++ b/config/encode_test.go
@@ -0,0 +1,24 @@
+package config
+
+import (
+ "testing"
+ "time"
+)
+
+func TestEncodeDecode(t *testing.T) {
+ c := time.Now()
+ d := c.Add(0)
+
+ b, err := Encode(c)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := Decode(b, &c); err != nil {
+ t.Fatal(err)
+ }
+
+ if c.Sub(d) > time.Millisecond*5 {
+ t.Fatalf("%v => %v", d, c)
+ }
+}
diff --git a/monitor/item.go b/monitor/item.go
index 7c54837..b34ff31 100644
--- a/monitor/item.go
+++ b/monitor/item.go
@@ -12,7 +12,6 @@ import (
type Item struct {
Key string
- *config.Encodable
}
const nsLast = "nsLast"
@@ -23,8 +22,7 @@ var forever = time.Duration(time.Hour * 99999)
func NewItem(key string, interval time.Duration) (*Item, error) {
i := &Item{
- Key: key,
- Encodable: &config.Encodable{},
+ Key: key,
}
if err := i.setInterval(interval); err != nil {
@@ -124,3 +122,11 @@ func (i *Item) getLast() (time.Time, error) {
}
return t, nil
}
+
+func (i *Item) Encode() ([]byte, error) {
+ return config.Encode(i)
+}
+
+func (i *Item) Decode(b []byte) error {
+ return config.Decode(b, i)
+}
diff --git a/rss/feed.go b/rss/feed.go
index 166c8c5..1b741c7 100644
--- a/rss/feed.go
+++ b/rss/feed.go
@@ -13,24 +13,30 @@ import (
const NSFeeds = "NSFeeds"
-type feed struct {
+type Feed struct {
Key string
URL string
Updated time.Time
TitleFilter string
ContentFilter string
Tags []string
- *config.Encodable
}
-func newFeed(key string) *feed {
- return &feed{
- Key: key,
- Encodable: &config.Encodable{},
+func newFeed(key string) *Feed {
+ return &Feed{
+ Key: key,
}
}
-func (f *feed) load() error {
+func (f *Feed) Encode() ([]byte, error) {
+ return config.Encode(f)
+}
+
+func (f *Feed) Decode(b []byte) error {
+ return config.Decode(b, f)
+}
+
+func (f *Feed) load() error {
if f.Key == "" {
return errors.New("cannot load nil feed")
}
@@ -42,7 +48,7 @@ func (f *feed) load() error {
return f.Decode(b)
}
-func (f *feed) pull() error {
+func (f *Feed) pull() error {
if f.URL == "" {
if err := f.load(); err != nil {
return err
@@ -61,21 +67,20 @@ func (f *feed) pull() error {
itemTSs := []*time.Time{}
for _, i := range gofeed.Items {
- ts := latestTSPtr(i.UpdatedParsed, i.PublishedParsed)
- itemTSs = append(itemTSs, &ts)
- if ts.Before(f.Updated) {
+ item, err := newItem(i, f.ContentFilter, f.Key)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ itemTSs = append(itemTSs, &item.TS)
+ if item.TS.Before(f.Updated) {
log.Println("Skipping old item")
continue
}
- if ok := regexp.MustCompile(f.TitleFilter).MatchString(i.Title); !ok {
+ if ok := regexp.MustCompile(f.TitleFilter).MatchString(item.Title); !ok {
log.Println("Skipping bad titled item")
continue
}
- item, err := newItem(i, f.ContentFilter)
- if err != nil {
- log.Println(err)
- continue
- }
if err := item.save(); err != nil {
log.Println(err)
continue
@@ -87,7 +92,7 @@ func (f *feed) pull() error {
return nil
}
-func (f *feed) save() error {
+func (f *Feed) save() error {
b, err := f.Encode()
if err != nil {
return err
diff --git a/rss/feed_test.go b/rss/feed_test.go
index e603ae6..076cce4 100644
--- a/rss/feed_test.go
+++ b/rss/feed_test.go
@@ -1,9 +1,17 @@
package rss
import (
+ "bytes"
"encoding/json"
+ "fmt"
+ "io"
"local/rssmon3/config"
+ "log"
+ "net/http"
+ "net/http/httptest"
"os"
+ "path"
+ "reflect"
"testing"
"time"
)
@@ -15,6 +23,20 @@ func initRSSFeed() {
}
}
+type mockStruct struct{}
+
+func mockRSS() *httptest.Server {
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var v mockStruct
+ base := reflect.TypeOf(v).PkgPath()
+ f, err := os.Open(path.Join(os.Getenv("GOPATH"), "src", base, "./testdata/rss.xml"))
+ if err != nil {
+ panic(err)
+ }
+ io.Copy(w, f)
+ }))
+}
+
func TestRSSFeedEncodeDecode(t *testing.T) {
initRSSFeed()
@@ -39,6 +61,97 @@ func TestRSSFeedEncodeDecode(t *testing.T) {
if string(fb) != string(gb) {
t.Errorf("%v => %v", *f, *g)
}
-
- t.Logf("%s vs. %s", fb, gb)
+}
+
+func TestRSSFeedSaveLoad(t *testing.T) {
+ initRSSFeed()
+
+ f := newFeed("key")
+ f.Updated = time.Now()
+ f.TitleFilter = "a"
+ f.ContentFilter = "b"
+ f.Tags = []string{"c"}
+
+ if err := f.save(); err != nil {
+ t.Fatal(err)
+ }
+
+ g := newFeed("key")
+ if err := g.load(); err != nil {
+ t.Fatal(err)
+ }
+
+ fb, _ := json.Marshal(f)
+ gb, _ := json.Marshal(g)
+ if string(fb) != string(gb) {
+ t.Fatalf("%v != %v", f, g)
+ }
+
+ h := newFeed("key2")
+ if err := h.load(); err == nil {
+ t.Fatal("can load nil feed")
+ }
+}
+
+func TestRSSFeedPull(t *testing.T) {
+ initRSSFeed()
+
+ s := mockRSS()
+ defer s.Close()
+
+ f := newFeed("key")
+ f.TitleFilter = "500"
+ f.ContentFilter = "b"
+ f.Tags = []string{"c"}
+ f.URL = s.URL
+
+ log.SetOutput(bytes.NewBuffer(nil))
+ defer log.SetOutput(os.Stderr)
+ if err := f.pull(); err != nil {
+ t.Fatal(err)
+ }
+ log.SetOutput(os.Stderr)
+
+ if f.Updated.Format("2006-01-02 15:04:05") != "2019-06-18 19:00:00" {
+ t.Errorf("updated is wrong: %v", f)
+ }
+
+ keys, err := config.Values().DB.List([]string{nsItems, f.Key})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(keys) < 1 {
+ t.Fatal(len(keys))
+ }
+ if keys[0] != "https://roosterteeth.com/episode/rooster-teeth-podcast-2018-rooster-teeth-podcast-500" {
+ t.Fatal(keys[0])
+ }
+
+ log.SetOutput(bytes.NewBuffer(nil))
+ defer log.SetOutput(os.Stderr)
+ if err := f.pull(); err != nil {
+ t.Fatal(err)
+ }
+ log.SetOutput(os.Stderr)
+ keysB, err := config.Values().DB.List([]string{nsItems, f.Key})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fmt.Sprintf("%v", keys) != fmt.Sprintf("%v", keysB) {
+ t.Fatalf("%v vs %v", keys, keysB)
+ }
+
+ i := &Item{
+ Link: keys[0],
+ FeedKey: f.Key,
+ }
+ if err := i.load(); err != nil {
+ t.Fatal(err)
+ }
+ if *i == (Item{}) {
+ t.Fatal(i)
+ }
+ if i.Link == "" || i.Title == "" || i.FeedKey == "" || i.Content == "" {
+ t.Fatal(i)
+ }
}
diff --git a/rss/item.go b/rss/item.go
index c21b42d..127fb7a 100644
--- a/rss/item.go
+++ b/rss/item.go
@@ -2,21 +2,91 @@ package rss
import (
"errors"
+ "fmt"
+ "io/ioutil"
"local/rssmon3/config"
+ "net/http"
+ "net/url"
+ "regexp"
+ "strings"
+ "time"
"github.com/mmcdole/gofeed"
)
+const nsItems = "nsItems"
+
type Item struct {
- *config.Encodable
+ Title string
+ Link string
+ Content string
+ TS time.Time
+ FeedKey string
}
-func newItem(i *gofeed.Item, contentFilter string) (*Item, error) {
- return &Item{
- Encodable: &config.Encodable{},
- }, errors.New("not impl")
+func newItem(i *gofeed.Item, contentFilter, feedKey string) (*Item, error) {
+ item := &Item{
+ Title: i.Title,
+ Link: i.Link,
+ Content: i.Content,
+ TS: latestTSPtr(i.UpdatedParsed, i.PublishedParsed),
+ FeedKey: feedKey,
+ }
+
+ if item.Content == "" {
+ item.Content = i.Description
+ }
+ if item.Content == "" {
+ resp, err := http.Get(item.Link)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ b, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ item.Content = string(b)
+ }
+ if unescaped, err := url.QueryUnescape(item.Content); err == nil {
+ item.Content = unescaped
+ }
+
+ if contentFilter == "" {
+ contentFilter = ".*"
+ }
+ item.Content = strings.Join(regexp.MustCompile(contentFilter).FindAllString(item.Content, -1), "
")
+
+ item.Content = fmt.Sprintf(`%s
%s`, item.Link, item.Title, item.Content)
+
+ return item, nil
+}
+
+func (i *Item) Encode() ([]byte, error) {
+ return config.Encode(i)
+}
+
+func (i *Item) Decode(b []byte) error {
+ return config.Decode(b, i)
}
func (i *Item) save() error {
- return errors.New("not impl")
+ db := config.Values().DB
+ b, err := i.Encode()
+ if err != nil {
+ return err
+ }
+ return db.Set(i.Link, b, nsItems, i.FeedKey)
+}
+
+func (i *Item) load() error {
+ if i.Link == "" || i.FeedKey == "" {
+ return errors.New("cannot load nil item")
+ }
+ db := config.Values().DB
+ b, err := db.Get(i.Link, nsItems, i.FeedKey)
+ if err != nil {
+ return err
+ }
+ return config.Decode(b, i)
}
diff --git a/rss/item_test.go b/rss/item_test.go
new file mode 100644
index 0000000..1d9c45d
--- /dev/null
+++ b/rss/item_test.go
@@ -0,0 +1,84 @@
+package rss
+
+import (
+ "fmt"
+ "net/http"
+ "testing"
+
+ "github.com/mmcdole/gofeed"
+)
+
+func initRSSItem() {
+ initRSSFeed()
+}
+
+func TestRSSItemNewEncodeDecode(t *testing.T) {
+ initRSSItem()
+
+ s := mockRSS()
+ defer s.Close()
+
+ resp, err := http.Get(s.URL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ gofeed, err := gofeed.NewParser().Parse(resp.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ gofeed.Items[0].Content = ""
+ gofeed.Items[0].Description = ""
+
+ item, err := newItem(gofeed.Items[0], ".*", "key")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ itemB, err := newItem(gofeed.Items[0], "Podcast", "key")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if itemB.Content == item.Content {
+ t.Errorf("content filter did not apply")
+ }
+
+ b, err := item.Encode()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := itemB.Decode(b); err != nil {
+ t.Fatal(err)
+ }
+ if itemB.Content != item.Content {
+ t.Errorf("decode found %v, want %v", itemB.Content, item.Content)
+ }
+
+ if err := item.save(); err != nil {
+ t.Fatal(err)
+ }
+
+ itemC := &Item{
+ Link: item.Link,
+ FeedKey: item.FeedKey,
+ }
+ if err := itemC.load(); err != nil {
+ t.Fatal(err)
+ }
+ if fmt.Sprintf("%v", item) != fmt.Sprintf("%v", itemC) {
+ t.Fatalf("%v != %v", item, itemC)
+ }
+
+ itemD := &Item{}
+ if err := itemD.load(); err == nil {
+ t.Fatal(err)
+ }
+
+ itemE := &Item{
+ Link: "nothing",
+ FeedKey: item.FeedKey,
+ }
+ if err := itemE.load(); err == nil {
+ t.Fatal(err)
+ }
+}
diff --git a/rss/testdata/rss.xml.REMOVED.git-id b/rss/testdata/rss.xml.REMOVED.git-id
new file mode 100644
index 0000000..bcc45df
--- /dev/null
+++ b/rss/testdata/rss.xml.REMOVED.git-id
@@ -0,0 +1 @@
+d36f6c94dfbbaaeac339bfaec9d0dd13b0ff099b
\ No newline at end of file
diff --git a/rssmon3.REMOVED.git-id b/rssmon3.REMOVED.git-id
index b802e9d..482b93d 100644
--- a/rssmon3.REMOVED.git-id
+++ b/rssmon3.REMOVED.git-id
@@ -1 +1 @@
-bf5140d93694bf40f561841cdee34065d4cc2455
\ No newline at end of file
+2ecdcaccf6c3f6135a21544a465ace01b7326ec2
\ No newline at end of file