From fc94f94bd209d64bc0621640dec03a1d34d7090b Mon Sep 17 00:00:00 2001 From: bel Date: Sat, 22 Jun 2019 10:07:52 -0600 Subject: [PATCH] rss and rss item and encode Former-commit-id: 333b5635cc0eb22964403cc78f660e02a830f077 --- config/encodable.go | 22 ------ config/encode.go | 20 +++++ config/encode_test.go | 24 ++++++ monitor/item.go | 12 ++- rss/feed.go | 41 +++++----- rss/feed_test.go | 117 +++++++++++++++++++++++++++- rss/item.go | 82 +++++++++++++++++-- rss/item_test.go | 84 ++++++++++++++++++++ rss/testdata/rss.xml.REMOVED.git-id | 1 + rssmon3.REMOVED.git-id | 2 +- 10 files changed, 353 insertions(+), 52 deletions(-) delete mode 100644 config/encodable.go create mode 100644 config/encode.go create mode 100644 config/encode_test.go create mode 100644 rss/item_test.go create mode 100644 rss/testdata/rss.xml.REMOVED.git-id diff --git a/config/encodable.go b/config/encodable.go deleted file mode 100644 index 4bf3331..0000000 --- a/config/encodable.go +++ /dev/null @@ -1,22 +0,0 @@ -package config - -import ( - "bytes" - "encoding/gob" -) - -type Encodable struct{} - -func (e *Encodable) Encode() ([]byte, error) { - buff := bytes.NewBuffer(nil) - enc := gob.NewEncoder(buff) - err := enc.Encode(e) - return buff.Bytes(), err -} - -func (e *Encodable) Decode(b []byte) error { - buff := bytes.NewBuffer(b) - dec := gob.NewDecoder(buff) - err := dec.Decode(e) - return err -} diff --git a/config/encode.go b/config/encode.go new file mode 100644 index 0000000..d9f4b3d --- /dev/null +++ b/config/encode.go @@ -0,0 +1,20 @@ +package config + +import ( + "bytes" + "encoding/gob" +) + +func Encode(v interface{}) ([]byte, error) { + buff := bytes.NewBuffer(nil) + enc := gob.NewEncoder(buff) + err := enc.Encode(v) + return buff.Bytes(), err +} + +func Decode(b []byte, v interface{}) error { + buff := bytes.NewBuffer(b) + enc := gob.NewDecoder(buff) + err := enc.Decode(v) + return err +} diff --git a/config/encode_test.go b/config/encode_test.go new file mode 100644 index 0000000..8ceed45 --- /dev/null +++ b/config/encode_test.go @@ -0,0 +1,24 @@ +package config + +import ( + "testing" + "time" +) + +func TestEncodeDecode(t *testing.T) { + c := time.Now() + d := c.Add(0) + + b, err := Encode(c) + if err != nil { + t.Fatal(err) + } + + if err := Decode(b, &c); err != nil { + t.Fatal(err) + } + + if c.Sub(d) > time.Millisecond*5 { + t.Fatalf("%v => %v", d, c) + } +} diff --git a/monitor/item.go b/monitor/item.go index 7c54837..b34ff31 100644 --- a/monitor/item.go +++ b/monitor/item.go @@ -12,7 +12,6 @@ import ( type Item struct { Key string - *config.Encodable } const nsLast = "nsLast" @@ -23,8 +22,7 @@ var forever = time.Duration(time.Hour * 99999) func NewItem(key string, interval time.Duration) (*Item, error) { i := &Item{ - Key: key, - Encodable: &config.Encodable{}, + Key: key, } if err := i.setInterval(interval); err != nil { @@ -124,3 +122,11 @@ func (i *Item) getLast() (time.Time, error) { } return t, nil } + +func (i *Item) Encode() ([]byte, error) { + return config.Encode(i) +} + +func (i *Item) Decode(b []byte) error { + return config.Decode(b, i) +} diff --git a/rss/feed.go b/rss/feed.go index 166c8c5..1b741c7 100644 --- a/rss/feed.go +++ b/rss/feed.go @@ -13,24 +13,30 @@ import ( const NSFeeds = "NSFeeds" -type feed struct { +type Feed struct { Key string URL string Updated time.Time TitleFilter string ContentFilter string Tags []string - *config.Encodable } -func newFeed(key string) *feed { - return &feed{ - Key: key, - Encodable: &config.Encodable{}, +func newFeed(key string) *Feed { + return &Feed{ + Key: key, } } -func (f *feed) load() error { +func (f *Feed) Encode() ([]byte, error) { + return config.Encode(f) +} + +func (f *Feed) Decode(b []byte) error { + return config.Decode(b, f) +} + +func (f *Feed) load() error { if f.Key == "" { return errors.New("cannot load nil feed") } @@ -42,7 +48,7 @@ func (f *feed) load() error { return f.Decode(b) } -func (f *feed) pull() error { +func (f *Feed) pull() error { if f.URL == "" { if err := f.load(); err != nil { return err @@ -61,21 +67,20 @@ func (f *feed) pull() error { itemTSs := []*time.Time{} for _, i := range gofeed.Items { - ts := latestTSPtr(i.UpdatedParsed, i.PublishedParsed) - itemTSs = append(itemTSs, &ts) - if ts.Before(f.Updated) { + item, err := newItem(i, f.ContentFilter, f.Key) + if err != nil { + log.Println(err) + continue + } + itemTSs = append(itemTSs, &item.TS) + if item.TS.Before(f.Updated) { log.Println("Skipping old item") continue } - if ok := regexp.MustCompile(f.TitleFilter).MatchString(i.Title); !ok { + if ok := regexp.MustCompile(f.TitleFilter).MatchString(item.Title); !ok { log.Println("Skipping bad titled item") continue } - item, err := newItem(i, f.ContentFilter) - if err != nil { - log.Println(err) - continue - } if err := item.save(); err != nil { log.Println(err) continue @@ -87,7 +92,7 @@ func (f *feed) pull() error { return nil } -func (f *feed) save() error { +func (f *Feed) save() error { b, err := f.Encode() if err != nil { return err diff --git a/rss/feed_test.go b/rss/feed_test.go index e603ae6..076cce4 100644 --- a/rss/feed_test.go +++ b/rss/feed_test.go @@ -1,9 +1,17 @@ package rss import ( + "bytes" "encoding/json" + "fmt" + "io" "local/rssmon3/config" + "log" + "net/http" + "net/http/httptest" "os" + "path" + "reflect" "testing" "time" ) @@ -15,6 +23,20 @@ func initRSSFeed() { } } +type mockStruct struct{} + +func mockRSS() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var v mockStruct + base := reflect.TypeOf(v).PkgPath() + f, err := os.Open(path.Join(os.Getenv("GOPATH"), "src", base, "./testdata/rss.xml")) + if err != nil { + panic(err) + } + io.Copy(w, f) + })) +} + func TestRSSFeedEncodeDecode(t *testing.T) { initRSSFeed() @@ -39,6 +61,97 @@ func TestRSSFeedEncodeDecode(t *testing.T) { if string(fb) != string(gb) { t.Errorf("%v => %v", *f, *g) } - - t.Logf("%s vs. %s", fb, gb) +} + +func TestRSSFeedSaveLoad(t *testing.T) { + initRSSFeed() + + f := newFeed("key") + f.Updated = time.Now() + f.TitleFilter = "a" + f.ContentFilter = "b" + f.Tags = []string{"c"} + + if err := f.save(); err != nil { + t.Fatal(err) + } + + g := newFeed("key") + if err := g.load(); err != nil { + t.Fatal(err) + } + + fb, _ := json.Marshal(f) + gb, _ := json.Marshal(g) + if string(fb) != string(gb) { + t.Fatalf("%v != %v", f, g) + } + + h := newFeed("key2") + if err := h.load(); err == nil { + t.Fatal("can load nil feed") + } +} + +func TestRSSFeedPull(t *testing.T) { + initRSSFeed() + + s := mockRSS() + defer s.Close() + + f := newFeed("key") + f.TitleFilter = "500" + f.ContentFilter = "b" + f.Tags = []string{"c"} + f.URL = s.URL + + log.SetOutput(bytes.NewBuffer(nil)) + defer log.SetOutput(os.Stderr) + if err := f.pull(); err != nil { + t.Fatal(err) + } + log.SetOutput(os.Stderr) + + if f.Updated.Format("2006-01-02 15:04:05") != "2019-06-18 19:00:00" { + t.Errorf("updated is wrong: %v", f) + } + + keys, err := config.Values().DB.List([]string{nsItems, f.Key}) + if err != nil { + t.Fatal(err) + } + if len(keys) < 1 { + t.Fatal(len(keys)) + } + if keys[0] != "https://roosterteeth.com/episode/rooster-teeth-podcast-2018-rooster-teeth-podcast-500" { + t.Fatal(keys[0]) + } + + log.SetOutput(bytes.NewBuffer(nil)) + defer log.SetOutput(os.Stderr) + if err := f.pull(); err != nil { + t.Fatal(err) + } + log.SetOutput(os.Stderr) + keysB, err := config.Values().DB.List([]string{nsItems, f.Key}) + if err != nil { + t.Fatal(err) + } + if fmt.Sprintf("%v", keys) != fmt.Sprintf("%v", keysB) { + t.Fatalf("%v vs %v", keys, keysB) + } + + i := &Item{ + Link: keys[0], + FeedKey: f.Key, + } + if err := i.load(); err != nil { + t.Fatal(err) + } + if *i == (Item{}) { + t.Fatal(i) + } + if i.Link == "" || i.Title == "" || i.FeedKey == "" || i.Content == "" { + t.Fatal(i) + } } diff --git a/rss/item.go b/rss/item.go index c21b42d..127fb7a 100644 --- a/rss/item.go +++ b/rss/item.go @@ -2,21 +2,91 @@ package rss import ( "errors" + "fmt" + "io/ioutil" "local/rssmon3/config" + "net/http" + "net/url" + "regexp" + "strings" + "time" "github.com/mmcdole/gofeed" ) +const nsItems = "nsItems" + type Item struct { - *config.Encodable + Title string + Link string + Content string + TS time.Time + FeedKey string } -func newItem(i *gofeed.Item, contentFilter string) (*Item, error) { - return &Item{ - Encodable: &config.Encodable{}, - }, errors.New("not impl") +func newItem(i *gofeed.Item, contentFilter, feedKey string) (*Item, error) { + item := &Item{ + Title: i.Title, + Link: i.Link, + Content: i.Content, + TS: latestTSPtr(i.UpdatedParsed, i.PublishedParsed), + FeedKey: feedKey, + } + + if item.Content == "" { + item.Content = i.Description + } + if item.Content == "" { + resp, err := http.Get(item.Link) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + item.Content = string(b) + } + if unescaped, err := url.QueryUnescape(item.Content); err == nil { + item.Content = unescaped + } + + if contentFilter == "" { + contentFilter = ".*" + } + item.Content = strings.Join(regexp.MustCompile(contentFilter).FindAllString(item.Content, -1), "
") + + item.Content = fmt.Sprintf(`%s
%s`, item.Link, item.Title, item.Content) + + return item, nil +} + +func (i *Item) Encode() ([]byte, error) { + return config.Encode(i) +} + +func (i *Item) Decode(b []byte) error { + return config.Decode(b, i) } func (i *Item) save() error { - return errors.New("not impl") + db := config.Values().DB + b, err := i.Encode() + if err != nil { + return err + } + return db.Set(i.Link, b, nsItems, i.FeedKey) +} + +func (i *Item) load() error { + if i.Link == "" || i.FeedKey == "" { + return errors.New("cannot load nil item") + } + db := config.Values().DB + b, err := db.Get(i.Link, nsItems, i.FeedKey) + if err != nil { + return err + } + return config.Decode(b, i) } diff --git a/rss/item_test.go b/rss/item_test.go new file mode 100644 index 0000000..1d9c45d --- /dev/null +++ b/rss/item_test.go @@ -0,0 +1,84 @@ +package rss + +import ( + "fmt" + "net/http" + "testing" + + "github.com/mmcdole/gofeed" +) + +func initRSSItem() { + initRSSFeed() +} + +func TestRSSItemNewEncodeDecode(t *testing.T) { + initRSSItem() + + s := mockRSS() + defer s.Close() + + resp, err := http.Get(s.URL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + gofeed, err := gofeed.NewParser().Parse(resp.Body) + if err != nil { + t.Fatal(err) + } + gofeed.Items[0].Content = "" + gofeed.Items[0].Description = "" + + item, err := newItem(gofeed.Items[0], ".*", "key") + if err != nil { + t.Fatal(err) + } + + itemB, err := newItem(gofeed.Items[0], "Podcast", "key") + if err != nil { + t.Fatal(err) + } + if itemB.Content == item.Content { + t.Errorf("content filter did not apply") + } + + b, err := item.Encode() + if err != nil { + t.Fatal(err) + } + if err := itemB.Decode(b); err != nil { + t.Fatal(err) + } + if itemB.Content != item.Content { + t.Errorf("decode found %v, want %v", itemB.Content, item.Content) + } + + if err := item.save(); err != nil { + t.Fatal(err) + } + + itemC := &Item{ + Link: item.Link, + FeedKey: item.FeedKey, + } + if err := itemC.load(); err != nil { + t.Fatal(err) + } + if fmt.Sprintf("%v", item) != fmt.Sprintf("%v", itemC) { + t.Fatalf("%v != %v", item, itemC) + } + + itemD := &Item{} + if err := itemD.load(); err == nil { + t.Fatal(err) + } + + itemE := &Item{ + Link: "nothing", + FeedKey: item.FeedKey, + } + if err := itemE.load(); err == nil { + t.Fatal(err) + } +} diff --git a/rss/testdata/rss.xml.REMOVED.git-id b/rss/testdata/rss.xml.REMOVED.git-id new file mode 100644 index 0000000..bcc45df --- /dev/null +++ b/rss/testdata/rss.xml.REMOVED.git-id @@ -0,0 +1 @@ +d36f6c94dfbbaaeac339bfaec9d0dd13b0ff099b \ No newline at end of file diff --git a/rssmon3.REMOVED.git-id b/rssmon3.REMOVED.git-id index b802e9d..482b93d 100644 --- a/rssmon3.REMOVED.git-id +++ b/rssmon3.REMOVED.git-id @@ -1 +1 @@ -bf5140d93694bf40f561841cdee34065d4cc2455 \ No newline at end of file +2ecdcaccf6c3f6135a21544a465ace01b7326ec2 \ No newline at end of file