package rss import ( "errors" "local/rssmon3/config" "log" "net/http" "regexp" "sort" "time" "github.com/mmcdole/gofeed" ) const nsFeeds = "nsFeeds" type Feed struct { Key string URL string Updated time.Time TitleFilter string ContentFilter string Tags []string Copyright string } func SubmitFeed(f *Feed) error { return f.save() } func newFeed(key string) *Feed { return &Feed{ Key: key, } } func (f *Feed) Encode() ([]byte, error) { return config.Encode(f) } func (f *Feed) Decode(b []byte) error { return config.Decode(b, f) } func (f *Feed) Load() error { if f.Key == "" { return errors.New("cannot load nil feed") } db := config.Values().DB b, err := db.Get(f.Key, nsFeeds) if err != nil { return err } return f.Decode(b) } func (f *Feed) Pull() error { if f.URL == "" { if err := f.Load(); err != nil { return err } } resp, err := http.Get(f.URL) if err != nil { return err } defer resp.Body.Close() gofeed, err := gofeed.NewParser().Parse(resp.Body) if err != nil { return err } itemTSs := []*time.Time{} for _, i := range gofeed.Items { item, err := newItem(i, f.ContentFilter, f.Copyright) if err != nil { log.Println("[Pull]", err) continue } itemTSs = append(itemTSs, &item.TS) if item.TS.Before(f.Updated) { log.Println("[Pull]", "Skipping old item") continue } if ok := regexp.MustCompile(f.TitleFilter).MatchString(item.Title); !ok { log.Printf("[Pull] Skipping bad titled item: %v doesn't match /%v/", item.Title, f.TitleFilter) continue } log.Printf("[Pull] Saving item %v for %v /%v/", f.Key, f.URL, f.TitleFilter) if err := item.save(f.Key); err != nil { log.Println("[Pull]", err) continue } } f.Updated = latestTSPtr(gofeed.PublishedParsed, gofeed.UpdatedParsed) f.Updated = latestTSPtr(append(itemTSs, &f.Updated)...) return nil } func (f *Feed) save() error { b, err := f.Encode() if err != nil { return err } db := config.Values().DB return db.Set(f.Key, b, nsFeeds) } func (f *Feed) Items(limit int) ([]*Item, error) { keys, err := f.List(limit) if err != nil { return nil, err } results := []*Item{} for j := range keys { i := &Item{} if err := i.Load(keys[j], f.Key); err != nil { return nil, err } results = append(results, i) } return results, nil } func (f *Feed) List(limit int) ([]string, error) { keys, err := config.Values().DB.List([]string{nsItems, f.Key}) if err != nil { return nil, err } sorted := sort.StringSlice(keys) sorted.Sort() if len(sorted) > limit { sorted = sorted[len(sorted)-limit:] } for i := 0; i < len(sorted)/2; i++ { j := len(sorted) - 1 - i sorted.Swap(i, j) } return sorted, nil } func latestTSPtr(times ...*time.Time) time.Time { var t time.Time for i := range times { if times[i] == nil { continue } if times[i].After(t) { t = *times[i] } } return t }