package rss import ( "bytes" "encoding/gob" "fmt" "io/ioutil" "local/logger" "net/http" "regexp" "strings" "time" "github.com/mmcdole/gofeed" ) type Feed struct { Updated time.Time Title string Items []string Tags []string ItemFilter string ContentFilter string Link string Interval time.Duration } func (feed *Feed) String() string { return fmt.Sprintf("Title: %s, Updated: %v, ItemFilter: %q, ContentFilter: %q, Link: %v, Interval: %s, Tags: %v", feed.Title, feed.Updated.Local(), feed.ItemFilter, feed.ContentFilter, feed.Link, feed.Interval, feed.Tags, ) } func (feed *Feed) ID() string { return strings.Join(regexp.MustCompile("[a-zA-Z0-9]*").FindAllString(feed.Link, -1), "_") } func New(source, itemFilter, contentFilter string, tags []string, interval time.Duration) (*Feed, error) { if _, err := regexp.Compile(itemFilter); err != nil { return nil, err } if _, err := regexp.Compile(contentFilter); err != nil { return nil, err } f := &Feed{ Items: []string{}, ItemFilter: itemFilter, ContentFilter: contentFilter, Link: source, Tags: tags, Interval: interval, } return f, nil } func Deserialize(src []byte) (*Feed, error) { buffer := bytes.NewBuffer(src) dec := gob.NewDecoder(buffer) var dst Feed err := dec.Decode(&dst) return &dst, err } func (feed *Feed) Serialize() ([]byte, error) { var buffer bytes.Buffer enc := gob.NewEncoder(&buffer) err := enc.Encode(feed) return buffer.Bytes(), err } func (feed *Feed) Update() ([]*Item, error) { resp, err := http.Get(feed.Link) if err != nil { return nil, err } body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err } parser := gofeed.NewParser() gofeed, err := parser.Parse(bytes.NewBuffer(body)) if err != nil { return nil, err } return feed.fromGofeed(gofeed) } func (feed *Feed) fromGofeed(gofeed *gofeed.Feed) ([]*Item, error) { updated := gofeed.PublishedParsed if updated == nil { updated = gofeed.UpdatedParsed } if updated == nil && len(gofeed.Items) > 0 { updated = gofeedItemTS(gofeed.Items[len(gofeed.Items)-1]) } if updated == nil { t := time.Now() updated = &t } newitems, latest, err := feed.appendNewItems(gofeed.Items) if err != nil { return nil, err } feed.Updated = latest.Add(time.Second) //time.Now() //*updated //time.Now().UTC() //*updated feed.Title = gofeed.Title return newitems, nil } func (feed *Feed) appendNewItems(items []*gofeed.Item) ([]*Item, time.Time, error) { latest := feed.Updated newitems := []*Item{} for i := range items { t := gofeedItemTS(items[i]) if *t != (time.Time{}) && t.Before(feed.Updated) { logger.Logf("\tnot adding %v because its timestamp before %v", items[i].Link, feed.Updated) continue } if t.After(latest) { latest = *t } if ok, _ := regexp.MatchString(feed.ItemFilter, items[i].Title); !ok { logger.Logf("\tnot adding %v because its title doesnt match item filter %v", items[i].Link, feed.ItemFilter) continue } item := FromGofeedItem(items[i], feed.ContentFilter) newitems = append(newitems, item) } logger.Logf("\tnew latest is %v", latest) return newitems, latest, nil }