package rss import ( "fmt" "io/ioutil" "local/rssmon3/config" "net/http" "net/url" "regexp" "strings" "time" "github.com/mmcdole/gofeed" ) const nsItems = "nsItems" type Item struct { Title string Link string Content string TS time.Time Copyright string } type Items []*Item func newItem(i *gofeed.Item, contentFilter, copyright string) (*Item, error) { item := &Item{ Title: i.Title, Link: i.Link, Content: i.Content, TS: latestTSPtr(i.UpdatedParsed, i.PublishedParsed), Copyright: copyright, } if item.Content == "" { item.Content = i.Description } if item.Content == "" { resp, err := http.Get(item.Link) if err != nil { return nil, err } defer resp.Body.Close() b, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err } item.Content = string(b) } for _, enclosure := range i.Enclosures { item.Content += fmt.Sprintf(`
%s`, enclosure.URL, enclosure.URL) } if unescaped, err := url.QueryUnescape(item.Content); err == nil { item.Content = unescaped } if contentFilter == "" { contentFilter = ".*" } item.Content = strings.Join(regexp.MustCompile(contentFilter).FindAllString(item.Content, -1), "
") item.Content = fmt.Sprintf(`%s
%s`, item.Link, item.Title, item.Content) item.Content = cleanSpecialCharacters(clearBlankLines(item.Content)) return item, nil } func (i *Item) Encode() ([]byte, error) { return config.Encode(i) } func (i *Item) Decode(b []byte) error { if err := config.Decode(b, i); err != nil { return err } i.Content = clearBlankLines(i.Content) return nil } func (i *Item) save(ns1 string, ns ...string) error { db := config.Values().DB b, err := i.Encode() if err != nil { return err } return db.Set(i.ID(), b, append([]string{nsItems, ns1}, ns...)...) } func (i *Item) ID() string { return fmt.Sprintf("%s:%s", i.TS.Format("2006-01-02-15-04"), i.Link) } func (i *Item) Load(key, ns1 string, ns ...string) error { db := config.Values().DB b, err := db.Get(key, append([]string{nsItems, ns1}, ns...)...) if err != nil { return err } err = config.Decode(b, i) if err == nil { i.Content = clearBlankLines(i.Content) } return err } func (is Items) Len() int { return len(is) } func (is Items) Less(i, j int) bool { return is[j].TS.Before(is[i].TS) } func (is Items) Swap(i, j int) { k := is[i] is[i] = is[j] is[j] = k } func cleanSpecialCharacters(s string) string { return strings.ToValidUTF8(s, "?") } func clearBlankLines(s string) string { r := regexp.MustCompile(`
`) s = r.ReplaceAllLiteralString(s, "
") r = regexp.MustCompile(`(?m)
\s*(
\s*)*`) s = r.ReplaceAllLiteralString(s, "
") r = regexp.MustCompile(`(?m)\s\s*`) return r.ReplaceAllLiteralString(s, "\n") }