rssmon3/rss/feed.go

package rss

import (
	"encoding/json"
	"errors"
	"local/rssmon3/config"
	"log"
	"net/http"
	"regexp"
	"sort"
	"strconv"
	"time"

	"github.com/mmcdole/gofeed"
)

const nsFeeds = "nsFeeds"

type Feed struct {
	Key           string
	URL           string
	Updated       time.Time
	TitleFilter   string
	ContentFilter string
	Tags          []string
	Copyright     string
	Headers       []byte
}

func SubmitFeed(f *Feed) error {
	return f.save()
}

func newFeed(key string) *Feed {
	return &Feed{
		Key: key,
	}
}

func (f *Feed) Encode() ([]byte, error) {
	return config.Encode(f)
}

func (f *Feed) Decode(b []byte) error {
	return config.Decode(b, f)
}

func (f *Feed) Load() error {
	if f.Key == "" {
		return errors.New("cannot load nil feed")
	}
	db := config.Values().DB
	b, err := db.Get(f.Key, nsFeeds)
	if err != nil {
		return err
	}
	return f.Decode(b)
}

func (f *Feed) Pull() error {
	if f.URL == "" {
		if err := f.Load(); err != nil {
			return err
		}
	}
	req, err := http.NewRequest(http.MethodGet, f.URL, nil)
	if err != nil {
		return err
	}
	var headers map[string]string
	json.Unmarshal(f.Headers, &headers)
	for k, v := range headers {
		req.Header.Set(k, v)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	gofeed, err := gofeed.NewParser().Parse(resp.Body)
	if err != nil {
		return err
	}

	itemTSs := []*time.Time{}
	for _, i := range gofeed.Items {
		item, err := newItem(i, f.ContentFilter, f.Copyright)
		if err != nil {
			log.Println("[Pull]", err)
			continue
		}
		itemTSs = append(itemTSs, &item.TS)
		if item.TS.Before(f.Updated) {
			log.Println("[Pull]", "Skipping old item")
			continue
		}
		if ok := regexp.MustCompile(f.TitleFilter).MatchString(item.Title); !ok {
			log.Printf("[Pull] Skipping bad titled item: %v doesn't match /%v/", item.Title, f.TitleFilter)
			continue
		}
		log.Printf("[Pull] Saving item %v for %v /%v/", f.Key, f.URL, f.TitleFilter)
		if err := item.save(f.Key); err != nil {
			log.Println("[Pull]", err)
			continue
		}
	}

	f.Updated = latestTSPtr(gofeed.PublishedParsed, gofeed.UpdatedParsed)
	f.Updated = latestTSPtr(append(itemTSs, &f.Updated)...)
	return nil
}

func (f *Feed) save() error {
	b, err := f.Encode()
	if err != nil {
		return err
	}
	db := config.Values().DB
	return db.Set(f.Key, b, nsFeeds)
}

func (f *Feed) Items(limit int) ([]*Item, error) {
	keys, err := f.List(limit)
	if err != nil {
		return nil, err
	}
	results := []*Item{}
	for j := range keys {
		i := &Item{}
		if err := i.Load(keys[j], f.Key); err != nil {
			return nil, err
		}
		results = append(results, i)
	}
	return results, nil
}

func (f *Feed) List(limit int) ([]string, error) {
	keys, err := config.Values().DB.List([]string{nsItems, f.Key}, " ", "}}}}}", strconv.Itoa(limit), "-")
	if err != nil {
		return nil, err
	}
	sorted := sort.StringSlice(keys)
	sorted.Sort()
	if len(sorted) > limit {
		sorted = sorted[len(sorted)-limit:]
	}
	for i := 0; i < len(sorted)/2; i++ {
		j := len(sorted) - 1 - i
		sorted.Swap(i, j)
	}
	return sorted, nil
}

func latestTSPtr(times ...*time.Time) time.Time {
	var t time.Time
	for i := range times {
		if times[i] == nil {
			continue
		}
		if times[i].After(t) {
			t = *times[i]
		}
	}
	return t
}