From 1aadfc65b6967b6fddc22e7a56e4a3b73ccb05fb Mon Sep 17 00:00:00 2001 From: bel Date: Fri, 27 Dec 2019 20:04:00 -0700 Subject: [PATCH] Fix giant lists --- rss/feed.go | 3 ++- rss/feed_test.go | 34 ++++++++++++++++++++++++++++++++++ server/routes.go | 7 ++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/rss/feed.go b/rss/feed.go index d6108fc..144acfd 100755 --- a/rss/feed.go +++ b/rss/feed.go @@ -7,6 +7,7 @@ import ( "net/http" "regexp" "sort" + "strconv" "time" "github.com/mmcdole/gofeed" @@ -125,7 +126,7 @@ func (f *Feed) Items(limit int) ([]*Item, error) { } func (f *Feed) List(limit int) ([]string, error) { - keys, err := config.Values().DB.List([]string{nsItems, f.Key}) + keys, err := config.Values().DB.List([]string{nsItems, f.Key}, " ", "}}}}}", strconv.Itoa(limit), "-") if err != nil { return nil, err } diff --git a/rss/feed_test.go b/rss/feed_test.go index 6de0d19..882a715 100755 --- a/rss/feed_test.go +++ b/rss/feed_test.go @@ -177,3 +177,37 @@ func TestRSSFeedPull(t *testing.T) { t.Fatal(i) } } + +func TestRSSFeedListLimitedDescending(t *testing.T) { + initRSSFeed() + + s := mockRSS() + defer s.Close() + + f := newFeed("key") + f.TitleFilter = "50." + f.ContentFilter = "b" + f.Tags = []string{"c"} + f.URL = s.URL + + log.SetOutput(bytes.NewBuffer(nil)) + defer log.SetOutput(os.Stderr) + if err := f.Pull(); err != nil { + t.Fatal(err) + } + log.SetOutput(os.Stderr) + + itemKeys, err := f.List(5) + if err != nil { + t.Fatal(err) + } + if len(itemKeys) != 5 { + t.Fatal(len(itemKeys)) + } + + for i := range itemKeys { + if i > 0 && itemKeys[i] > itemKeys[i-1] { + t.Error(itemKeys[i], ">", itemKeys[i-1]) + } + } +} diff --git a/server/routes.go b/server/routes.go index 06a4b93..c6b83ad 100755 --- a/server/routes.go +++ b/server/routes.go @@ -9,6 +9,7 @@ import ( "log" "net/http" "regexp" + "strconv" "time" ) @@ -50,6 +51,10 @@ func (s *Server) tag(w http.ResponseWriter, r *http.Request) { s.notFound(w, r) return } + limit, err := strconv.Atoi(r.URL.Query().Get("n")) + if err != nil { + limit = 20 + } tag := regexp.MustCompile("^.*\\/").ReplaceAllString(r.URL.Path, "") feeds, err := rss.TaggedFeeds(tag) if err != nil { @@ -58,7 +63,7 @@ func (s *Server) tag(w http.ResponseWriter, r *http.Request) { } items := []*rss.Item{} for _, feed := range feeds { - feedItems, err := feed.Items(20) + feedItems, err := feed.Items(limit) if err != nil { s.error(w, r, err) }