From 35b3ff2c2d9b69c38248370b5000a913516be15a Mon Sep 17 00:00:00 2001 From: bel Date: Sat, 22 Jun 2019 16:23:27 -0600 Subject: [PATCH] Missing handlers but there we goddamn GO Former-commit-id: 2505146a54acaf18eadfdebf1dd2720889aef795 --- handlers/handler.go | 52 +++++++++++++++++++++++++++ handlers/lookup.go | 12 +++++++ handlers/podcast.go | 7 ++++ handlers/torrent.go | 7 ++++ main.go | 16 +++++---- monitor/.monitor_test.go.swp | Bin 12288 -> 0 bytes monitor/item.go | 4 +-- monitor/item_test.go | 4 +-- monitor/monitor.go | 3 ++ monitor/queue_test.go | 2 +- rss/feed.go | 2 +- rss/feed_test.go | 4 +-- rss/item.go | 16 +++++++++ rss/rss.go | 2 +- rss/serialize.go | 5 ++- rss/serialize_test.go | 2 +- rssmon3.REMOVED.git-id | 2 +- server/new.go | 5 ++- server/new_test.go | 2 +- server/routes.go | 22 ++++++------ server/routes_test.go | 6 ++-- server/server_test.go | 2 +- testdata/2add | 68 +++++++++++++++++++++++++++++++++++ testdata/2do | 8 +++-- 24 files changed, 216 insertions(+), 37 deletions(-) create mode 100644 handlers/handler.go create mode 100644 handlers/lookup.go create mode 100644 handlers/podcast.go create mode 100644 handlers/torrent.go delete mode 100644 monitor/.monitor_test.go.swp create mode 100644 testdata/2add diff --git a/handlers/handler.go b/handlers/handler.go new file mode 100644 index 0000000..a31ad40 --- /dev/null +++ b/handlers/handler.go @@ -0,0 +1,52 @@ +package handlers + +import ( + "local/rssmon3/config" + "local/rssmon3/monitor" + "local/rssmon3/rss" + "log" +) + +type Handler struct { + Jobs <-chan *monitor.Item + config.Stoppable +} + +func New(jobs <-chan *monitor.Item) *Handler { + return &Handler{ + Jobs: jobs, + } +} + +func (h *Handler) Run() error { + for { + select { + case <-h.Stopped(): + return nil + case j := <-h.Jobs: + go func(key string) { + if err := h.Job(key); err != nil { + log.Println(err) + } + }(j.Key) + } + } +} + +func (h *Handler) Job(key string) error { + f := &rss.Feed{Key: key} + if err := f.Load(); err != nil { + return err + } + if err := f.Pull(); err != nil { + return err + } + for _, tag := range f.Tags { + if foo := ByTag(tag); foo != nil { + if err := foo(key); err != nil { + return err + } + } + } + return nil +} diff --git a/handlers/lookup.go b/handlers/lookup.go new file mode 100644 index 0000000..ea98caa --- /dev/null +++ b/handlers/lookup.go @@ -0,0 +1,12 @@ +package handlers + +func ByTag(tag string) func(string) error { + var foo func(string) error + switch tag { + case "torrent": + foo = torrent + case "podcast": + foo = podcast + } + return foo +} diff --git a/handlers/podcast.go b/handlers/podcast.go new file mode 100644 index 0000000..b21ad37 --- /dev/null +++ b/handlers/podcast.go @@ -0,0 +1,7 @@ +package handlers + +import "errors" + +func podcast(key string) error { + return errors.New("not impl") +} diff --git a/handlers/torrent.go b/handlers/torrent.go new file mode 100644 index 0000000..e1b7c2f --- /dev/null +++ b/handlers/torrent.go @@ -0,0 +1,7 @@ +package handlers + +import "errors" + +func torrent(key string) error { + return errors.New("not impl") +} diff --git a/main.go b/main.go index 2e486ee..dd64c02 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "local/rssmon3/config" + "local/rssmon3/handlers" "local/rssmon3/monitor" "local/rssmon3/server" "log" @@ -21,18 +22,21 @@ func main() { } log.Println(config.Values()) - s := server.New() - if err := s.Routes(); err != nil { - panic(err) - } - go InterruptAfter(s.Run, sigc) - m, err := monitor.New() if err != nil { panic(err) } go InterruptAfter(m.Run, sigc) + s := server.New(m.Incoming) + if err := s.Routes(); err != nil { + panic(err) + } + go InterruptAfter(s.Run, sigc) + + h := handlers.New(m.Outgoing) + go InterruptAfter(h.Run, sigc) + signal.Notify(sigc, syscall.SIGHUP, syscall.SIGINT, diff --git a/monitor/.monitor_test.go.swp b/monitor/.monitor_test.go.swp deleted file mode 100644 index d16c030d2456c6984bf5dc9ec90cd4b2bd94ab25..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2UuYaf9LFbp5H~fo)CUP7OjcU*}c15(|W1+VwrcI#(*(k3>X8(fH7bU7z4(DG4O~OknIujCf2eiUr&^`BlooVOVbzw#(*(k z3>X8(fH7bU7z4(DF<=ZB1IEDr(0~^Za_2EZK6xC$;VM)wGVTF8{ix81$YN^z!|Uz=0OcS14hB0V}$$x zegU_^XW$BW8?-?PHb4_BfGVhf32^rbLcRvq!8LFhyboRn=fEpq8teySU@sU2KkOys z7Ptw%0v~|{2(S)Tz!Z2EjDz1su`X~M+yXbj$KXTo0eBa5Knl)-C9nYIK@E(9QDA}J zEkeEr--1uURqz&w!EsOpPlGXFfuE4KAHfx%a(ES(hcRFb7z4(DG4Kc(*dms6j|mZr zx?}Umk3$|cD3>gx0S_WcStOY#3#ME8YRXb(7l?)HEXhOWo^CPLcJ$K~-fHoL`EeA` zNm{L-o9<@`CQ}jN6|d9K)nm9PSmYFFtwyVKlUVgw&?cR&(aKS@b94sH)Rg+#>scghbQH_S)26Oy%FQSubk|M3D9{rhd0V2UYnEB** zT~8yQuHrt-emst(lj_b)jOnfxuwnwW+HRH>XFwQc*x_&@bEu+Yk(TQmQUPslJ#;+B z^6Q7hgi^_83i2w!-Kwf8d@o^iuA~Jo)=@ZFVVu^k3$xvN$Scm_xJ0doyT{pU6zk~b zDUTztRR!olxk~3;skg+yQNtyy#eAux-BSaZ0E{}jb4Fsg zl`)lqHyTW^K$(MutSqqynUhFQ$O4_;UaCq3LLaI)?-07Z@ds5|dT^yvPk1uEqzkoM zW~b6*T_=XS36m$hL}oeE{oKVM$X>n>rvjV2R4u6B`$a#n-Bx=m9N2WY;^DSQZ=7L% zou;cJ)P`B4jaiiMuEo3A~=WkRS(9W7GDqM!OWOsV4N= z_EvTV4;;(t{J`REwI6KNX2-F58Yk-S;C=O32sYtiI~I~U_|5G38(xFaF#DDve*@Zd Bn9u+K diff --git a/monitor/item.go b/monitor/item.go index ee0024d..67c35c5 100644 --- a/monitor/item.go +++ b/monitor/item.go @@ -40,9 +40,9 @@ func (i *Item) Compare(other queue.Item) int { iNext := i.Last().Add(i.Interval()) jNext := j.Last().Add(j.Interval()) if iNext.Before(jNext) { - return 1 - } else if jNext.Before(iNext) { return -1 + } else if jNext.Before(iNext) { + return 1 } else { return 0 } diff --git a/monitor/item_test.go b/monitor/item_test.go index 6ee9315..63bb6ae 100644 --- a/monitor/item_test.go +++ b/monitor/item_test.go @@ -31,7 +31,7 @@ func TestItemCompare(t *testing.T) { i, _ := NewItem("iCompare", time.Second) j, _ := NewItem("jCompare", 10*time.Second) - if c := i.Compare(j); c != -1 { + if c := i.Compare(j); c != 1 { t.Fatal(c) } @@ -43,7 +43,7 @@ func TestItemCompare(t *testing.T) { i, _ = NewItem("iCompare", 10*time.Second) j, _ = NewItem("jCompare", time.Second) - if c := i.Compare(j); c != 1 { + if c := i.Compare(j); c != -1 { t.Fatal(c) } } diff --git a/monitor/monitor.go b/monitor/monitor.go index d1d10d7..57b6819 100644 --- a/monitor/monitor.go +++ b/monitor/monitor.go @@ -2,6 +2,7 @@ package monitor import ( "local/rssmon3/config" + "log" "time" ) @@ -51,6 +52,8 @@ func (m *Monitor) triggered() <-chan time.Time { if top == nil { return nil } + block := time.Until(top.Last().Add(top.Interval())) + log.Printf("blocking %v until next task", block) return time.After(time.Until(top.Last().Add(top.Interval()))) } diff --git a/monitor/queue_test.go b/monitor/queue_test.go index c3bc8b6..aeb1518 100644 --- a/monitor/queue_test.go +++ b/monitor/queue_test.go @@ -39,7 +39,7 @@ func TestQueuePriority(t *testing.T) { j, _ := NewItem("jQueuePriority", time.Hour) q.Push(i) q.Push(j) - if k := q.Peek(); k.Key != "iQueuePriority" { + if k := q.Peek(); k.Key != "jQueuePriority" { t.Errorf("compare is backwards") } diff --git a/rss/feed.go b/rss/feed.go index e934dbd..c83c116 100644 --- a/rss/feed.go +++ b/rss/feed.go @@ -53,7 +53,7 @@ func (f *Feed) Load() error { return f.Decode(b) } -func (f *Feed) pull() error { +func (f *Feed) Pull() error { if f.URL == "" { if err := f.Load(); err != nil { return err diff --git a/rss/feed_test.go b/rss/feed_test.go index ae0c6ec..6de0d19 100644 --- a/rss/feed_test.go +++ b/rss/feed_test.go @@ -107,7 +107,7 @@ func TestRSSFeedPull(t *testing.T) { log.SetOutput(bytes.NewBuffer(nil)) defer log.SetOutput(os.Stderr) - if err := f.pull(); err != nil { + if err := f.Pull(); err != nil { t.Fatal(err) } log.SetOutput(os.Stderr) @@ -129,7 +129,7 @@ func TestRSSFeedPull(t *testing.T) { log.SetOutput(bytes.NewBuffer(nil)) defer log.SetOutput(os.Stderr) - if err := f.pull(); err != nil { + if err := f.Pull(); err != nil { t.Fatal(err) } log.SetOutput(os.Stderr) diff --git a/rss/item.go b/rss/item.go index ec15d0f..51782c7 100644 --- a/rss/item.go +++ b/rss/item.go @@ -22,6 +22,8 @@ type Item struct { TS time.Time } +type Items []*Item + func newItem(i *gofeed.Item, contentFilter string) (*Item, error) { item := &Item{ Title: i.Title, @@ -88,3 +90,17 @@ func (i *Item) Load(key, ns1 string, ns ...string) error { } return config.Decode(b, i) } + +func (is Items) Len() int { + return len(is) +} + +func (is Items) Less(i, j int) bool { + return is[j].TS.Before(is[i].TS) +} + +func (is Items) Swap(i, j int) { + k := is[i] + is[i] = is[j] + is[j] = k +} diff --git a/rss/rss.go b/rss/rss.go index 7bc40a6..e494e5b 100644 --- a/rss/rss.go +++ b/rss/rss.go @@ -32,7 +32,7 @@ func (rss *RSS) Run() error { func (rss *RSS) update(item *monitor.Item) error { f := newFeed(item.Key) - if err := f.pull(); err != nil { + if err := f.Pull(); err != nil { return err } if err := f.save(); err != nil { diff --git a/rss/serialize.go b/rss/serialize.go index fe0323c..e3ad4b1 100644 --- a/rss/serialize.go +++ b/rss/serialize.go @@ -1,8 +1,10 @@ package rss import ( + "fmt" "io" "local/rssmon3/config" + "sort" "github.com/gorilla/feeds" ) @@ -29,6 +31,7 @@ func TaggedFeeds(tag string) ([]*Feed, error) { } func WriteFeed(w io.Writer, tag string, items []*Item) error { + sort.Sort(Items(items)) feed := &feeds.Feed{ Title: tag, Link: &feeds.Link{}, @@ -47,6 +50,6 @@ func WriteFeed(w io.Writer, tag string, items []*Item) error { if err != nil { return err } - w.Write([]byte(s)) + fmt.Fprintf(w, "%s\n", s) return nil } diff --git a/rss/serialize_test.go b/rss/serialize_test.go index 43e7085..19f91d3 100644 --- a/rss/serialize_test.go +++ b/rss/serialize_test.go @@ -50,7 +50,7 @@ func TestRSSWriteFeed(t *testing.T) { log.SetOutput(bytes.NewBuffer(nil)) defer log.SetOutput(os.Stderr) - if err := f.pull(); err != nil { + if err := f.Pull(); err != nil { t.Fatal(err) } log.SetOutput(os.Stderr) diff --git a/rssmon3.REMOVED.git-id b/rssmon3.REMOVED.git-id index c5ebaf3..9f7ae2a 100644 --- a/rssmon3.REMOVED.git-id +++ b/rssmon3.REMOVED.git-id @@ -1 +1 @@ -cc79df3e2bb33d684975d23cd008eae93e7dffc2 \ No newline at end of file +49c1645f536cea7463202894e751295a3ec589f1 \ No newline at end of file diff --git a/server/new.go b/server/new.go index 3fe393c..de0862f 100644 --- a/server/new.go +++ b/server/new.go @@ -3,19 +3,22 @@ package server import ( "local/router" "local/rssmon3/config" + "local/rssmon3/monitor" "strings" ) type Server struct { Port string router *router.Router + Jobs chan<- *monitor.Item } -func New() *Server { +func New(jobs chan<- *monitor.Item) *Server { config := config.Values() s := &Server{ Port: ":" + strings.TrimPrefix(config.Port, ":"), router: router.New(), + Jobs: jobs, } return s } diff --git a/server/new_test.go b/server/new_test.go index da51979..6998594 100644 --- a/server/new_test.go +++ b/server/new_test.go @@ -10,5 +10,5 @@ func TestNew(t *testing.T) { os.Setenv("DB", "MAP") os.Args = []string{"ok"} config.New() - New() + New(nil) } diff --git a/server/routes.go b/server/routes.go index 6aee5b2..478c86f 100644 --- a/server/routes.go +++ b/server/routes.go @@ -2,7 +2,6 @@ package server import ( "encoding/json" - "errors" "fmt" "local/router" "local/rssmon3/monitor" @@ -77,16 +76,21 @@ func (s *Server) feed(w http.ResponseWriter, r *http.Request) { return } var putFeed struct { - URL string `json:"url"` - Interval time.Duration `json:"refresh"` - TitleFilter string `json:"items"` - ContentFilter string `json:"content"` - Tags []string `json:"tags"` + URL string `json:"url"` + Interval string `json:"refresh"` + TitleFilter string `json:"items"` + ContentFilter string `json:"content"` + Tags []string `json:"tags"` } if err := json.NewDecoder(r.Body).Decode(&putFeed); err != nil { s.userError(w, r, err) return } + interval, err := time.ParseDuration(putFeed.Interval) + if err != nil { + s.userError(w, r, err) + return + } f := &rss.Feed{ Key: putFeed.URL, URL: putFeed.URL, @@ -98,12 +102,10 @@ func (s *Server) feed(w http.ResponseWriter, r *http.Request) { s.error(w, r, err) return } - i, err := monitor.NewItem(f.Key, putFeed.Interval) + i, err := monitor.NewItem(f.Key, interval) if err != nil { s.error(w, r, err) return } - log.Println(i) - s.error(w, r, errors.New("not impl")) - // TODO + s.Jobs <- i } diff --git a/server/routes_test.go b/server/routes_test.go index a1c0028..f1e2b78 100644 --- a/server/routes_test.go +++ b/server/routes_test.go @@ -19,7 +19,7 @@ func TestRoutes(t *testing.T) { if err := config.New(); err != nil { t.Fatal(err) } - s := New() + s := New(nil) if err := s.Routes(); err != nil { t.Fatalf("cannot routes(): %v", err) } @@ -66,7 +66,7 @@ func TestRoutes(t *testing.T) { func TestServerHandlersNotFound(t *testing.T) { os.Args = []string{"a"} config.New() - s := New() + s := New(nil) w := httptest.NewRecorder() r, _ := http.NewRequest("GET", "/", nil) s.notFound(w, r) @@ -78,7 +78,7 @@ func TestServerHandlersNotFound(t *testing.T) { func TestServerHandlers(t *testing.T) { os.Args = []string{"a"} config.New() - s := New() + s := New(nil) cases := []struct { foo func(http.ResponseWriter, *http.Request, error) code int diff --git a/server/server_test.go b/server/server_test.go index eb2dc6e..f4368f4 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -8,7 +8,7 @@ import ( func TestServerRun(t *testing.T) { config.New() - s := New() + s := New(nil) go s.Run() time.Sleep(time.Millisecond * 100) } diff --git a/testdata/2add b/testdata/2add new file mode 100644 index 0000000..bfb1bd6 --- /dev/null +++ b/testdata/2add @@ -0,0 +1,68 @@ +#! /bin/bash + +set -e + +remote="${remote:-192.168.0.86}" + +feeds=' +http://lizclimo.tumblr.com/rss comic na na "24h" +https://mangadex.org/rss/follows/ryZGFkVgv9DAPqf45EsTHuQcxmteKBN6 manga na na "1h" +https://siftrss.com/f/WXPo6AKQLX manga na na "1h" +https://xkcd.com/atom.xml comic na na "24h" +http://yaoi-blcd.tumblr.com/tagged/19-days/rss manga "Old.Xian.update.of..19" na "4h" +http://www.shanaproject.com/feeds/secure/user/5310/5MUGNU9B3I/ anime na na "24h" +http://www.manga.megchan.com/blog/?feed=rss2 manga na na "12h" +https://siftrss.com/f/6Q8a0ZK0RmL manga na na "12h" +https://dropoutmanga.wordpress.com/feed/ manga "(Hatsukoi|Tomo.chan)" na "6h" +http://merakiscans.com/manga-rss/solo-leveling/ manga na na "8h" +http://www.mangago.me/r/rsslink/a_man_like_you.xml manga na na "24h" +http://www.mangago.me/r/rsslink/unriped_expression.xml manga na na "24h" +http://www.mangago.me/r/rsslink/heesu_in_class_2.xml manga na na "24h" +http://www.mangago.me/r/rsslink/bj_alex.xml manga na na "24h" +https://heroscans.tumblr.com/rss manga "(Saizu)" na "24h" +http://feeds.pinboard.in/rss/popular/ news ".*" ".*" "24h" +https://www.vox.com/rss/index.xml news ".*" ".*" "24h" +https://www.techdirt.com/techdirt_rss.xml news ".*" ".*" "24h" +http://hotdiggedydemon.com/ comic ".*" ".*" "24h" +https://nyaa.si/?page=rss&u=Nauti anime "(akegurui.*XX)" ".*Anime...English.translated.*" "3h" +https://tapas.io/rss/series/72832 manga na na "24h" +https://tapas.io/rss/series/88570 manga na na "24h" +https://nyaa.si/?page=rss&q=kakegurui&c=0_0&f=0&u=Nauti anime "(akegurui.XX)" ".*Anime...English.translated.*" "3h" +https://nyaa.si/?page=rss&q=horriblesubs+mob+psycho+720&c=0_0&f=0 anime ".*" ".*Anime...English.translated.*" "3h" +https://www.youtube.com/feeds/videos.xml?channel_id=UC9CuvdOVfMPvKCiwdGKL3cQ youtube","gampegrumps ".*" ".*" "6h" +https://www.youtube.com/feeds/videos.xml?channel_id=UCbGO1tCyjMlzqIT-tyrJNqQ youtube","kshaway ".*" ".*" "6h" +https://www.youtube.com/feeds/videos.xml?channel_id=UC9YLd0REiXxLqQQH_CpJKZQ youtube","instalok ".*" ".*" "6h" +https://www.youtube.com/feeds/videos.xml?channel_id=UCsvn_Po0SmunchJYOWpOxMg youtube","videogamedunkey ".*" ".*" "6h" +https://eztv.ag/ezrss.xml momma","torrent "^(Wheel.[oO]f.Fortune.[S2]|Jeopardy.20|Gold.Rush.*720.*WEB|Survivor.S.*720p|Gold.Rush.*S.*E.*720.*WEB|Archer|Pointless.*720p.*WEB|Mock.The.Week.*480p|8.Out.Of.10.Cats.*480p|Masterchef.*Australia.*720p|Price.[iI]s.[rR]ight|Let.*[mM]ake.[aA].[dD]eal)" "" "2h" +https://www.youtube.com/feeds/videos.xml?channel_id=UCs-w7E2HZWwXmjt9RTvBB_A youtube","settled ".*" ".*" "6h" +https://roosterteeth.com/show/rt-podcast/feed/mp3 podcast ".*" ".*" "6h" +' +feedsB="$(printf "%s\n" "$feeds" | grep -v '^$' | tail -n 1)" +#feeds="$feedsB" + +defaultIF='""' +defaultCF='""' +defaultIN='"6h"' + +feeds="${feeds// /,,}" + +for feed in $feeds; do + feed="${feed//,,/ }" + link="$(echo "$feed" | awk '{print $1}')" + tag="$(echo "$feed" | awk '{print $2}')" + if="$(echo "$feed" | awk '{print $3}')" + cf="$(echo "$feed" | awk '{print $4}')" + in="$(echo "$feed" | awk '{print $5}')" + if [ "$if" == "na" ]; then + if="$defaultIF" + fi + if [ "$cf" == "na" ]; then + cf="$defaultCF" + fi + if [ "$in" == "na" ]; then + in="$defaultIN" + fi + cmd="curl ${PROXY:-} -i ${remote}:33419/api/feed -X PUT --data '{\"url\":\"$link\", \"refresh\":$in, \"items\":$if, \"content\":$cf, \"tags\":[\"$tag\"]}'" + echo "$cmd" >&2 + eval "$cmd" +done diff --git a/testdata/2do b/testdata/2do index 9c14062..3318648 100644 --- a/testdata/2do +++ b/testdata/2do @@ -1,5 +1,7 @@ -* load feed jobs on startup and queue -* implement endpoints for server * implement torret+podcast handlers * fetch and embed? -* render tags -> feeds for server + +x render tags -> feeds for server +x load feed jobs on startup and queue +x push new jobs/queue items from server +x implement endpoints for server