Missing handlers but there we goddamn GO

Former-commit-id: 2505146a54acaf18eadfdebf1dd2720889aef795

handlers/handler.go

@@ -0,0 +1,52 @@
+package handlers
+
+import (
+	"local/rssmon3/config"
+	"local/rssmon3/monitor"
+	"local/rssmon3/rss"
+	"log"
+)
+
+type Handler struct {
+	Jobs <-chan *monitor.Item
+	config.Stoppable
+}
+
+func New(jobs <-chan *monitor.Item) *Handler {
+	return &Handler{
+		Jobs: jobs,
+	}
+}
+
+func (h *Handler) Run() error {
+	for {
+		select {
+		case <-h.Stopped():
+			return nil
+		case j := <-h.Jobs:
+			go func(key string) {
+				if err := h.Job(key); err != nil {
+					log.Println(err)
+				}
+			}(j.Key)
+		}
+	}
+}
+
+func (h *Handler) Job(key string) error {
+	f := &rss.Feed{Key: key}
+	if err := f.Load(); err != nil {
+		return err
+	}
+	if err := f.Pull(); err != nil {
+		return err
+	}
+	for _, tag := range f.Tags {
+		if foo := ByTag(tag); foo != nil {
+			if err := foo(key); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

handlers/lookup.go

@@ -0,0 +1,12 @@
+package handlers
+
+func ByTag(tag string) func(string) error {
+	var foo func(string) error
+	switch tag {
+	case "torrent":
+		foo = torrent
+	case "podcast":
+		foo = podcast
+	}
+	return foo
+}

handlers/podcast.go

@@ -0,0 +1,7 @@
+package handlers
+
+import "errors"
+
+func podcast(key string) error {
+	return errors.New("not impl")
+}

handlers/torrent.go

@@ -0,0 +1,7 @@
+package handlers
+
+import "errors"
+
+func torrent(key string) error {
+	return errors.New("not impl")
+}

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"local/rssmon3/config"
+	"local/rssmon3/handlers"
 	"local/rssmon3/monitor"
 	"local/rssmon3/server"
 	"log"
@@ -21,18 +22,21 @@ func main() {
 	}
 	log.Println(config.Values())
 
-	s := server.New()
-	if err := s.Routes(); err != nil {
-		panic(err)
-	}
-	go InterruptAfter(s.Run, sigc)
-
 	m, err := monitor.New()
 	if err != nil {
 		panic(err)
 	}
 	go InterruptAfter(m.Run, sigc)
 
+	s := server.New(m.Incoming)
+	if err := s.Routes(); err != nil {
+		panic(err)
+	}
+	go InterruptAfter(s.Run, sigc)
+
+	h := handlers.New(m.Outgoing)
+	go InterruptAfter(h.Run, sigc)
+
 	signal.Notify(sigc,
 		syscall.SIGHUP,
 		syscall.SIGINT,

monitor/item.go

@@ -40,9 +40,9 @@ func (i *Item) Compare(other queue.Item) int {
 	iNext := i.Last().Add(i.Interval())
 	jNext := j.Last().Add(j.Interval())
 	if iNext.Before(jNext) {
-		return 1
-	} else if jNext.Before(iNext) {
 		return -1
+	} else if jNext.Before(iNext) {
+		return 1
 	} else {
 		return 0
 	}

monitor/item_test.go

@@ -31,7 +31,7 @@ func TestItemCompare(t *testing.T) {
 
 	i, _ := NewItem("iCompare", time.Second)
 	j, _ := NewItem("jCompare", 10*time.Second)
-	if c := i.Compare(j); c != -1 {
+	if c := i.Compare(j); c != 1 {
 		t.Fatal(c)
 	}
 
@@ -43,7 +43,7 @@ func TestItemCompare(t *testing.T) {
 
 	i, _ = NewItem("iCompare", 10*time.Second)
 	j, _ = NewItem("jCompare", time.Second)
-	if c := i.Compare(j); c != 1 {
+	if c := i.Compare(j); c != -1 {
 		t.Fatal(c)
 	}
 }

monitor/monitor.go

@@ -2,6 +2,7 @@ package monitor
 
 import (
 	"local/rssmon3/config"
+	"log"
 	"time"
 )
 
@@ -51,6 +52,8 @@ func (m *Monitor) triggered() <-chan time.Time {
 	if top == nil {
 		return nil
 	}
+	block := time.Until(top.Last().Add(top.Interval()))
+	log.Printf("blocking %v until next task", block)
 	return time.After(time.Until(top.Last().Add(top.Interval())))
 }
 

monitor/queue_test.go

@@ -39,7 +39,7 @@ func TestQueuePriority(t *testing.T) {
 	j, _ := NewItem("jQueuePriority", time.Hour)
 	q.Push(i)
 	q.Push(j)
-	if k := q.Peek(); k.Key != "iQueuePriority" {
+	if k := q.Peek(); k.Key != "jQueuePriority" {
 		t.Errorf("compare is backwards")
 	}
 

rss/feed.go

@@ -53,7 +53,7 @@ func (f *Feed) Load() error {
 	return f.Decode(b)
 }
 
-func (f *Feed) pull() error {
+func (f *Feed) Pull() error {
 	if f.URL == "" {
 		if err := f.Load(); err != nil {
 			return err

rss/feed_test.go

@@ -107,7 +107,7 @@ func TestRSSFeedPull(t *testing.T) {
 
 	log.SetOutput(bytes.NewBuffer(nil))
 	defer log.SetOutput(os.Stderr)
-	if err := f.pull(); err != nil {
+	if err := f.Pull(); err != nil {
 		t.Fatal(err)
 	}
 	log.SetOutput(os.Stderr)
@@ -129,7 +129,7 @@ func TestRSSFeedPull(t *testing.T) {
 
 	log.SetOutput(bytes.NewBuffer(nil))
 	defer log.SetOutput(os.Stderr)
-	if err := f.pull(); err != nil {
+	if err := f.Pull(); err != nil {
 		t.Fatal(err)
 	}
 	log.SetOutput(os.Stderr)

rss/item.go

@@ -22,6 +22,8 @@ type Item struct {
 	TS      time.Time
 }
 
+type Items []*Item
+
 func newItem(i *gofeed.Item, contentFilter string) (*Item, error) {
 	item := &Item{
 		Title:   i.Title,
@@ -88,3 +90,17 @@ func (i *Item) Load(key, ns1 string, ns ...string) error {
 	}
 	return config.Decode(b, i)
 }
+
+func (is Items) Len() int {
+	return len(is)
+}
+
+func (is Items) Less(i, j int) bool {
+	return is[j].TS.Before(is[i].TS)
+}
+
+func (is Items) Swap(i, j int) {
+	k := is[i]
+	is[i] = is[j]
+	is[j] = k
+}

rss/rss.go

@@ -32,7 +32,7 @@ func (rss *RSS) Run() error {
 
 func (rss *RSS) update(item *monitor.Item) error {
 	f := newFeed(item.Key)
-	if err := f.pull(); err != nil {
+	if err := f.Pull(); err != nil {
 		return err
 	}
 	if err := f.save(); err != nil {

rss/serialize.go

@@ -1,8 +1,10 @@
 package rss
 
 import (
+	"fmt"
 	"io"
 	"local/rssmon3/config"
+	"sort"
 
 	"github.com/gorilla/feeds"
 )
@@ -29,6 +31,7 @@ func TaggedFeeds(tag string) ([]*Feed, error) {
 }
 
 func WriteFeed(w io.Writer, tag string, items []*Item) error {
+	sort.Sort(Items(items))
 	feed := &feeds.Feed{
 		Title:       tag,
 		Link:        &feeds.Link{},
@@ -47,6 +50,6 @@ func WriteFeed(w io.Writer, tag string, items []*Item) error {
 	if err != nil {
 		return err
 	}
-	w.Write([]byte(s))
+	fmt.Fprintf(w, "%s\n", s)
 	return nil
 }

rss/serialize_test.go

@@ -50,7 +50,7 @@ func TestRSSWriteFeed(t *testing.T) {
 
 	log.SetOutput(bytes.NewBuffer(nil))
 	defer log.SetOutput(os.Stderr)
-	if err := f.pull(); err != nil {
+	if err := f.Pull(); err != nil {
 		t.Fatal(err)
 	}
 	log.SetOutput(os.Stderr)

rssmon3.REMOVED.git-id

@@ -1 +1 @@
-cc79df3e2bb33d684975d23cd008eae93e7dffc2
+49c1645f536cea7463202894e751295a3ec589f1

server/new.go

@@ -3,19 +3,22 @@ package server
 import (
 	"local/router"
 	"local/rssmon3/config"
+	"local/rssmon3/monitor"
 	"strings"
 )
 
 type Server struct {
 	Port   string
 	router *router.Router
+	Jobs   chan<- *monitor.Item
 }
 
-func New() *Server {
+func New(jobs chan<- *monitor.Item) *Server {
 	config := config.Values()
 	s := &Server{
 		Port:   ":" + strings.TrimPrefix(config.Port, ":"),
 		router: router.New(),
+		Jobs:   jobs,
 	}
 	return s
 }

server/new_test.go

@@ -10,5 +10,5 @@ func TestNew(t *testing.T) {
 	os.Setenv("DB", "MAP")
 	os.Args = []string{"ok"}
 	config.New()
-	New()
+	New(nil)
 }

server/routes.go

@@ -2,7 +2,6 @@ package server
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"local/router"
 	"local/rssmon3/monitor"
@@ -77,16 +76,21 @@ func (s *Server) feed(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	var putFeed struct {
-		URL           string        `json:"url"`
+		URL           string   `json:"url"`
-		Interval      time.Duration `json:"refresh"`
+		Interval      string   `json:"refresh"`
-		TitleFilter   string        `json:"items"`
+		TitleFilter   string   `json:"items"`
-		ContentFilter string        `json:"content"`
+		ContentFilter string   `json:"content"`
-		Tags          []string      `json:"tags"`
+		Tags          []string `json:"tags"`
 	}
 	if err := json.NewDecoder(r.Body).Decode(&putFeed); err != nil {
 		s.userError(w, r, err)
 		return
 	}
+	interval, err := time.ParseDuration(putFeed.Interval)
+	if err != nil {
+		s.userError(w, r, err)
+		return
+	}
 	f := &rss.Feed{
 		Key:           putFeed.URL,
 		URL:           putFeed.URL,
@@ -98,12 +102,10 @@ func (s *Server) feed(w http.ResponseWriter, r *http.Request) {
 		s.error(w, r, err)
 		return
 	}
-	i, err := monitor.NewItem(f.Key, putFeed.Interval)
+	i, err := monitor.NewItem(f.Key, interval)
 	if err != nil {
 		s.error(w, r, err)
 		return
 	}
-	log.Println(i)
+	s.Jobs <- i
-	s.error(w, r, errors.New("not impl"))
-	// TODO
 }

server/routes_test.go

@@ -19,7 +19,7 @@ func TestRoutes(t *testing.T) {
 	if err := config.New(); err != nil {
 		t.Fatal(err)
 	}
-	s := New()
+	s := New(nil)
 	if err := s.Routes(); err != nil {
 		t.Fatalf("cannot routes(): %v", err)
 	}
@@ -66,7 +66,7 @@ func TestRoutes(t *testing.T) {
 func TestServerHandlersNotFound(t *testing.T) {
 	os.Args = []string{"a"}
 	config.New()
-	s := New()
+	s := New(nil)
 	w := httptest.NewRecorder()
 	r, _ := http.NewRequest("GET", "/", nil)
 	s.notFound(w, r)
@@ -78,7 +78,7 @@ func TestServerHandlersNotFound(t *testing.T) {
 func TestServerHandlers(t *testing.T) {
 	os.Args = []string{"a"}
 	config.New()
-	s := New()
+	s := New(nil)
 	cases := []struct {
 		foo  func(http.ResponseWriter, *http.Request, error)
 		code int

server/server_test.go

@@ -8,7 +8,7 @@ import (
 
 func TestServerRun(t *testing.T) {
 	config.New()
-	s := New()
+	s := New(nil)
 	go s.Run()
 	time.Sleep(time.Millisecond * 100)
 }

testdata/2add

@@ -0,0 +1,68 @@
+#! /bin/bash
+
+set -e
+
+remote="${remote:-192.168.0.86}"
+
+feeds='
+http://lizclimo.tumblr.com/rss                                                   comic                      na                                     na "24h"
+https://mangadex.org/rss/follows/ryZGFkVgv9DAPqf45EsTHuQcxmteKBN6                manga                      na                                     na "1h"
+https://siftrss.com/f/WXPo6AKQLX                                                 manga                      na                                     na "1h"
+https://xkcd.com/atom.xml                                                        comic                      na                                     na "24h"
+http://yaoi-blcd.tumblr.com/tagged/19-days/rss                                   manga                      "Old.Xian.update.of..19"               na "4h"
+http://www.shanaproject.com/feeds/secure/user/5310/5MUGNU9B3I/                   anime                      na                                     na "24h"
+http://www.manga.megchan.com/blog/?feed=rss2                                     manga                      na                                     na "12h"
+https://siftrss.com/f/6Q8a0ZK0RmL                                                manga                      na                                     na "12h"
+https://dropoutmanga.wordpress.com/feed/                                         manga                      "(Hatsukoi|Tomo.chan)"                 na "6h"
+http://merakiscans.com/manga-rss/solo-leveling/                                  manga                      na                                     na "8h"
+http://www.mangago.me/r/rsslink/a_man_like_you.xml                               manga                      na                                     na "24h"
+http://www.mangago.me/r/rsslink/unriped_expression.xml                           manga                      na                                     na "24h"
+http://www.mangago.me/r/rsslink/heesu_in_class_2.xml                             manga                      na                                     na "24h"
+http://www.mangago.me/r/rsslink/bj_alex.xml                                      manga                      na                                     na "24h"
+https://heroscans.tumblr.com/rss                                                 manga                      "(Saizu)"                              na "24h"
+http://feeds.pinboard.in/rss/popular/                                            news                       ".*"                                   ".*" "24h"
+https://www.vox.com/rss/index.xml                                                news                       ".*"                                   ".*" "24h"
+https://www.techdirt.com/techdirt_rss.xml                                        news                       ".*"                                   ".*" "24h"
+http://hotdiggedydemon.com/                                                      comic                      ".*"                                   ".*"  "24h"
+https://nyaa.si/?page=rss&u=Nauti                                                anime                      "(akegurui.*XX)"                       ".*Anime...English.translated.*" "3h"
+https://tapas.io/rss/series/72832                                                manga                      na                                     na "24h"
+https://tapas.io/rss/series/88570                                                manga                      na                                     na "24h"
+https://nyaa.si/?page=rss&q=kakegurui&c=0_0&f=0&u=Nauti                          anime                      "(akegurui.XX)"                        ".*Anime...English.translated.*" "3h"
+https://nyaa.si/?page=rss&q=horriblesubs+mob+psycho+720&c=0_0&f=0                anime                      ".*"                                   ".*Anime...English.translated.*" "3h"
+https://www.youtube.com/feeds/videos.xml?channel_id=UC9CuvdOVfMPvKCiwdGKL3cQ     youtube","gampegrumps      ".*"                                   ".*" "6h"
+https://www.youtube.com/feeds/videos.xml?channel_id=UCbGO1tCyjMlzqIT-tyrJNqQ     youtube","kshaway          ".*"                                   ".*" "6h"
+https://www.youtube.com/feeds/videos.xml?channel_id=UC9YLd0REiXxLqQQH_CpJKZQ     youtube","instalok         ".*"                                   ".*" "6h"
+https://www.youtube.com/feeds/videos.xml?channel_id=UCsvn_Po0SmunchJYOWpOxMg     youtube","videogamedunkey  ".*"                                   ".*" "6h"
+https://eztv.ag/ezrss.xml                                                        momma","torrent            "^(Wheel.[oO]f.Fortune.[S2]|Jeopardy.20|Gold.Rush.*720.*WEB|Survivor.S.*720p|Gold.Rush.*S.*E.*720.*WEB|Archer|Pointless.*720p.*WEB|Mock.The.Week.*480p|8.Out.Of.10.Cats.*480p|Masterchef.*Australia.*720p|Price.[iI]s.[rR]ight|Let.*[mM]ake.[aA].[dD]eal)"  "<a.*?href=.magnet:.*?</a>" "2h"
+https://www.youtube.com/feeds/videos.xml?channel_id=UCs-w7E2HZWwXmjt9RTvBB_A      youtube","settled         ".*"                                   ".*" "6h"
+https://roosterteeth.com/show/rt-podcast/feed/mp3                                podcast                    ".*"                                   ".*" "6h"
+'
+feedsB="$(printf "%s\n" "$feeds" | grep -v '^$' | tail -n 1)"
+#feeds="$feedsB"
+
+defaultIF='""'
+defaultCF='"<img.*?/(img)?>"'
+defaultIN='"6h"'
+
+feeds="${feeds// /,,}"
+
+for feed in $feeds; do
+   feed="${feed//,,/ }"
+   link="$(echo "$feed" | awk '{print $1}')"
+   tag="$(echo "$feed" | awk '{print $2}')"
+   if="$(echo "$feed" | awk '{print $3}')"
+   cf="$(echo "$feed" | awk '{print $4}')"
+   in="$(echo "$feed" | awk '{print $5}')"
+   if [ "$if" == "na" ]; then
+      if="$defaultIF"
+   fi
+   if [ "$cf" == "na" ]; then
+      cf="$defaultCF"
+   fi
+   if [ "$in" == "na" ]; then
+      in="$defaultIN"
+   fi
+   cmd="curl ${PROXY:-} -i ${remote}:33419/api/feed -X PUT --data '{\"url\":\"$link\", \"refresh\":$in, \"items\":$if, \"content\":$cf, \"tags\":[\"$tag\"]}'"
+   echo "$cmd" >&2
+   eval "$cmd"
+done

testdata/2do

@@ -1,5 +1,7 @@
-*  load feed jobs on startup and queue
-*  implement endpoints for server
 *  implement torret+podcast handlers
 *  fetch <img> and embed?
-*  render tags -> feeds for server
+
+x  render tags -> feeds for server
+x  load feed jobs on startup and queue
+x  push new jobs/queue items from server
+x  implement endpoints for server