local
/
rssmon3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Missing handlers but there we goddamn GO 

Former-commit-id: 2505146a54acaf18eadfdebf1dd2720889aef795
master v0.0
bel 2019-06-22 16:23:27 -06:00
parent 093d468f87
commit 35b3ff2c2d
24 changed files with 216 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
handlers/handler.go Normal file
View File

@ -0,0 +1,52 @@
package handlers
import (
"local/rssmon3/config"
"local/rssmon3/monitor"
"local/rssmon3/rss"
"log"
)
type Handler struct {
Jobs <-chan *monitor.Item
config.Stoppable
}
func New(jobs <-chan *monitor.Item) *Handler {
return &Handler{
Jobs: jobs,
}
}
func (h *Handler) Run() error {
for {
select {
case <-h.Stopped():
return nil
case j := <-h.Jobs:
go func(key string) {
if err := h.Job(key); err != nil {
log.Println(err)
}
}(j.Key)
}
}
}
func (h *Handler) Job(key string) error {
f := &rss.Feed{Key: key}
if err := f.Load(); err != nil {
return err
}
if err := f.Pull(); err != nil {
return err
}
for _, tag := range f.Tags {
if foo := ByTag(tag); foo != nil {
if err := foo(key); err != nil {
return err
}
}
}
return nil
}

12
handlers/lookup.go Normal file
View File

@ -0,0 +1,12 @@
package handlers
func ByTag(tag string) func(string) error {
var foo func(string) error
switch tag {
case "torrent":
foo = torrent
case "podcast":
foo = podcast
}
return foo
}

7
handlers/podcast.go Normal file
View File

@ -0,0 +1,7 @@
package handlers
import "errors"
func podcast(key string) error {
return errors.New("not impl")
}

7
handlers/torrent.go Normal file
View File

@ -0,0 +1,7 @@
package handlers
import "errors"
func torrent(key string) error {
return errors.New("not impl")
}

16
main.go
View File

@ -2,6 +2,7 @@ package main
import (
"local/rssmon3/config"
"local/rssmon3/handlers"
"local/rssmon3/monitor"
"local/rssmon3/server"
"log"
@ -21,18 +22,21 @@ func main() {
}
log.Println(config.Values())
s := server.New()
if err := s.Routes(); err != nil {
panic(err)
}
go InterruptAfter(s.Run, sigc)
m, err := monitor.New()
if err != nil {
panic(err)
}
go InterruptAfter(m.Run, sigc)
s := server.New(m.Incoming)
if err := s.Routes(); err != nil {
panic(err)
}
go InterruptAfter(s.Run, sigc)
h := handlers.New(m.Outgoing)
go InterruptAfter(h.Run, sigc)
signal.Notify(sigc,
syscall.SIGHUP,
syscall.SIGINT,

BIN
monitor/.monitor_test.go.swp
View File

Binary file not shown.

4
monitor/item.go
View File

@ -40,9 +40,9 @@ func (i *Item) Compare(other queue.Item) int {
iNext := i.Last().Add(i.Interval())
jNext := j.Last().Add(j.Interval())
if iNext.Before(jNext) {
return 1
} else if jNext.Before(iNext) {
return -1
} else if jNext.Before(iNext) {
return 1
} else {
return 0
}

4
monitor/item_test.go
View File

@ -31,7 +31,7 @@ func TestItemCompare(t *testing.T) {
i, _ := NewItem("iCompare", time.Second)
j, _ := NewItem("jCompare", 10*time.Second)
if c := i.Compare(j); c != -1 {
if c := i.Compare(j); c != 1 {
t.Fatal(c)
}
@ -43,7 +43,7 @@ func TestItemCompare(t *testing.T) {
i, _ = NewItem("iCompare", 10*time.Second)
j, _ = NewItem("jCompare", time.Second)
if c := i.Compare(j); c != 1 {
if c := i.Compare(j); c != -1 {
t.Fatal(c)
}
}

3
monitor/monitor.go
View File

@ -2,6 +2,7 @@ package monitor
import (
"local/rssmon3/config"
"log"
"time"
)
@ -51,6 +52,8 @@ func (m *Monitor) triggered() <-chan time.Time {
if top == nil {
return nil
}
block := time.Until(top.Last().Add(top.Interval()))
log.Printf("blocking %v until next task", block)
return time.After(time.Until(top.Last().Add(top.Interval())))
}

2
monitor/queue_test.go
View File

@ -39,7 +39,7 @@ func TestQueuePriority(t *testing.T) {
j, _ := NewItem("jQueuePriority", time.Hour)
q.Push(i)
q.Push(j)
if k := q.Peek(); k.Key != "iQueuePriority" {
if k := q.Peek(); k.Key != "jQueuePriority" {
t.Errorf("compare is backwards")
}

2
rss/feed.go
View File

@ -53,7 +53,7 @@ func (f *Feed) Load() error {
return f.Decode(b)
}
func (f *Feed) pull() error {
func (f *Feed) Pull() error {
if f.URL == "" {
if err := f.Load(); err != nil {
return err

4
rss/feed_test.go
View File

@ -107,7 +107,7 @@ func TestRSSFeedPull(t *testing.T) {
log.SetOutput(bytes.NewBuffer(nil))
defer log.SetOutput(os.Stderr)
if err := f.pull(); err != nil {
if err := f.Pull(); err != nil {
t.Fatal(err)
}
log.SetOutput(os.Stderr)
@ -129,7 +129,7 @@ func TestRSSFeedPull(t *testing.T) {
log.SetOutput(bytes.NewBuffer(nil))
defer log.SetOutput(os.Stderr)
if err := f.pull(); err != nil {
if err := f.Pull(); err != nil {
t.Fatal(err)
}
log.SetOutput(os.Stderr)

16
rss/item.go
View File

@ -22,6 +22,8 @@ type Item struct {
TS time.Time
}
type Items []*Item
func newItem(i *gofeed.Item, contentFilter string) (*Item, error) {
item := &Item{
Title: i.Title,
@ -88,3 +90,17 @@ func (i *Item) Load(key, ns1 string, ns ...string) error {
}
return config.Decode(b, i)
}
func (is Items) Len() int {
return len(is)
}
func (is Items) Less(i, j int) bool {
return is[j].TS.Before(is[i].TS)
}
func (is Items) Swap(i, j int) {
k := is[i]
is[i] = is[j]
is[j] = k
}

2
rss/rss.go
View File

@ -32,7 +32,7 @@ func (rss *RSS) Run() error {
func (rss *RSS) update(item *monitor.Item) error {
f := newFeed(item.Key)
if err := f.pull(); err != nil {
if err := f.Pull(); err != nil {
return err
}
if err := f.save(); err != nil {

5
rss/serialize.go
View File

@ -1,8 +1,10 @@
package rss
import (
"fmt"
"io"
"local/rssmon3/config"
"sort"
"github.com/gorilla/feeds"
)
@ -29,6 +31,7 @@ func TaggedFeeds(tag string) ([]*Feed, error) {
}
func WriteFeed(w io.Writer, tag string, items []*Item) error {
sort.Sort(Items(items))
feed := &feeds.Feed{
Title: tag,
Link: &feeds.Link{},
@ -47,6 +50,6 @@ func WriteFeed(w io.Writer, tag string, items []*Item) error {
if err != nil {
return err
}
w.Write([]byte(s))
fmt.Fprintf(w, "%s\n", s)
return nil
}

2
rss/serialize_test.go
View File

@ -50,7 +50,7 @@ func TestRSSWriteFeed(t *testing.T) {
log.SetOutput(bytes.NewBuffer(nil))
defer log.SetOutput(os.Stderr)
if err := f.pull(); err != nil {
if err := f.Pull(); err != nil {
t.Fatal(err)
}
log.SetOutput(os.Stderr)

2
rssmon3.REMOVED.git-id
View File

@ -1 +1 @@
cc79df3e2bb33d684975d23cd008eae93e7dffc2
49c1645f536cea7463202894e751295a3ec589f1

5
server/new.go
View File

@ -3,19 +3,22 @@ package server
import (
"local/router"
"local/rssmon3/config"
"local/rssmon3/monitor"
"strings"
)
type Server struct {
Port string
router *router.Router
Jobs chan<- *monitor.Item
}
func New() *Server {
func New(jobs chan<- *monitor.Item) *Server {
config := config.Values()
s := &Server{
Port: ":" + strings.TrimPrefix(config.Port, ":"),
router: router.New(),
Jobs: jobs,
}
return s
}

2
server/new_test.go
View File

@ -10,5 +10,5 @@ func TestNew(t *testing.T) {
os.Setenv("DB", "MAP")
os.Args = []string{"ok"}
config.New()
New()
New(nil)
}

14
server/routes.go
View File

@ -2,7 +2,6 @@ package server
import (
"encoding/json"
"errors"
"fmt"
"local/router"
"local/rssmon3/monitor"
@ -78,7 +77,7 @@ func (s *Server) feed(w http.ResponseWriter, r *http.Request) {
}
var putFeed struct {
URL string `json:"url"`
Interval time.Duration `json:"refresh"`
Interval string `json:"refresh"`
TitleFilter string `json:"items"`
ContentFilter string `json:"content"`
Tags []string `json:"tags"`
@ -87,6 +86,11 @@ func (s *Server) feed(w http.ResponseWriter, r *http.Request) {
s.userError(w, r, err)
return
}
interval, err := time.ParseDuration(putFeed.Interval)
if err != nil {
s.userError(w, r, err)
return
}
f := &rss.Feed{
Key: putFeed.URL,
URL: putFeed.URL,
@ -98,12 +102,10 @@ func (s *Server) feed(w http.ResponseWriter, r *http.Request) {
s.error(w, r, err)
return
}
i, err := monitor.NewItem(f.Key, putFeed.Interval)
i, err := monitor.NewItem(f.Key, interval)
if err != nil {
s.error(w, r, err)
return
}
log.Println(i)
s.error(w, r, errors.New("not impl"))
// TODO
s.Jobs <- i
}

6
server/routes_test.go
View File

@ -19,7 +19,7 @@ func TestRoutes(t *testing.T) {
if err := config.New(); err != nil {
t.Fatal(err)
}
s := New()
s := New(nil)
if err := s.Routes(); err != nil {
t.Fatalf("cannot routes(): %v", err)
}
@ -66,7 +66,7 @@ func TestRoutes(t *testing.T) {
func TestServerHandlersNotFound(t *testing.T) {
os.Args = []string{"a"}
config.New()
s := New()
s := New(nil)
w := httptest.NewRecorder()
r, _ := http.NewRequest("GET", "/", nil)
s.notFound(w, r)
@ -78,7 +78,7 @@ func TestServerHandlersNotFound(t *testing.T) {
func TestServerHandlers(t *testing.T) {
os.Args = []string{"a"}
config.New()
s := New()
s := New(nil)
cases := []struct {
foo func(http.ResponseWriter, *http.Request, error)
code int

2
server/server_test.go
View File

@ -8,7 +8,7 @@ import (
func TestServerRun(t *testing.T) {
config.New()
s := New()
s := New(nil)
go s.Run()
time.Sleep(time.Millisecond * 100)
}

68
testdata/2add vendored Normal file
View File

@ -0,0 +1,68 @@
#! /bin/bash
set -e
remote="${remote:-192.168.0.86}"
feeds='
http://lizclimo.tumblr.com/rss comic na na "24h"
https://mangadex.org/rss/follows/ryZGFkVgv9DAPqf45EsTHuQcxmteKBN6 manga na na "1h"
https://siftrss.com/f/WXPo6AKQLX manga na na "1h"
https://xkcd.com/atom.xml comic na na "24h"
http://yaoi-blcd.tumblr.com/tagged/19-days/rss manga "Old.Xian.update.of..19" na "4h"
http://www.shanaproject.com/feeds/secure/user/5310/5MUGNU9B3I/ anime na na "24h"
http://www.manga.megchan.com/blog/?feed=rss2 manga na na "12h"
https://siftrss.com/f/6Q8a0ZK0RmL manga na na "12h"
https://dropoutmanga.wordpress.com/feed/ manga "(Hatsukoi|Tomo.chan)" na "6h"
http://merakiscans.com/manga-rss/solo-leveling/ manga na na "8h"
http://www.mangago.me/r/rsslink/a_man_like_you.xml manga na na "24h"
http://www.mangago.me/r/rsslink/unriped_expression.xml manga na na "24h"
http://www.mangago.me/r/rsslink/heesu_in_class_2.xml manga na na "24h"
http://www.mangago.me/r/rsslink/bj_alex.xml manga na na "24h"
https://heroscans.tumblr.com/rss manga "(Saizu)" na "24h"
http://feeds.pinboard.in/rss/popular/ news ".*" ".*" "24h"
https://www.vox.com/rss/index.xml news ".*" ".*" "24h"
https://www.techdirt.com/techdirt_rss.xml news ".*" ".*" "24h"
http://hotdiggedydemon.com/ comic ".*" ".*" "24h"
https://nyaa.si/?page=rss&u=Nauti anime "(akegurui.*XX)" ".*Anime...English.translated.*" "3h"
https://tapas.io/rss/series/72832 manga na na "24h"
https://tapas.io/rss/series/88570 manga na na "24h"
https://nyaa.si/?page=rss&q=kakegurui&c=0_0&f=0&u=Nauti anime "(akegurui.XX)" ".*Anime...English.translated.*" "3h"
https://nyaa.si/?page=rss&q=horriblesubs+mob+psycho+720&c=0_0&f=0 anime ".*" ".*Anime...English.translated.*" "3h"
https://www.youtube.com/feeds/videos.xml?channel_id=UC9CuvdOVfMPvKCiwdGKL3cQ youtube","gampegrumps ".*" ".*" "6h"
https://www.youtube.com/feeds/videos.xml?channel_id=UCbGO1tCyjMlzqIT-tyrJNqQ youtube","kshaway ".*" ".*" "6h"
https://www.youtube.com/feeds/videos.xml?channel_id=UC9YLd0REiXxLqQQH_CpJKZQ youtube","instalok ".*" ".*" "6h"
https://www.youtube.com/feeds/videos.xml?channel_id=UCsvn_Po0SmunchJYOWpOxMg youtube","videogamedunkey ".*" ".*" "6h"
https://eztv.ag/ezrss.xml momma","torrent "^(Wheel.[oO]f.Fortune.[S2]|Jeopardy.20|Gold.Rush.*720.*WEB|Survivor.S.*720p|Gold.Rush.*S.*E.*720.*WEB|Archer|Pointless.*720p.*WEB|Mock.The.Week.*480p|8.Out.Of.10.Cats.*480p|Masterchef.*Australia.*720p|Price.[iI]s.[rR]ight|Let.*[mM]ake.[aA].[dD]eal)" "<a.*?href=.magnet:.*?</a>" "2h"
https://www.youtube.com/feeds/videos.xml?channel_id=UCs-w7E2HZWwXmjt9RTvBB_A youtube","settled ".*" ".*" "6h"
https://roosterteeth.com/show/rt-podcast/feed/mp3 podcast ".*" ".*" "6h"
'
feedsB="$(printf "%s\n" "$feeds" | grep -v '^$' | tail -n 1)"
#feeds="$feedsB"
defaultIF='""'
defaultCF='"<img.*?/(img)?>"'
defaultIN='"6h"'
feeds="${feeds// /,,}"
for feed in $feeds; do
feed="${feed//,,/ }"
link="$(echo "$feed" | awk '{print $1}')"
tag="$(echo "$feed" | awk '{print $2}')"
if="$(echo "$feed" | awk '{print $3}')"
cf="$(echo "$feed" | awk '{print $4}')"
in="$(echo "$feed" | awk '{print $5}')"
if [ "$if" == "na" ]; then
if="$defaultIF"
fi
if [ "$cf" == "na" ]; then
cf="$defaultCF"
fi
if [ "$in" == "na" ]; then
in="$defaultIN"
fi
cmd="curl ${PROXY:-} -i ${remote}:33419/api/feed -X PUT --data '{\"url\":\"$link\", \"refresh\":$in, \"items\":$if, \"content\":$cf, \"tags\":[\"$tag\"]}'"
echo "$cmd" >&2
eval "$cmd"
done

8
testdata/2do vendored
View File

@ -1,5 +1,7 @@
* load feed jobs on startup and queue
* implement endpoints for server
* implement torret+podcast handlers
* fetch <img> and embed?
* render tags -> feeds for server
x render tags -> feeds for server
x load feed jobs on startup and queue
x push new jobs/queue items from server
x implement endpoints for server