encodable a good

Former-commit-id: a0afebd82dfd256ec70c02c607af2a9b3b4b046f

config/encodable.go

@@ -0,0 +1,22 @@
+package config
+
+import (
+	"bytes"
+	"encoding/gob"
+)
+
+type Encodable struct{}
+
+func (e *Encodable) Encode() ([]byte, error) {
+	buff := bytes.NewBuffer(nil)
+	enc := gob.NewEncoder(buff)
+	err := enc.Encode(e)
+	return buff.Bytes(), err
+}
+
+func (e *Encodable) Decode(b []byte) error {
+	buff := bytes.NewBuffer(b)
+	dec := gob.NewDecoder(buff)
+	err := dec.Decode(e)
+	return err
+}

config/new_test.go

@@ -34,6 +34,7 @@ func TestNew(t *testing.T) {
 			}
 		}
 	}()
+	os.Args = []string{"a", "-db", "map"}
 	if err := New(); err != nil {
 		t.Fatal(err)
 	}

config/stoppable.go

@@ -0,0 +1,7 @@
+package config
+
+type Stoppable struct{}
+
+func (s Stoppable) Stopped() <-chan struct{} {
+	return Values().Ctx.Done()
+}

main.go

@@ -1,8 +1,8 @@
 package main
 
 import (
-	"local/rssmon2/monitor"
 	"local/rssmon3/config"
+	"local/rssmon3/monitor"
 	"local/rssmon3/server"
 	"log"
 	"os"
@@ -27,10 +27,7 @@ func main() {
 	}
 	go InterruptAfter(s.Run, sigc)
 
-	m, err := monitor.New()
-	if err != nil {
-		panic(err)
-	}
+	m := monitor.New()
 	go InterruptAfter(m.Run, sigc)
 
 	signal.Notify(sigc,
@@ -40,7 +37,7 @@ func main() {
 		syscall.SIGQUIT,
 	)
 	<-sigc
-	can()
+	config.Values().Can()
 	config.Values().DB.Close()
 }
 

monitor/item.go

@@ -12,6 +12,7 @@ import (
 
 type Item struct {
 	Key string
+	*config.Encodable
 }
 
 const nsLast = "nsLast"
@@ -23,6 +24,7 @@ var forever = time.Duration(time.Hour * 99999)
 func NewItem(key string, interval time.Duration) (*Item, error) {
 	i := &Item{
 		Key:       key,
+		Encodable: &config.Encodable{},
 	}
 
 	if err := i.setInterval(interval); err != nil {

monitor/monitor.go

@@ -9,6 +9,7 @@ type Monitor struct {
 	queue    *Queue
 	Incoming chan *Item
 	Outgoing chan *Item
+	config.Stoppable
 }
 
 func New() *Monitor {
@@ -22,7 +23,7 @@ func New() *Monitor {
 func (m *Monitor) Run() error {
 	for {
 		select {
-		case <-m.stopped():
+		case <-m.Stopped():
 			return nil
 		case i := <-m.enqueued():
 			m.enqueue(i)
@@ -33,10 +34,6 @@ func (m *Monitor) Run() error {
 	}
 }
 
-func (m *Monitor) stopped() <-chan struct{} {
-	return config.Values().Ctx.Done()
-}
-
 func (m *Monitor) enqueued() chan *Item {
 	return m.Incoming
 }

rss/feed.go

@@ -0,0 +1,110 @@
+package rss
+
+import (
+	"errors"
+	"local/rssmon3/config"
+	"log"
+	"net/http"
+	"regexp"
+	"time"
+
+	"github.com/mmcdole/gofeed"
+)
+
+const NSFeeds = "NSFeeds"
+
+type feed struct {
+	Key           string
+	URL           string
+	Updated       time.Time
+	TitleFilter   string
+	ContentFilter string
+	Tags          []string
+	*config.Encodable
+}
+
+func newFeed(key string) *feed {
+	return &feed{
+		Key:       key,
+		Encodable: &config.Encodable{},
+	}
+}
+
+func (f *feed) load() error {
+	if f.Key == "" {
+		return errors.New("cannot load nil feed")
+	}
+	db := config.Values().DB
+	b, err := db.Get(f.Key, NSFeeds)
+	if err != nil {
+		return err
+	}
+	return f.Decode(b)
+}
+
+func (f *feed) pull() error {
+	if f.URL == "" {
+		if err := f.load(); err != nil {
+			return err
+		}
+	}
+	resp, err := http.Get(f.URL)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	gofeed, err := gofeed.NewParser().Parse(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	itemTSs := []*time.Time{}
+	for _, i := range gofeed.Items {
+		ts := latestTSPtr(i.UpdatedParsed, i.PublishedParsed)
+		itemTSs = append(itemTSs, &ts)
+		if ts.Before(f.Updated) {
+			log.Println("Skipping old item")
+			continue
+		}
+		if ok := regexp.MustCompile(f.TitleFilter).MatchString(i.Title); !ok {
+			log.Println("Skipping bad titled item")
+			continue
+		}
+		item, err := newItem(i, f.ContentFilter)
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+		if err := item.save(); err != nil {
+			log.Println(err)
+			continue
+		}
+	}
+
+	f.Updated = latestTSPtr(gofeed.PublishedParsed, gofeed.UpdatedParsed)
+	f.Updated = latestTSPtr(append(itemTSs, &f.Updated)...)
+	return nil
+}
+
+func (f *feed) save() error {
+	b, err := f.Encode()
+	if err != nil {
+		return err
+	}
+	db := config.Values().DB
+	return db.Set(f.Key, b, NSFeeds)
+}
+
+func latestTSPtr(times ...*time.Time) time.Time {
+	var t time.Time
+	for i := range times {
+		if times[i] == nil {
+			continue
+		}
+		if times[i].After(t) {
+			t = *times[i]
+		}
+	}
+	return t
+}

rss/feed_test.go

@@ -0,0 +1,44 @@
+package rss
+
+import (
+	"encoding/json"
+	"local/rssmon3/config"
+	"os"
+	"testing"
+	"time"
+)
+
+func initRSSFeed() {
+	os.Args = []string{"a", "-db", "map"}
+	if err := config.New(); err != nil {
+		panic(err)
+	}
+}
+
+func TestRSSFeedEncodeDecode(t *testing.T) {
+	initRSSFeed()
+
+	f := newFeed("key")
+	f.Updated = time.Now()
+	f.TitleFilter = "a"
+	f.ContentFilter = "b"
+	f.Tags = []string{"c"}
+
+	b, err := f.Encode()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	g := newFeed("key")
+	if err := g.Decode(b); err != nil {
+		t.Fatal(err)
+	}
+
+	fb, _ := json.Marshal(f)
+	gb, _ := json.Marshal(f)
+	if string(fb) != string(gb) {
+		t.Errorf("%v => %v", *f, *g)
+	}
+
+	t.Logf("%s vs. %s", fb, gb)
+}

rss/item.go

@@ -0,0 +1,22 @@
+package rss
+
+import (
+	"errors"
+	"local/rssmon3/config"
+
+	"github.com/mmcdole/gofeed"
+)
+
+type Item struct {
+	*config.Encodable
+}
+
+func newItem(i *gofeed.Item, contentFilter string) (*Item, error) {
+	return &Item{
+		Encodable: &config.Encodable{},
+	}, errors.New("not impl")
+}
+
+func (i *Item) save() error {
+	return errors.New("not impl")
+}

rss/rss.go

@@ -0,0 +1,44 @@
+package rss
+
+import (
+	"local/rssmon3/config"
+	"local/rssmon3/monitor"
+	"log"
+)
+
+const nsFeeds = "nsFeeds"
+
+type RSS struct {
+	items chan *monitor.Item
+	config.Stoppable
+}
+
+func New(items chan *monitor.Item) *RSS {
+	return &RSS{
+		items: items,
+	}
+}
+
+func (rss *RSS) Run() error {
+	for {
+		select {
+		case <-rss.Stopped():
+			return nil
+		case i := <-rss.items:
+			if err := rss.update(i); err != nil {
+				log.Println(err)
+			}
+		}
+	}
+}
+
+func (rss *RSS) update(item *monitor.Item) error {
+	f := newFeed(item.Key)
+	if err := f.pull(); err != nil {
+		return err
+	}
+	if err := f.save(); err != nil {
+		return err
+	}
+	return nil
+}

rssmon3.REMOVED.git-id

@@ -0,0 +1 @@
+bf5140d93694bf40f561841cdee34065d4cc2455

server/routes.go

@@ -2,13 +2,11 @@ package server
 
 import (
 	"bytes"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"local/router"
 	"local/rssmon3/config"
-	"local/sandbox/selenium/copart/copart/auction"
 	"local/storage"
 	"log"
 	"net/http"
@@ -68,12 +66,8 @@ func (s *Server) auctions(w http.ResponseWriter, r *http.Request) {
 	r.ParseForm()
 	switch strings.ToLower(r.Method) {
 	case "get":
-		if len(r.Form) > 0 {
-			foo = s.query
-		} else {
 		foo = s.get
 	}
-	}
 	foo(w, r)
 }
 
@@ -96,70 +90,3 @@ func (s *Server) get(w http.ResponseWriter, r *http.Request) {
 	}
 	io.Copy(w, bytes.NewBuffer(value))
 }
-
-func (s *Server) query(w http.ResponseWriter, r *http.Request) {
-	log.Println("QUERY", r.URL.Path)
-	list, err := s.list()
-	if err != nil {
-		s.error(w, r, err)
-		return
-	}
-	form := r.Form
-	for _, value := range form {
-		for _, chunk := range value {
-			iteration := []string{}
-			for _, name := range list {
-				if strings.Contains(strings.ToLower(name), strings.ToLower(chunk)) {
-					iteration = append(iteration, name)
-				}
-			}
-			list = iteration
-		}
-	}
-	if len(list) < 1 {
-		s.notFound(w, r)
-		return
-	}
-	carsJSON, err := s.gatherCarsTruncatedJSON(list)
-	if err != nil {
-		s.error(w, r, fmt.Errorf("gather cars(%v): %v", list, err))
-		return
-	}
-	w.Write(carsJSON)
-}
-
-func (s *Server) list() ([]string, error) {
-	db := config.Values().DB
-	list := []string{}
-	if b, err := db.Get("LIST"); err == storage.ErrNotFound {
-		list = []string{}
-		db.Set("LIST", []byte("[]"))
-	} else if err != nil {
-		return nil, fmt.Errorf("bad list in storage: %v", err)
-	} else if err := json.Unmarshal(b, &list); err != nil {
-		return nil, fmt.Errorf("bad list in storage: %v", err)
-	}
-	return list, nil
-}
-
-func (s *Server) gatherCarsTruncatedJSON(keys []string) ([]byte, error) {
-	cars := []*auction.Car{}
-	k := 0
-	for _, key := range keys {
-		k += 1
-		if k > 50 {
-			cars = append(cars, &auction.Car{Title: "Max results reached, please search more specifically"})
-			break
-		}
-		b, err := config.Values().DB.Get(key)
-		if err != nil {
-			return nil, err
-		}
-		c := auction.NewCar()
-		if err := c.Decode(b); err != nil {
-			return nil, err
-		}
-		cars = append(cars, c)
-	}
-	return json.Marshal(cars)
-}