From be065c5dcb1edc13987bfe611d2208444f4fadfd Mon Sep 17 00:00:00 2001 From: bel Date: Sat, 22 Jun 2019 08:58:54 -0600 Subject: [PATCH] encodable a good Former-commit-id: a0afebd82dfd256ec70c02c607af2a9b3b4b046f --- config/encodable.go | 22 +++++++++ config/new_test.go | 1 + config/stoppable.go | 7 +++ main.go | 9 ++-- monitor/item.go | 4 +- monitor/monitor.go | 7 +-- rss/feed.go | 110 +++++++++++++++++++++++++++++++++++++++++ rss/feed_test.go | 44 +++++++++++++++++ rss/item.go | 22 +++++++++ rss/rss.go | 44 +++++++++++++++++ rssmon3.REMOVED.git-id | 1 + server/routes.go | 75 +--------------------------- 12 files changed, 260 insertions(+), 86 deletions(-) create mode 100644 config/encodable.go create mode 100644 config/stoppable.go create mode 100644 rss/feed.go create mode 100644 rss/feed_test.go create mode 100644 rss/item.go create mode 100644 rss/rss.go create mode 100644 rssmon3.REMOVED.git-id diff --git a/config/encodable.go b/config/encodable.go new file mode 100644 index 0000000..4bf3331 --- /dev/null +++ b/config/encodable.go @@ -0,0 +1,22 @@ +package config + +import ( + "bytes" + "encoding/gob" +) + +type Encodable struct{} + +func (e *Encodable) Encode() ([]byte, error) { + buff := bytes.NewBuffer(nil) + enc := gob.NewEncoder(buff) + err := enc.Encode(e) + return buff.Bytes(), err +} + +func (e *Encodable) Decode(b []byte) error { + buff := bytes.NewBuffer(b) + dec := gob.NewDecoder(buff) + err := dec.Decode(e) + return err +} diff --git a/config/new_test.go b/config/new_test.go index 8117dcc..0fc247e 100644 --- a/config/new_test.go +++ b/config/new_test.go @@ -34,6 +34,7 @@ func TestNew(t *testing.T) { } } }() + os.Args = []string{"a", "-db", "map"} if err := New(); err != nil { t.Fatal(err) } diff --git a/config/stoppable.go b/config/stoppable.go new file mode 100644 index 0000000..df61597 --- /dev/null +++ b/config/stoppable.go @@ -0,0 +1,7 @@ +package config + +type Stoppable struct{} + +func (s Stoppable) Stopped() <-chan struct{} { + return Values().Ctx.Done() +} diff --git a/main.go b/main.go index 4b286bc..de357c8 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,8 @@ package main import ( - "local/rssmon2/monitor" "local/rssmon3/config" + "local/rssmon3/monitor" "local/rssmon3/server" "log" "os" @@ -27,10 +27,7 @@ func main() { } go InterruptAfter(s.Run, sigc) - m, err := monitor.New() - if err != nil { - panic(err) - } + m := monitor.New() go InterruptAfter(m.Run, sigc) signal.Notify(sigc, @@ -40,7 +37,7 @@ func main() { syscall.SIGQUIT, ) <-sigc - can() + config.Values().Can() config.Values().DB.Close() } diff --git a/monitor/item.go b/monitor/item.go index 41bdf34..7c54837 100644 --- a/monitor/item.go +++ b/monitor/item.go @@ -12,6 +12,7 @@ import ( type Item struct { Key string + *config.Encodable } const nsLast = "nsLast" @@ -22,7 +23,8 @@ var forever = time.Duration(time.Hour * 99999) func NewItem(key string, interval time.Duration) (*Item, error) { i := &Item{ - Key: key, + Key: key, + Encodable: &config.Encodable{}, } if err := i.setInterval(interval); err != nil { diff --git a/monitor/monitor.go b/monitor/monitor.go index 50e5dfb..89c4e00 100644 --- a/monitor/monitor.go +++ b/monitor/monitor.go @@ -9,6 +9,7 @@ type Monitor struct { queue *Queue Incoming chan *Item Outgoing chan *Item + config.Stoppable } func New() *Monitor { @@ -22,7 +23,7 @@ func New() *Monitor { func (m *Monitor) Run() error { for { select { - case <-m.stopped(): + case <-m.Stopped(): return nil case i := <-m.enqueued(): m.enqueue(i) @@ -33,10 +34,6 @@ func (m *Monitor) Run() error { } } -func (m *Monitor) stopped() <-chan struct{} { - return config.Values().Ctx.Done() -} - func (m *Monitor) enqueued() chan *Item { return m.Incoming } diff --git a/rss/feed.go b/rss/feed.go new file mode 100644 index 0000000..166c8c5 --- /dev/null +++ b/rss/feed.go @@ -0,0 +1,110 @@ +package rss + +import ( + "errors" + "local/rssmon3/config" + "log" + "net/http" + "regexp" + "time" + + "github.com/mmcdole/gofeed" +) + +const NSFeeds = "NSFeeds" + +type feed struct { + Key string + URL string + Updated time.Time + TitleFilter string + ContentFilter string + Tags []string + *config.Encodable +} + +func newFeed(key string) *feed { + return &feed{ + Key: key, + Encodable: &config.Encodable{}, + } +} + +func (f *feed) load() error { + if f.Key == "" { + return errors.New("cannot load nil feed") + } + db := config.Values().DB + b, err := db.Get(f.Key, NSFeeds) + if err != nil { + return err + } + return f.Decode(b) +} + +func (f *feed) pull() error { + if f.URL == "" { + if err := f.load(); err != nil { + return err + } + } + resp, err := http.Get(f.URL) + if err != nil { + return err + } + defer resp.Body.Close() + + gofeed, err := gofeed.NewParser().Parse(resp.Body) + if err != nil { + return err + } + + itemTSs := []*time.Time{} + for _, i := range gofeed.Items { + ts := latestTSPtr(i.UpdatedParsed, i.PublishedParsed) + itemTSs = append(itemTSs, &ts) + if ts.Before(f.Updated) { + log.Println("Skipping old item") + continue + } + if ok := regexp.MustCompile(f.TitleFilter).MatchString(i.Title); !ok { + log.Println("Skipping bad titled item") + continue + } + item, err := newItem(i, f.ContentFilter) + if err != nil { + log.Println(err) + continue + } + if err := item.save(); err != nil { + log.Println(err) + continue + } + } + + f.Updated = latestTSPtr(gofeed.PublishedParsed, gofeed.UpdatedParsed) + f.Updated = latestTSPtr(append(itemTSs, &f.Updated)...) + return nil +} + +func (f *feed) save() error { + b, err := f.Encode() + if err != nil { + return err + } + db := config.Values().DB + return db.Set(f.Key, b, NSFeeds) +} + +func latestTSPtr(times ...*time.Time) time.Time { + var t time.Time + for i := range times { + if times[i] == nil { + continue + } + if times[i].After(t) { + t = *times[i] + } + } + return t +} diff --git a/rss/feed_test.go b/rss/feed_test.go new file mode 100644 index 0000000..e603ae6 --- /dev/null +++ b/rss/feed_test.go @@ -0,0 +1,44 @@ +package rss + +import ( + "encoding/json" + "local/rssmon3/config" + "os" + "testing" + "time" +) + +func initRSSFeed() { + os.Args = []string{"a", "-db", "map"} + if err := config.New(); err != nil { + panic(err) + } +} + +func TestRSSFeedEncodeDecode(t *testing.T) { + initRSSFeed() + + f := newFeed("key") + f.Updated = time.Now() + f.TitleFilter = "a" + f.ContentFilter = "b" + f.Tags = []string{"c"} + + b, err := f.Encode() + if err != nil { + t.Fatal(err) + } + + g := newFeed("key") + if err := g.Decode(b); err != nil { + t.Fatal(err) + } + + fb, _ := json.Marshal(f) + gb, _ := json.Marshal(f) + if string(fb) != string(gb) { + t.Errorf("%v => %v", *f, *g) + } + + t.Logf("%s vs. %s", fb, gb) +} diff --git a/rss/item.go b/rss/item.go new file mode 100644 index 0000000..c21b42d --- /dev/null +++ b/rss/item.go @@ -0,0 +1,22 @@ +package rss + +import ( + "errors" + "local/rssmon3/config" + + "github.com/mmcdole/gofeed" +) + +type Item struct { + *config.Encodable +} + +func newItem(i *gofeed.Item, contentFilter string) (*Item, error) { + return &Item{ + Encodable: &config.Encodable{}, + }, errors.New("not impl") +} + +func (i *Item) save() error { + return errors.New("not impl") +} diff --git a/rss/rss.go b/rss/rss.go new file mode 100644 index 0000000..88a553b --- /dev/null +++ b/rss/rss.go @@ -0,0 +1,44 @@ +package rss + +import ( + "local/rssmon3/config" + "local/rssmon3/monitor" + "log" +) + +const nsFeeds = "nsFeeds" + +type RSS struct { + items chan *monitor.Item + config.Stoppable +} + +func New(items chan *monitor.Item) *RSS { + return &RSS{ + items: items, + } +} + +func (rss *RSS) Run() error { + for { + select { + case <-rss.Stopped(): + return nil + case i := <-rss.items: + if err := rss.update(i); err != nil { + log.Println(err) + } + } + } +} + +func (rss *RSS) update(item *monitor.Item) error { + f := newFeed(item.Key) + if err := f.pull(); err != nil { + return err + } + if err := f.save(); err != nil { + return err + } + return nil +} diff --git a/rssmon3.REMOVED.git-id b/rssmon3.REMOVED.git-id new file mode 100644 index 0000000..b802e9d --- /dev/null +++ b/rssmon3.REMOVED.git-id @@ -0,0 +1 @@ +bf5140d93694bf40f561841cdee34065d4cc2455 \ No newline at end of file diff --git a/server/routes.go b/server/routes.go index 53511a4..9639a34 100644 --- a/server/routes.go +++ b/server/routes.go @@ -2,13 +2,11 @@ package server import ( "bytes" - "encoding/json" "errors" "fmt" "io" "local/router" "local/rssmon3/config" - "local/sandbox/selenium/copart/copart/auction" "local/storage" "log" "net/http" @@ -68,11 +66,7 @@ func (s *Server) auctions(w http.ResponseWriter, r *http.Request) { r.ParseForm() switch strings.ToLower(r.Method) { case "get": - if len(r.Form) > 0 { - foo = s.query - } else { - foo = s.get - } + foo = s.get } foo(w, r) } @@ -96,70 +90,3 @@ func (s *Server) get(w http.ResponseWriter, r *http.Request) { } io.Copy(w, bytes.NewBuffer(value)) } - -func (s *Server) query(w http.ResponseWriter, r *http.Request) { - log.Println("QUERY", r.URL.Path) - list, err := s.list() - if err != nil { - s.error(w, r, err) - return - } - form := r.Form - for _, value := range form { - for _, chunk := range value { - iteration := []string{} - for _, name := range list { - if strings.Contains(strings.ToLower(name), strings.ToLower(chunk)) { - iteration = append(iteration, name) - } - } - list = iteration - } - } - if len(list) < 1 { - s.notFound(w, r) - return - } - carsJSON, err := s.gatherCarsTruncatedJSON(list) - if err != nil { - s.error(w, r, fmt.Errorf("gather cars(%v): %v", list, err)) - return - } - w.Write(carsJSON) -} - -func (s *Server) list() ([]string, error) { - db := config.Values().DB - list := []string{} - if b, err := db.Get("LIST"); err == storage.ErrNotFound { - list = []string{} - db.Set("LIST", []byte("[]")) - } else if err != nil { - return nil, fmt.Errorf("bad list in storage: %v", err) - } else if err := json.Unmarshal(b, &list); err != nil { - return nil, fmt.Errorf("bad list in storage: %v", err) - } - return list, nil -} - -func (s *Server) gatherCarsTruncatedJSON(keys []string) ([]byte, error) { - cars := []*auction.Car{} - k := 0 - for _, key := range keys { - k += 1 - if k > 50 { - cars = append(cars, &auction.Car{Title: "Max results reached, please search more specifically"}) - break - } - b, err := config.Values().DB.Get(key) - if err != nil { - return nil, err - } - c := auction.NewCar() - if err := c.Decode(b); err != nil { - return nil, err - } - cars = append(cars, c) - } - return json.Marshal(cars) -}