134 lines
2.5 KiB
Go
134 lines
2.5 KiB
Go
package fetch
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"show-rss/src/cron"
|
|
"show-rss/src/feeds"
|
|
"show-rss/src/webhooks"
|
|
"strings"
|
|
"text/template"
|
|
"time"
|
|
)
|
|
|
|
func Main(ctx context.Context) error {
|
|
return cron.Cron(ctx, feeds.Next, One)
|
|
}
|
|
|
|
func One(ctx context.Context) error {
|
|
return feeds.ForEach(ctx, func(feed feeds.Feed) error {
|
|
if err := one(ctx, feed); err != nil {
|
|
return fmt.Errorf("failed to cron %s (%+v): %w", feed.Entry.ID, feed.Version, err)
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func one(ctx context.Context, feed feeds.Feed) error {
|
|
if should, err := feed.ShouldExecute(); err != nil {
|
|
return err
|
|
} else if !should {
|
|
return nil
|
|
}
|
|
log.Printf("fetching %s", feed.Version.URL)
|
|
|
|
items, err := feed.Fetch(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Printf("fetched feed %s: %d items", feed.Version.URL, len(items))
|
|
|
|
for _, item := range items {
|
|
if err := oneItem(ctx, feed, item); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return feed.Executed(ctx)
|
|
}
|
|
|
|
func oneItem(ctx context.Context, feed feeds.Feed, item feeds.Item) error {
|
|
type Arg struct {
|
|
Feed feeds.Feed
|
|
Item feeds.Item
|
|
}
|
|
arg := Arg{
|
|
Feed: feed,
|
|
Item: item,
|
|
}
|
|
|
|
wmethod, wurl, wbody := feed.Webhook(ctx)
|
|
|
|
method, err := render(wmethod, arg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
wurl, err = render(wurl, arg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
body, err := render(wbody, arg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var user *url.Userinfo
|
|
if u, _ := url.Parse(wurl); u.User != nil {
|
|
user = u.User
|
|
u.User = &url.Userinfo{}
|
|
wurl = u.String()
|
|
}
|
|
|
|
if did, err := webhooks.Did(ctx, method, wurl, body); err != nil || did {
|
|
return err
|
|
}
|
|
log.Printf("webhooking %s %s: %s", method, wurl, body)
|
|
|
|
req, err := http.NewRequest(method, wurl, strings.NewReader(body))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req = req.WithContext(ctx)
|
|
if user != nil {
|
|
u := user.Username()
|
|
p, _ := user.Password()
|
|
req.SetBasicAuth(u, p)
|
|
}
|
|
|
|
c := http.Client{
|
|
Timeout: time.Minute,
|
|
Transport: &http.Transport{DisableKeepAlives: true},
|
|
}
|
|
resp, err := c.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
defer io.Copy(io.Discard, resp.Body)
|
|
|
|
if resp.StatusCode > 204 {
|
|
b, _ := io.ReadAll(resp.Body)
|
|
return fmt.Errorf("failed %s %s w/ %s: (%d) %s", method, wurl, body, resp.StatusCode, b)
|
|
}
|
|
|
|
return webhooks.Record(ctx, method, wurl, body)
|
|
}
|
|
|
|
func render(text string, arg any) (string, error) {
|
|
tmpl, err := template.New("render").Parse(text)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
b := bytes.NewBuffer(nil)
|
|
err = tmpl.Execute(b, arg)
|
|
return string(b.Bytes()), err
|
|
}
|