package fetch import ( "bytes" "context" "fmt" "io" "log" "net/http" "net/url" "show-rss/src/cron" "show-rss/src/feeds" "show-rss/src/webhooks" "strings" "text/template" "time" ) func Main(ctx context.Context) error { return cron.Cron(ctx, feeds.Next, One) } func One(ctx context.Context) error { return feeds.ForEach(ctx, func(feed feeds.Feed) error { if err := one(ctx, feed); err != nil { return fmt.Errorf("failed to cron %s (%+v): %w", feed.Entry.ID, feed.Version, err) } return nil }) } func one(ctx context.Context, feed feeds.Feed) error { if should, err := feed.ShouldExecute(); err != nil { return err } else if !should { return nil } log.Printf("fetching %s", feed.Version.URL) items, err := feed.Fetch(ctx) if err != nil { return err } log.Printf("fetched feed %s: %d items", feed.Version.URL, len(items)) for _, item := range items { if err := oneItem(ctx, feed, item); err != nil { return err } } return feed.Executed(ctx) } func oneItem(ctx context.Context, feed feeds.Feed, item feeds.Item) error { type Arg struct { Feed feeds.Feed Item feeds.Item } arg := Arg{ Feed: feed, Item: item, } wmethod, wurl, wbody := feed.Webhook(ctx) method, err := render(wmethod, arg) if err != nil { return err } wurl, err = render(wurl, arg) if err != nil { return err } body, err := render(wbody, arg) if err != nil { return err } var user *url.Userinfo if u, _ := url.Parse(wurl); u.User != nil { user = u.User u.User = &url.Userinfo{} wurl = u.String() } if did, err := webhooks.Did(ctx, method, wurl, body); err != nil || did { return err } log.Printf("webhooking %s %s: %s", method, wurl, body) req, err := http.NewRequest(method, wurl, strings.NewReader(body)) if err != nil { return err } req = req.WithContext(ctx) if user != nil { u := user.Username() p, _ := user.Password() req.SetBasicAuth(u, p) } c := http.Client{ Timeout: time.Minute, Transport: &http.Transport{DisableKeepAlives: true}, } resp, err := c.Do(req) if err != nil { return err } defer resp.Body.Close() defer io.Copy(io.Discard, resp.Body) if resp.StatusCode > 204 { b, _ := io.ReadAll(resp.Body) return fmt.Errorf("failed %s %s w/ %s: (%d) %s", method, wurl, body, resp.StatusCode, b) } return webhooks.Record(ctx, method, wurl, body) } func render(text string, arg any) (string, error) { tmpl, err := template.New("render").Parse(text) if err != nil { return "", err } b := bytes.NewBuffer(nil) err = tmpl.Execute(b, arg) return string(b.Bytes()), err }