Implement podcast sidecar

Former-commit-id: 93d6ac101e02ea562c87853549080eb1ac85a3c6

handlers/podcast/main.go

@@ -0,0 +1,234 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"local/args"
+	"local/storage"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/mmcdole/gofeed"
+)
+
+const sessionHeader = "X-Transmission-Session-Id"
+
+type Config struct {
+	url      string
+	vpntor   string
+	outdir   string
+	interval time.Duration
+	last     time.Time
+	db       storage.DB
+	ctx      context.Context
+	can      context.CancelFunc
+}
+
+func main() {
+	config, err := config()
+	if err != nil {
+		panic(err)
+	}
+	log.Println(config)
+
+	for {
+		if err := mainLoop(config); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func mainLoop(config *Config) error {
+	block := config.interval - time.Since(config.last)
+	log.Printf("Blocking %v", block)
+	select {
+	case <-time.After(block):
+		if err := pull(config.db, config.vpntor, config.outdir, config.url); err != nil {
+			log.Println(err)
+		}
+		config.last = time.Now()
+	case <-config.ctx.Done():
+		if err := config.ctx.Err(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func config() (*Config, error) {
+	as := args.NewArgSet()
+	as.Append(args.STRING, "url", "url of rss feed", "http://192.168.0.86:33419/api/tag/podcast")
+	as.Append(args.STRING, "vpntor", "url of vpntor", "http://192.168.0.86:9091/transmission/rpc")
+	as.Append(args.DURATION, "interval", "interval to check feed", "30m")
+	as.Append(args.STRING, "outdir", "save dir", "/data/completed-rss")
+	as.Append(args.STRING, "db", "db type", "map")
+	as.Append(args.STRING, "addr", "db addr", "")
+	as.Append(args.STRING, "user", "db user", "")
+	as.Append(args.STRING, "pass", "db pass", "")
+	if err := as.Parse(); err != nil {
+		return &Config{}, err
+	}
+
+	db, err := storage.New(
+		storage.TypeFromString(as.Get("db").GetString()),
+		as.Get("addr").GetString(),
+		as.Get("user").GetString(),
+		as.Get("pass").GetString(),
+	)
+	if err != nil {
+		panic(err)
+	}
+
+	ctx, can := context.WithCancel(context.Background())
+	return &Config{
+		url:      as.Get("url").GetString(),
+		vpntor:   as.Get("vpntor").GetString(),
+		interval: as.Get("interval").GetDuration(),
+		outdir:   as.Get("outdir").GetString(),
+		db:       db,
+		ctx:      ctx,
+		can:      can,
+	}, nil
+}
+
+func pull(db storage.DB, vpntor, outdir, url string) error {
+	gofeed, err := getGoFeed(url)
+	if err != nil {
+		return err
+	}
+	log.Printf("feed: %v", gofeed.Title)
+	for _, item := range gofeed.Items {
+		if ok, err := isDone(db, item.Link); err != nil {
+			return err
+		} else if ok {
+			continue
+		}
+		s, err := getItemContent(item)
+		if err != nil {
+			return err
+		}
+		if err := handle(vpntor, outdir, s); err != nil {
+			return err
+		}
+		if err := db.Set(item.Link, []byte{}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func getGoFeed(url string) (*gofeed.Feed, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	return gofeed.NewParser().Parse(resp.Body)
+}
+
+func getItemContent(item *gofeed.Item) (string, error) {
+	s := item.Description
+	if s == "" {
+		s = item.Content
+	}
+	if s == "" {
+		resp, err := http.Get(item.Link)
+		if err != nil {
+			return s, err
+		}
+		defer resp.Body.Close()
+		b, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return s, err
+		}
+		s = string(b)
+	}
+	return s, nil
+}
+
+func isDone(db storage.DB, url string) (bool, error) {
+	_, err := db.Get(url)
+	if err == storage.ErrNotFound {
+		return false, nil
+	}
+	return true, err
+}
+
+func handle(vpntor, outdir, content string) error {
+	links := findMagnets(content)
+	sort.Strings(links)
+	for i := range links {
+		link := links[i]
+		if i > 0 && link == links[i-1] {
+			continue
+		}
+		log.Println(link)
+		if err := fetch(link, outdir); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func findMagnets(s string) []string {
+	magnetRegexp := regexp.MustCompile(`http[^"]*[0-9]+\.mp3`)
+	return magnetRegexp.FindAllString(s, -1)
+}
+
+func fetch(link, outdir string) error {
+	out := path.Join(outdir, path.Base(link))
+	if _, err := os.Stat(out); err == nil {
+		return nil
+	} else if os.IsNotExist(err) {
+	} else if err != nil {
+		return err
+	}
+
+	resp, err := http.Get(link)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	f, err := os.Create(out)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	io.Copy(f, resp.Body)
+	return nil
+}
+
+func buildReqBody(outdir, magnet string) io.Reader {
+	return strings.NewReader(fmt.Sprintf(`
+      {
+         "method": "torrent-add",
+         "arguments": {
+            "filename": %q,
+            "download-dir": %q
+         }
+      }
+   `, magnet, outdir))
+}
+
+func getSessionID(vpntor string) (string, error) {
+	resp, err := http.Get(vpntor)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	id := resp.Header.Get(sessionHeader)
+	if id == "" {
+		err = errors.New("session id header not found")
+	}
+	return id, err
+}

rss/testdata/rss.xml.REMOVED.git-id

@@ -1 +0,0 @@
-d36f6c94dfbbaaeac339bfaec9d0dd13b0ff099b