From ccfefd5b772d8e7b6ab6f48a291f83b08c907a9e Mon Sep 17 00:00:00 2001 From: bel Date: Wed, 29 Jan 2020 03:44:44 +0000 Subject: [PATCH] works with ubuntu --- .gitignore | 3 ++ config/config.go | 20 +++++++++ feed/feed.go | 37 ++++++++++++++++ feed/item.go | 9 ++++ main.go | 41 ++++++++++++++++++ testdata/main.go | 101 +++++++++++++++++++++++++++++++++++++++++++ testdata/rsync.sh | 3 ++ youtubedl/client.go | 32 ++++++++++++++ youtubedl/install.go | 39 +++++++++++++++++ 9 files changed, 285 insertions(+) create mode 100755 .gitignore create mode 100644 config/config.go create mode 100644 feed/feed.go create mode 100644 feed/item.go create mode 100644 main.go create mode 100755 testdata/main.go create mode 100755 testdata/rsync.sh create mode 100644 youtubedl/client.go create mode 100644 youtubedl/install.go diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..324ccec --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +ytubeSpdrun +**/*.sw* +youtuber diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..736c228 --- /dev/null +++ b/config/config.go @@ -0,0 +1,20 @@ +package config + +var ( + Feed = "https://www.youtube.com/feeds/videos.xml?channel_id=UCwX0AEx-qIhQ9kgtlNhyIXw" + Root = "/tmp" +) + +/* +config = { + "path" : "../TV/Shepherds_Chapel" , + "rss" : "https://www.youtube.com/feeds/videos.xml?channel_id=UCwX0AEx-qIhQ9kgtlNhyIXw" , + "last" : 0 , + "lasttime" : datetime.datetime(1999,1,1) , + "onlynew" : False , + "delay" : 12 , + "clear" : False , + "logloc" : "./log" , + "idle" : False , +} +*/ diff --git a/feed/feed.go b/feed/feed.go new file mode 100644 index 0000000..c669867 --- /dev/null +++ b/feed/feed.go @@ -0,0 +1,37 @@ +package feed + +import ( + "net/http" + "strings" + "time" + + "github.com/mmcdole/gofeed" +) + +func Fetch(s string) ([]Item, error) { + resp, err := http.Get(s) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + gofeed, err := gofeed.NewParser().Parse(resp.Body) + if err != nil { + return nil, err + } + + links := make([]Item, 0) + + for _, item := range gofeed.Items { + timestamp := strings.Split(item.Title, " ")[1] + t, err := time.Parse("1/2/06", timestamp) + if err == nil { + links = append(links, Item{ + Link: item.Link, + Title: strings.ReplaceAll(item.Title, "/", "-"), + Date: t, + }) + } + } + return links, nil +} diff --git a/feed/item.go b/feed/item.go new file mode 100644 index 0000000..2457555 --- /dev/null +++ b/feed/item.go @@ -0,0 +1,9 @@ +package feed + +import "time" + +type Item struct { + Link string + Date time.Time + Title string +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..1df4fa8 --- /dev/null +++ b/main.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "local/youtuber/config" + "local/youtuber/feed" + "local/youtuber/youtubedl" + "log" + "os" + "path" + "strings" +) + +func main() { + os.Setenv("LC_ALL", "C.UTF-8") + results, err := feed.Fetch(config.Feed) + if err != nil { + panic(err) + } + log.Println(results) + + youtubedl, err := youtubedl.New() + if err != nil { + panic(err) + } + + for _, result := range results { + target := fmt.Sprintf( + "s%02d%02de%02d_%s.%%(ext)s", + result.Date.Year()-2000, + result.Date.Month(), + result.Date.Day(), + strings.Join(strings.Split(result.Title, " ")[2:], "_"), + ) + if err := youtubedl.Download(result.Link, path.Join(config.Root, target)); err != nil { + panic(err) + } + return + } + log.Println(youtubedl) +} diff --git a/testdata/main.go b/testdata/main.go new file mode 100755 index 0000000..c13870c --- /dev/null +++ b/testdata/main.go @@ -0,0 +1,101 @@ +package main + +import ( + "errors" + "fmt" + "io/ioutil" + "local/jbtserve/jbt" + "local/system/sysconf" + "log" + "net" + "net/http" + "os" + "os/signal" + "path" + "regexp" + "strings" + "syscall" + + "github.com/otium/ytdl" +) + +var validURL = regexp.MustCompile("(?is)https?://([a-z1-9]+.)+[a-z]+/.*") + +func main() { + if os.Getenv("MNT") == "" { + os.Setenv("MNT", "./") + } + if !strings.HasSuffix(os.Getenv("MNT"), "/") { + os.Setenv("MNT", os.Getenv("MNT")+"/") + } + config := sysconf.Get("ytubespdrun") + exit := make(chan bool) + go jbt.RegisterDeregister(config.Name, config.Port, exit) + log.Print("Listen on", net.JoinHostPort(config.IP, config.Port)) + go func() { + log.Print(http.ListenAndServe(net.JoinHostPort(config.IP, config.Port), http.HandlerFunc(accept))) + }() + sigc := make(chan os.Signal) + signal.Notify(sigc, + syscall.SIGINT, + ) + <-sigc + <-exit + <-exit +} + +func accept(w http.ResponseWriter, r *http.Request) { + config := sysconf.Get("ytubespdrun") + if r.Method != "POST" { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + panic(err) + } + token := string(body) + token = strings.Split(token, "&")[0] + log.Print("Received request", token) + if !validURL.MatchString(token) { + w.WriteHeader(http.StatusBadRequest) + return + } + if err := download(token, config.Log); err != nil { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "Cannot download %s, encountered error %v", token, err) + log.Print("Failed request", token) + return + } + w.WriteHeader(http.StatusOK) + log.Print("Completed request", token) +} + +func download(url, log string) error { + vid, err := ytdl.GetVideoInfo(url) + if err != nil { + return err + } + format := vid.Formats.Best(ytdl.FormatExtensionKey) + if len(format) < 1 { + return errors.New("no formats available") + } + log.Print("Downloading format", format[0].Resolution, format[0].Extension, format[0].Itag, "for video", vid.Title) + file, err := os.Create(os.Getenv("MNT") + strings.Replace(fmt.Sprintf("%s.%s", vid.Title, format[0].Extension), " ", "_", -1)) + if err != nil { + return err + } + defer file.Close() + return vid.Download(format[0], file) +} + +func pathAdvance(r *http.Request) string { + p := path.Clean("/" + r.URL.Path) + i := strings.Index(p[1:], "/") + 1 + if i <= 0 { + r.URL.Path = "/" + return p[1:] + } + r.URL.Path = p[i:] + return p[1:i] +} diff --git a/testdata/rsync.sh b/testdata/rsync.sh new file mode 100755 index 0000000..8f7abc1 --- /dev/null +++ b/testdata/rsync.sh @@ -0,0 +1,3 @@ +#! /bin/bash + +rsync -avzP --delete --exclude "alt.db" --exclude "alt.log" . /mnt/ipod/squeaky2x3D/curProjects/curProjects/Go/src/local/$(basename $(pwd))/ diff --git a/youtubedl/client.go b/youtubedl/client.go new file mode 100644 index 0000000..737f640 --- /dev/null +++ b/youtubedl/client.go @@ -0,0 +1,32 @@ +package youtubedl + +import ( + "fmt" + "log" + "os/exec" + "path" +) + +type Client struct { +} + +func New() (*Client, error) { + return &Client{}, install() +} + +func (c *Client) Download(video, local string) error { + cmd := exec.Command("youtube-dl", "-f", "22", "-o", local, "--write-sub", "--write-auto-sub", "--sub-format", "srt", video) + log.Println(cmd.Path, cmd.Args) + out, err := cmd.CombinedOutput() + if err != nil { + err = fmt.Errorf("%v: %s", err, out) + return err + } + cmd = exec.Command("python3", "-m", "vtt_to_srt", path.Dir(local)) + out, err = cmd.CombinedOutput() + if err != nil { + err = fmt.Errorf("%v: %s", err, out) + return err + } + return nil +} diff --git a/youtubedl/install.go b/youtubedl/install.go new file mode 100644 index 0000000..fe905a4 --- /dev/null +++ b/youtubedl/install.go @@ -0,0 +1,39 @@ +package youtubedl + +import "os/exec" + +func install() error { + cmd := exec.Command("youtube-dl", "--version") + if err := cmd.Run(); err == nil { + return err + } + cmd = exec.Command("sudo", "pip3", "install", "youtube-dl") + if err := cmd.Run(); err == nil { + return err + } + return nil +} + +func installYtdl() error { + cmd := exec.Command("youtube-dl", "--version") + if err := cmd.Run(); err == nil { + return err + } + cmd = exec.Command("sudo", "pip3", "install", "youtube-dl") + if err := cmd.Run(); err == nil { + return err + } + return nil +} + +func installVTT() error { + cmd := exec.Command("python3", "-m", "vtt_to_srt") + if err := cmd.Run(); err == nil { + return err + } + cmd = exec.Command("sudo", "pip3", "install", "vtt_to_srt3") + if err := cmd.Run(); err == nil { + return err + } + return nil +}