youtube-dl to yt-dlp

.gitignore

@@ -1,3 +1,4 @@
 ytubeSpdrun
 **/*.sw*
+exec-*
 youtuber

Dockerfile

@@ -0,0 +1,15 @@
+FROM frolvlad/alpine-glibc:alpine-3.9_glibc-2.29
+RUN apk update && apk add --no-cache ca-certificates
+
+RUN mkdir -p /var/log
+WORKDIR /main
+
+COPY . .
+
+RUN test -e /main/exec-youtuber
+
+ENV GOPATH=""
+ENV MNT="/mnt/"
+ENTRYPOINT ["/main/exec-youtuber"]
+CMD []
+

config/config.go

@@ -1,20 +1,50 @@
 package config
 
-var (
-	Feed = "https://www.youtube.com/feeds/videos.xml?channel_id=UCwX0AEx-qIhQ9kgtlNhyIXw"
-	Root = "/tmp"
+import (
+	"local/args"
+	"local/sandbox/contact/contact"
+	"os"
+	"time"
 )
 
-/*
-config = {
-   "path"      :  "../TV/Shepherds_Chapel"   ,
-   "rss"       :  "https://www.youtube.com/feeds/videos.xml?channel_id=UCwX0AEx-qIhQ9kgtlNhyIXw"   ,
-   "last"      :  0  ,
-   "lasttime"  : datetime.datetime(1999,1,1) ,
-   "onlynew"   :  False ,
-   "delay"     :  12 ,
-   "clear"     :  False ,
-   "logloc"    :  "./log"  ,
-   "idle"      :  False ,
+var (
+	Feed     string
+	Root     string
+	To       string
+	Subject  string
+	Emailer  *contact.Emailer
+	Interval time.Duration
+)
+
+func init() {
+	os.Setenv("LC_ALL", "C.UTF-8")
+
+	as := args.NewArgSet()
+
+	as.Append(args.STRING, "to", "message recipient", "squeaky2x3@gmail.com")
+	as.Append(args.STRING, "subject", "message subject", "youtuber error")
+	as.Append(args.STRING, "from", "message sender", "breellocaldev@gmail.com")
+	as.Append(args.STRING, "password", "message sender password", "ML3WQRFSqe9rQ8qNkm")
+	as.Append(args.STRING, "smtp", "smtp server:port", "smtp.gmail.com:465")
+	as.Append(args.STRING, "pop3", "pop3 server:port", "pop.gmail.com:995")
+
+	as.Append(args.STRING, "feed", "feed URL", "https://www.youtube.com/feeds/videos.xml?channel_id=UCwX0AEx-qIhQ9kgtlNhyIXw")
+	as.Append(args.STRING, "root", "root to save videos", "/tmp")
+	as.Append(args.DURATION, "interval", "check interval", time.Hour)
+
+	if err := as.Parse(); err != nil {
+		panic(err)
+	}
+
+	Feed = as.Get("feed").GetString()
+	Root = as.Get("root").GetString()
+	To = as.Get("to").GetString()
+	Subject = as.Get("subject").GetString()
+	Emailer = &contact.Emailer{
+		From:     as.Get("from").GetString(),
+		SMTP:     as.Get("smtp").GetString(),
+		POP3:     as.Get("pop3").GetString(),
+		Password: as.Get("password").GetString(),
+	}
+	Interval = as.GetDuration("interval")
 }
-*/

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"local/youtuber/config"
 	"local/youtuber/feed"
@@ -9,33 +10,58 @@ import (
 	"os"
 	"path"
 	"strings"
+	"time"
 )
 
 func main() {
-	os.Setenv("LC_ALL", "C.UTF-8")
-	results, err := feed.Fetch(config.Feed)
+	client, err := youtubedl.New()
 	if err != nil {
 		panic(err)
 	}
-	log.Println(results)
+	interval := config.Interval
+	for true {
+		err := do(client)
+		if err != nil {
+			log.Println(err)
+			if err := config.Emailer.Send(
+				config.To,
+				config.Subject,
+				fmt.Sprintf("error getting youtuber: %v", err),
+			); err != nil {
+				panic(err)
+			}
+		}
+		log.Printf("sleeping until %v", time.Now().Add(interval))
+		time.Sleep(interval)
+	}
+}
 
-	youtubedl, err := youtubedl.New()
+func do(client *youtubedl.Client) error {
+	errs := ""
+
+	results, err := feed.Fetch(config.Feed)
 	if err != nil {
-		panic(err)
+		return err
 	}
 
 	for _, result := range results {
-		target := fmt.Sprintf(
-			"s%02d%02de%02d_%s.%%(ext)s",
-			result.Date.Year()-2000,
-			result.Date.Month(),
-			result.Date.Day(),
-			strings.Join(strings.Split(result.Title, " ")[2:], "_"),
-		)
-		if err := youtubedl.Download(result.Link, path.Join(config.Root, target)); err != nil {
-			panic(err)
+		seasonNumber := fmt.Sprintf("%02d%02d", result.Date.Year()-2000, result.Date.Month())
+		episodeNumber := fmt.Sprintf("%02d", result.Date.Day())
+		season := fmt.Sprintf("Season_%s", seasonNumber)
+		episode := fmt.Sprintf("Episode_%s_-_%s", episodeNumber, strings.Join(strings.Split(result.Title, " ")[2:], "_"))
+		target := fmt.Sprintf("%s/%s.%%(ext)s", season, episode)
+		target = path.Join(config.Root, target)
+		if _, err := os.Stat(strings.ReplaceAll(target, "%(ext)s", "mp4")); !os.IsNotExist(err) {
+			log.Printf("already exists: %s", target)
+			continue
+		}
+		os.MkdirAll(path.Dir(target), os.ModePerm)
+		if err := client.Download(result.Link, target); err != nil {
+			errs += ", " + err.Error()
 		}
-		return
 	}
-	log.Println(youtubedl)
+	if errs != "" {
+		return errors.New(errs)
+	}
+	return nil
 }

youtubedl/client.go

@@ -3,6 +3,7 @@ package youtubedl
 import (
 	"fmt"
 	"log"
+	"os"
 	"os/exec"
 	"path"
 )
@@ -15,15 +16,55 @@ func New() (*Client, error) {
 }
 
 func (c *Client) Download(video, local string) error {
-	cmd := exec.Command("youtube-dl", "-f", "22", "-o", local, "--write-sub", "--write-auto-sub", "--sub-format", "srt", video)
-	log.Println(cmd.Path, cmd.Args)
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		err = fmt.Errorf("%v: %s", err, out)
+	if _, ok := os.LookupEnv("TEST"); ok {
+		log.Println("youtube-dl download", video, local)
+		return nil
+	}
+	errs := []error{}
+	for _, extras := range [][]string{
+		[]string{"--add-metadata", "--metadata-from-title", ".*[0-9]/[0-9][0-9] (?P<title>.+)"},
+		[]string{},
+	} {
+		args := append([]string{
+			"/tmp/yt-dlp",
+			"-f",
+			"22",
+			"-o",
+			local,
+			"--write-sub",
+			"--write-auto-sub",
+			"--sub-format",
+			"srt",
+			"--no-check-certificate",
+		}, extras...)
+		args = append(args, video)
+		cmd := exec.Command(args[0], args[1:]...)
+		log.Println(cmd.Path, cmd.Args)
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			log.Printf("failed with %v: %s", err, out)
+			errs = append(errs, fmt.Errorf("%v: %s", err, out))
+		} else {
+			log.Printf("passed with %s", out)
+			errs = nil
+			break
+		}
+	}
+	if len(errs) > 0 {
+		err := fmt.Errorf("%v", errs)
 		return err
 	}
-	cmd = exec.Command("python3", "-m", "vtt_to_srt", path.Dir(local))
-	out, err = cmd.CombinedOutput()
+	cmd := exec.Command(
+		"/bin/sh",
+		"-c",
+		fmt.Sprintf(
+			"true; python3 -m vtt_to_srt %q || python3 /usr/bin/vtt_to_srt.py %q || vtt_to_srt %q",
+			path.Dir(local),
+			path.Dir(local),
+			path.Dir(local),
+		),
+	)
+	out, err := cmd.CombinedOutput()
 	if err != nil {
 		err = fmt.Errorf("%v: %s", err, out)
 		return err

youtubedl/install.go

@@ -2,11 +2,17 @@ package youtubedl
 
 import (
 	"errors"
+	"fmt"
+	"log"
+	"os"
 	"os/exec"
 )
 
 func install() error {
-	if err := installPyPip3(); err != nil {
+	if _, ok := os.LookupEnv("TEST"); ok {
+		return nil
+	}
+	if err := installPyPip3FFMPEG(); err != nil {
 		return err
 	}
 	if err := installYtdl(); err != nil {
@@ -18,39 +24,59 @@ func install() error {
 	return nil
 }
 
-func installPyPip3() error {
+func installPyPip3FFMPEG() error {
 	hasPy := exec.Command("python3", "--version")
 	hasPip := exec.Command("pip3", "--version")
+	hasFFMPEG := exec.Command("which", "ffmpeg")
 	if err := hasPy.Run(); err != nil {
 	} else if err := hasPip.Run(); err != nil {
+	} else if err := hasFFMPEG.Run(); err != nil {
 	} else {
 		return nil
 	}
 	for _, combo := range [][]string{
-		[]string{"sudo", "apt", "install", "python3", "pip3"},
-		[]string{"apt", "install", "python3", "pip3"},
-		[]string{"apk", "add", "python3", "py3-pip"},
-		[]string{"sudo", "apk", "add", "python3", "py3-pip"},
+		[]string{"sudo", "apt", "install", "python3", "pip3", "ffmpeg"},
+		[]string{"apt", "install", "python3", "pip3", "ffmpeg"},
+		[]string{"apk", "add", "python3", "py3-pip", "ffmpeg"},
+		[]string{"sudo", "apk", "add", "python3", "py3-pip", "ffmpeg"},
+		[]string{"/sbin/apk", "add", "python3", "py3-pip", "ffmpeg"},
+		[]string{"sudo", "/sbin/apk", "add", "python3", "py3-pip", "ffmpeg"},
 	} {
 		cmd := exec.Command(combo[0], combo[1:]...)
-		if err := cmd.Run(); err == nil {
+		err := cmd.Run()
+		if err == nil {
 			return nil
 		}
+		log.Printf("%v: %v", err, combo)
 	}
-	return errors.New("cannot get python3 and pip3")
+	return errors.New("cannot get python3 and pip3 and ffmpeg")
 }
 
 func installYtdl() error {
-	cmd := exec.Command("youtube-dl", "--version")
+	cmd := exec.Command("/tmp/yt-dlp", "--version")
 	if err := cmd.Run(); err == nil {
 		return err
 	}
-	cmd = exec.Command("sudo", "pip3", "install", "youtube-dl")
+	os.Remove("/tmp/yt-dlp")
+	cmd = exec.Command(
+		"wget",
+		"https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp",
+		"--no-check-certificate",
+		"-O",
+		"/tmp/yt-dlp",
+	)
 	if err := cmd.Run(); err != nil {
-		cmd = exec.Command("pip3", "install", "youtube-dl")
-		if err := cmd.Run(); err != nil {
-			return err
-		}
+		o, _ := cmd.CombinedOutput()
+		return fmt.Errorf("failed wget yt-dlp: %v: %s", err, o)
+	}
+	cmd = exec.Command(
+		"chmod",
+		"a+rx",
+		"/tmp/yt-dlp",
+	)
+	if err := cmd.Run(); err != nil {
+		o, _ := cmd.CombinedOutput()
+		return fmt.Errorf("failed chmod yt-dlp: %v: %s", err, o)
 	}
 	return nil
 }