to go mod

Former-commit-id: 93d6ac101e02ea562c87853549080eb1ac85a3c6

Dockerfile

@@ -0,0 +1,16 @@
+
+FROM golang:1.13-alpine as certs
+RUN apk update && apk add --no-cache ca-certificates
+
+FROM busybox:glibc
+RUN mkdir -p /var/log
+WORKDIR /main
+COPY --from=certs /etc/ssl/certs /etc/ssl/certs
+
+COPY . .
+
+ENV GOPATH=""
+ENV MNT="/mnt/"
+ENTRYPOINT ["/main/exec-rssmon3"]
+CMD []
+

config/new.go

@@ -15,9 +15,13 @@ func New() error {
 	as.Append(args.STRING, "port", "port to listen on", "33419")
 	as.Append(args.STRING, "db", "database type code", storage.MAP.String())
 	as.Append(args.STRING, "ns", "namespace", storage.DefaultNamespace)
+	as.Append(args.STRING, "proxy", "http proxy", "")
 	if err := as.Parse(); err != nil {
 		return err
 	}
+	if proxy := as.Get("proxy").GetString(); len(proxy) > 0 {
+		os.Setenv("http_proxy", proxy)
+	}
 	ctx, can := context.WithCancel(context.Background())
 	config = Config{
 		db:               as.Get("db").GetString(),

go.mod

@@ -0,0 +1,20 @@
+module rssmon3
+
+go 1.23.3
+
+require (
+	github.com/golang-collections/go-datastructures v0.0.0-20150211160725-59788d5eb259
+	github.com/gorilla/feeds v1.2.0
+	github.com/mmcdole/gofeed v1.3.0
+)
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	golang.org/x/net v0.4.0 // indirect
+	golang.org/x/text v0.5.0 // indirect
+)

go.sum

@@ -0,0 +1,47 @@
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang-collections/go-datastructures v0.0.0-20150211160725-59788d5eb259 h1:ZHJ7+IGpuOXtVf6Zk/a3WuHQgkC+vXwaqfUBDFwahtI=
+github.com/golang-collections/go-datastructures v0.0.0-20150211160725-59788d5eb259/go.mod h1:9Qcha0gTWLw//0VNka1Cbnjvg3pNKGFdAm7E9sBabxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/gorilla/feeds v1.2.0 h1:O6pBiXJ5JHhPvqy53NsjKOThq+dNFm8+DFrxBEdzSCc=
+github.com/gorilla/feeds v1.2.0/go.mod h1:WMib8uJP3BbY+X8Szd1rA5Pzhdfh+HCCAYT2z7Fza6Y=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
+github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
+golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
+golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

handlers/torrent/go.mod

@@ -0,0 +1,58 @@
+module rssmon3-torrent
+
+go 1.22.2
+
+require (
+	gitea.inhome.blapointe.com/local/args v0.0.0-20240109214601-658deda479a4
+	gitea.inhome.blapointe.com/local/storage v0.0.0-20231109151605-736d446d407d
+	github.com/mmcdole/gofeed v1.3.0
+)
+
+require (
+	cloud.google.com/go v0.33.1 // indirect
+	gitea.inhome.blapointe.com/local/logb v0.0.0-20231109150430-1221d87a6dbc // indirect
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 // indirect
+	github.com/abbot/go-http-auth v0.4.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/aws/aws-sdk-go v1.15.81 // indirect
+	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/golang/protobuf v1.2.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/gomodule/redigo v1.8.5 // indirect
+	github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.9.5 // indirect
+	github.com/klauspost/cpuid v1.2.3 // indirect
+	github.com/minio/md5-simd v1.1.0 // indirect
+	github.com/minio/minio-go/v6 v6.0.57 // indirect
+	github.com/minio/sha256-simd v0.1.1 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/ncw/rclone v1.46.0 // indirect
+	github.com/ncw/swift v1.0.44 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46 // indirect
+	github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c // indirect
+	github.com/syndtr/goleveldb v1.0.0 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.0.2 // indirect
+	github.com/xdg-go/stringprep v1.0.2 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.mongodb.org/mongo-driver v1.7.2 // indirect
+	golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 // indirect
+	golang.org/x/net v0.4.0 // indirect
+	golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba // indirect
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
+	golang.org/x/sys v0.3.0 // indirect
+	golang.org/x/text v0.5.0 // indirect
+	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
+	google.golang.org/api v0.0.0-20181120235003-faade3cbb06a // indirect
+	google.golang.org/appengine v1.3.0 // indirect
+	gopkg.in/ini.v1 v1.42.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)

handlers/torrent/go.sum

@@ -0,0 +1,281 @@
+bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
+cloud.google.com/go v0.33.1 h1:fmJQWZ1w9PGkHR1YL/P7HloDvqlmKQ4Vpb7PC2e+aCk=
+cloud.google.com/go v0.33.1/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+gitea.inhome.blapointe.com/local/args v0.0.0-20240109214601-658deda479a4 h1:4qBHjKAiEwRV1A1tN1JK6PsLV1+UwESXKrjGqfCCdNk=
+gitea.inhome.blapointe.com/local/args v0.0.0-20240109214601-658deda479a4/go.mod h1:SqCOE3bE3wvrztVIQGHuyxHKfDjRKU9EWhBdkmkiwyc=
+gitea.inhome.blapointe.com/local/logb v0.0.0-20231109150430-1221d87a6dbc h1:u3akQkq12V8xWXlcDgjZxIK6hqo6f1eHd9KOxAKMoKc=
+gitea.inhome.blapointe.com/local/logb v0.0.0-20231109150430-1221d87a6dbc/go.mod h1:KwilawX4UgD4HxSJAVFEzkuckrnHeQrd49KwUX6GpYU=
+gitea.inhome.blapointe.com/local/storage v0.0.0-20231109151605-736d446d407d h1:SQq4hWImnvtrRfpPgOW4go+sBjMluuhRL/43b8L0yB4=
+gitea.inhome.blapointe.com/local/storage v0.0.0-20231109151605-736d446d407d/go.mod h1:TRK5z/XTT6jws++Q21Y8DQot+5vZGTNeHf+RjuY8aQk=
+github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
+github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 h1:6X8iB881g299aNEv6KXrcjL31iLOH7yA6NXoQX+MbDg=
+github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
+github.com/a8m/tree v0.0.0-20180321023834-3cf936ce15d6/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=
+github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
+github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
+github.com/anacrolix/dms v0.0.0-20180117034613-8af4925bffb5/go.mod h1:DGqLjaZ3ziKKNRt+U5Q9PLWJ52Q/4rxfaaH/b3QYKaE=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.15.81 h1:va7uoFaV9uKAtZ6BTmp1u7paoMsizYRRLvRuoC07nQ8=
+github.com/aws/aws-sdk-go v1.15.81/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
+github.com/billziss-gh/cgofuse v1.1.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
+github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/coreos/bbolt v0.0.0-20180318001526-af9db2027c98/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/djherbis/times v1.1.0/go.mod h1:CGMZlo255K5r4Yw0b9RRfFQpM2y7uOmxg4jm9HsaVf8=
+github.com/dropbox/dropbox-sdk-go-unofficial v5.4.0+incompatible/go.mod h1:lr+LhMM3F6Y3lW1T9j2U5l7QeuWm87N9+PPXo3yH4qY=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
+github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
+github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
+github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
+github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
+github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
+github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
+github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
+github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
+github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
+github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
+github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
+github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
+github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
+github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
+github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
+github.com/goftp/file-driver v0.0.0-20180502053751-5d604a0fc0c9/go.mod h1:GpOj6zuVBG3Inr9qjEnuVTgBlk2lZ1S9DcoFiXWyKss=
+github.com/goftp/server v0.0.0-20190111142836-88de73f463af/go.mod h1:k/SS6VWkxY7dHPhoMQ8IdRu8L4lQtmGbhyXGg+vCnXE=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.5 h1:nRAxCa+SVsyjSBrtZmG/cqb6VbTmuRzpg/PoTFlpumc=
+github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jlaffaye/ftp v0.0.0-20181101112434-47f21d10f0ee/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
+github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
+github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
+github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
+github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M=
+github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs=
+github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
+github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
+github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
+github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
+github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw=
+github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
+github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2/go.mod h1:MLIrzg7gp/kzVBxRE1olT7CWYMCklcUWU+ekoxOD9x0=
+github.com/ncw/rclone v1.46.0 h1:5SY9lB6LIIXqwOaCWp5twUqNWgJVZMW+0D4rX90A2+E=
+github.com/ncw/rclone v1.46.0/go.mod h1:+uFY4HNpat/yXXIEin5ETWXxIwEplC+eDe/vT8vlk1w=
+github.com/ncw/swift v1.0.44 h1:EKvOTvUxElbpDWqxsyVaVGvc2IfuOqQnRmjnR2AGhQ4=
+github.com/ncw/swift v1.0.44/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
+github.com/nsf/termbox-go v0.0.0-20181027232701-60ab7e3d12ed/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ=
+github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd/go.mod h1:4soZNh0zW0LtYGdQ416i0jO0EIqMGcbtaspRS4BDvRQ=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
+github.com/pengsrc/go-shared v0.2.0/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.8.3/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46 h1:w2CpS5muK+jyydnmlkqpAhzKmHmMBzBkfYUDjQNS1Dk=
+github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46/go.mod h1:U2bmx0hDj8EyDdcxmD5t3XHDnBFnyNNc22n1R4008eM=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/sevlyar/go-daemon v0.1.4/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE=
+github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
+github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
+github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c h1:fyKiXKO1/I/B6Y2U8T7WdQGWzwehOuGIrljPtt7YTTI=
+github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
+github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/t3rm1n4l/go-mega v0.0.0-20190205172012-55a226cf41da/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w=
+github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
+github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc=
+github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yunify/qingstor-sdk-go v2.2.15+incompatible/go.mod h1:w6wqLDQ5bBTzxGJ55581UrSwLrsTAsdo9N6yX/8d9RY=
+go.mongodb.org/mongo-driver v1.7.2 h1:pFttQyIiJUHEn50YfZgC9ECjITMT44oiN36uArf/OFg=
+go.mongodb.org/mongo-driver v1.7.2/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
+golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba h1:YDkOrzGLLYybtuP6ZgebnO4OWYEYVMFSniazXsxrFN8=
+golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
+golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.0.0-20181120235003-faade3cbb06a h1:yMfgT1baklxtECXVk3UtZBELVXtVhDbK3/7xLFkFypw=
+google.golang.org/api v0.0.0-20181120235003-faade3cbb06a/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
+gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

handlers/torrent/main.go

@@ -7,14 +7,16 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"local/args"
-	"local/storage"
 	"log"
 	"net/http"
+	"path"
 	"regexp"
 	"strings"
 	"time"
 
+	"gitea.inhome.blapointe.com/local/args"
+	"gitea.inhome.blapointe.com/local/storage"
+
 	"github.com/mmcdole/gofeed"
 )
 
@@ -99,31 +101,52 @@ func config() (*Config, error) {
 }
 
 func pull(db storage.DB, vpntor, outdir, url string) error {
-	gofeed, err := getGoFeed(url)
+	gfeed, err := getGoFeed(url)
 	if err != nil {
 		return err
 	}
-	log.Printf("feed: %v", gofeed.Title)
-	for _, item := range gofeed.Items {
-		if ok, err := isDone(db, item.Link); err != nil {
-			return err
-		} else if ok {
-			continue
-		}
-		s, err := getItemContent(item)
-		if err != nil {
-			return err
-		}
-		if err := handle(vpntor, outdir, s); err != nil {
-			return err
-		}
-		if err := db.Set(item.Link, []byte{}); err != nil {
+	return _pull(db, vpntor, outdir, gfeed.Title, gfeed.Items)
+}
+
+func _pull(db storage.DB, vpntor, outdir, title string, items []*gofeed.Item) error {
+	log.Printf("feed: %v", title)
+	for _, item := range items {
+		if err := __pull(db, vpntor, outdir, item); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
+func __pull(db storage.DB, vpntor, outdir string, item *gofeed.Item) error {
+	if ok, err := isDone(db, item.Link); err != nil {
+		return err
+	} else if ok {
+		return nil
+	}
+	s, err := getItemContent(item)
+	if err != nil {
+		return err
+	}
+	if item.Author == nil {
+		item.Author = &gofeed.Person{Name: "."}
+	}
+	if err := handle(vpntor, __pullOutput(outdir, item.Author.Name, item), item.Link, s); err != nil {
+		return err
+	}
+	return db.Set(item.Link, []byte{})
+}
+
+func __pullOutput(outdir, author string, item *gofeed.Item) string {
+	output := path.Join(outdir, author)
+	for _, category := range item.Categories {
+		if categoryOutDir := strings.TrimPrefix(category, "outdir:"); category != categoryOutDir {
+			output = categoryOutDir
+		}
+	}
+	return output
+}
+
 func getGoFeed(url string) (*gofeed.Feed, error) {
 	resp, err := http.Get(url)
 	if err != nil {
@@ -161,9 +184,9 @@ func isDone(db storage.DB, url string) (bool, error) {
 	return true, err
 }
 
-func handle(vpntor, outdir, content string) error {
-	log.Printf("magnets: %v", findMagnets(content))
-	for _, magnet := range findMagnets(content) {
+func handle(vpntor, outdir, link, content string) error {
+	log.Printf("magnets: %v", append(findMagnets(content), findMagnets(link)...))
+	for _, magnet := range append(findMagnets(content), findMagnets(link)...) {
 		resp, err := submit(vpntor, outdir, magnet)
 		if err != nil {
 			return err
@@ -176,8 +199,16 @@ func handle(vpntor, outdir, content string) error {
 }
 
 func findMagnets(s string) []string {
-	magnetRegexp := regexp.MustCompile(`magnet:.xt[^ $"]*`)
-	return magnetRegexp.FindAllString(s, -1)
+	patterns := []string{
+		`magnet:.xt[^ $"]*`,
+		`https:..[^ "]*\.torrent`,
+	}
+	matches := []string{}
+	for _, pattern := range patterns {
+		magnetRegexp := regexp.MustCompile(pattern)
+		matches = append(matches, magnetRegexp.FindAllString(s, -1)...)
+	}
+	return matches
 }
 
 func submit(vpntor, outdir, magnet string) (*http.Response, error) {

handlers/torrent/main_test.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"local/storage"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -14,6 +13,8 @@ import (
 	"testing"
 	"time"
 
+	"gitea.inhome.blapointe.com/local/storage"
+
 	"github.com/mmcdole/gofeed"
 )
 
@@ -239,6 +240,14 @@ func TestFindMagnets(t *testing.T) {
 			s: `here is some magnet:-xt1 and magnet:-xt2 another one <a href="magnet:-xt3">link</a>`,
 			l: 3,
 		},
+		{
+			s: `here is a link https://my.domain/a.b/c.torrent, and another <a href="https://my.domain/a/b/c.torrent"></a>`,
+			l: 2,
+		},
+		{
+			s: `https://my.domain/a.b/c.torrent`,
+			l: 1,
+		},
 	}
 
 	for i, c := range cases {
@@ -248,3 +257,12 @@ func TestFindMagnets(t *testing.T) {
 		}
 	}
 }
+
+func Test__PullOutput(t *testing.T) {
+	if got := __pullOutput("x", "y", &gofeed.Item{Categories: []string{}}); got != "x/y" {
+		t.Error(got)
+	}
+	if got := __pullOutput("", "", &gofeed.Item{Categories: []string{"outdir:x"}}); got != "x" {
+		t.Error(got)
+	}
+}

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"local/lastn/lastn"
 	"local/rssmon3/config"
 	"local/rssmon3/handlers"
 	"local/rssmon3/monitor"
@@ -8,6 +9,7 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"syscall"
 	"time"
 )
@@ -37,6 +39,8 @@ func main() {
 	h := handlers.New(m.Outgoing)
 	go InterruptAfter(h.Run, sigc)
 
+	go EnqueueBackups()
+
 	signal.Notify(sigc,
 		syscall.SIGHUP,
 		syscall.SIGINT,
@@ -54,3 +58,41 @@ func InterruptAfter(foo func() error, c chan os.Signal) {
 	}
 	c <- syscall.SIGINT
 }
+
+func EnqueueBackups() {
+	realpath, err := filepath.Abs(config.Values().Addr)
+	if err != nil {
+		log.Println("dir", config.Values().Addr, "not found, so no backups")
+		return
+	}
+	conf := lastn.Config{
+		N:         3,
+		Root:      realpath,
+		Ns:        "backups",
+		Store:     "files",
+		StoreAddr: realpath + "-backups",
+	}
+	log.Printf("backups conf: %+v", conf)
+	lastn, err := lastn.New(conf)
+	if err != nil {
+		log.Println("backups disabled:", realpath, ":", err)
+		return
+	}
+	ticker := time.NewTicker(time.Hour * 24)
+	log.Println("backup initial:", err)
+	if err := lastn.Push(); err != nil {
+		log.Println("backup failed:", err)
+	}
+	if err := lastn.Clean(); err != nil {
+		log.Println("backup clean failed:", err)
+	}
+	for _ = range ticker.C {
+		log.Println("backing up...")
+		if err := lastn.Push(); err != nil {
+			log.Println("backup push failed:", err)
+		}
+		if err := lastn.Clean(); err != nil {
+			log.Println("backup clean failed:", err)
+		}
+	}
+}

rss/feed.go

@@ -1,12 +1,14 @@
 package rss
 
 import (
+	"encoding/json"
 	"errors"
 	"local/rssmon3/config"
 	"log"
 	"net/http"
 	"regexp"
 	"sort"
+	"strconv"
 	"time"
 
 	"github.com/mmcdole/gofeed"
@@ -21,6 +23,8 @@ type Feed struct {
 	TitleFilter   string
 	ContentFilter string
 	Tags          []string
+	Copyright     string
+	Headers       []byte
 }
 
 func SubmitFeed(f *Feed) error {
@@ -59,7 +63,16 @@ func (f *Feed) Pull() error {
 			return err
 		}
 	}
-	resp, err := http.Get(f.URL)
+	req, err := http.NewRequest(http.MethodGet, f.URL, nil)
+	if err != nil {
+		return err
+	}
+	var headers map[string]string
+	json.Unmarshal(f.Headers, &headers)
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return err
 	}
@@ -72,7 +85,7 @@ func (f *Feed) Pull() error {
 
 	itemTSs := []*time.Time{}
 	for _, i := range gofeed.Items {
-		item, err := newItem(i, f.ContentFilter)
+		item, err := newItem(i, f.ContentFilter, f.Copyright)
 		if err != nil {
 			log.Println("[Pull]", err)
 			continue
@@ -83,9 +96,10 @@ func (f *Feed) Pull() error {
 			continue
 		}
 		if ok := regexp.MustCompile(f.TitleFilter).MatchString(item.Title); !ok {
-			log.Println("[Pull]", "Skipping bad titled item")
+			log.Printf("[Pull] Skipping bad titled item: %v doesn't match /%v/", item.Title, f.TitleFilter)
 			continue
 		}
+		log.Printf("[Pull] Saving item %v for %v /%v/", f.Key, f.URL, f.TitleFilter)
 		if err := item.save(f.Key); err != nil {
 			log.Println("[Pull]", err)
 			continue
@@ -123,7 +137,7 @@ func (f *Feed) Items(limit int) ([]*Item, error) {
 }
 
 func (f *Feed) List(limit int) ([]string, error) {
-	keys, err := config.Values().DB.List([]string{nsItems, f.Key})
+	keys, err := config.Values().DB.List([]string{nsItems, f.Key}, " ", "}}}}}", strconv.Itoa(limit), "-")
 	if err != nil {
 		return nil, err
 	}

rss/feed_test.go

@@ -177,3 +177,37 @@ func TestRSSFeedPull(t *testing.T) {
 		t.Fatal(i)
 	}
 }
+
+func TestRSSFeedListLimitedDescending(t *testing.T) {
+	initRSSFeed()
+
+	s := mockRSS()
+	defer s.Close()
+
+	f := newFeed("key")
+	f.TitleFilter = "50."
+	f.ContentFilter = "b"
+	f.Tags = []string{"c"}
+	f.URL = s.URL
+
+	log.SetOutput(bytes.NewBuffer(nil))
+	defer log.SetOutput(os.Stderr)
+	if err := f.Pull(); err != nil {
+		t.Fatal(err)
+	}
+	log.SetOutput(os.Stderr)
+
+	itemKeys, err := f.List(5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(itemKeys) != 5 {
+		t.Fatal(len(itemKeys))
+	}
+
+	for i := range itemKeys {
+		if i > 0 && itemKeys[i] > itemKeys[i-1] {
+			t.Error(itemKeys[i], ">", itemKeys[i-1])
+		}
+	}
+}

rss/item.go

@@ -16,20 +16,22 @@ import (
 const nsItems = "nsItems"
 
 type Item struct {
-	Title   string
-	Link    string
-	Content string
-	TS      time.Time
+	Title     string
+	Link      string
+	Content   string
+	TS        time.Time
+	Copyright string
 }
 
 type Items []*Item
 
-func newItem(i *gofeed.Item, contentFilter string) (*Item, error) {
+func newItem(i *gofeed.Item, contentFilter, copyright string) (*Item, error) {
 	item := &Item{
-		Title:   i.Title,
-		Link:    i.Link,
-		Content: i.Content,
-		TS:      latestTSPtr(i.UpdatedParsed, i.PublishedParsed),
+		Title:     i.Title,
+		Link:      i.Link,
+		Content:   i.Content,
+		TS:        latestTSPtr(i.UpdatedParsed, i.PublishedParsed),
+		Copyright: copyright,
 	}
 
 	if item.Content == "" {
@@ -61,6 +63,8 @@ func newItem(i *gofeed.Item, contentFilter string) (*Item, error) {
 
 	item.Content = fmt.Sprintf(`<a href="%s">%s</a><br>%s`, item.Link, item.Title, item.Content)
 
+	item.Content = cleanSpecialCharacters(clearBlankLines(item.Content))
+
 	return item, nil
 }
 
@@ -69,7 +73,11 @@ func (i *Item) Encode() ([]byte, error) {
 }
 
 func (i *Item) Decode(b []byte) error {
-	return config.Decode(b, i)
+	if err := config.Decode(b, i); err != nil {
+		return err
+	}
+	i.Content = clearBlankLines(i.Content)
+	return nil
 }
 
 func (i *Item) save(ns1 string, ns ...string) error {
@@ -91,7 +99,11 @@ func (i *Item) Load(key, ns1 string, ns ...string) error {
 	if err != nil {
 		return err
 	}
-	return config.Decode(b, i)
+	err = config.Decode(b, i)
+	if err == nil {
+		i.Content = clearBlankLines(i.Content)
+	}
+	return err
 }
 
 func (is Items) Len() int {
@@ -107,3 +119,16 @@ func (is Items) Swap(i, j int) {
 	is[i] = is[j]
 	is[j] = k
 }
+
+func cleanSpecialCharacters(s string) string {
+	return strings.ToValidUTF8(s, "?")
+}
+
+func clearBlankLines(s string) string {
+	r := regexp.MustCompile(`<br/>`)
+	s = r.ReplaceAllLiteralString(s, "<br>")
+	r = regexp.MustCompile(`(?m)<br>\s*(<br>\s*)*`)
+	s = r.ReplaceAllLiteralString(s, "<br>")
+	r = regexp.MustCompile(`(?m)\s\s*`)
+	return r.ReplaceAllLiteralString(s, "\n")
+}

rss/item_test.go

@@ -3,6 +3,7 @@ package rss
 import (
 	"fmt"
 	"net/http"
+	"strings"
 	"testing"
 
 	"github.com/mmcdole/gofeed"
@@ -30,12 +31,12 @@ func TestRSSItemNewEncodeDecode(t *testing.T) {
 	gofeed.Items[0].Content = ""
 	gofeed.Items[0].Description = ""
 
-	item, err := newItem(gofeed.Items[0], ".*")
+	item, err := newItem(gofeed.Items[0], ".*", "")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	itemB, err := newItem(gofeed.Items[0], "Podcast")
+	itemB, err := newItem(gofeed.Items[0], "Podcast", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -75,3 +76,101 @@ func TestRSSItemNewEncodeDecode(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestClearBlankLines(t *testing.T) {
+	cases := map[string]struct {
+		in       string
+		outLines int
+	}{
+		"remove with and without whitespace": {
+			in: `<html>
+<head>
+   <script>something</script>
+   <style>body{hello:"blue";}</style>
+</head>
+<body>
+<br>
+<br>
+   <br>  
+<br/><br/>
+<br/> <br/>
+</body>
+         </html>`,
+			outLines: 1,
+		},
+		"sample content": {
+			in: `<html>
+Linux
+Blog
+<br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>
+<br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>
+<img src="https://www.jeremymorgan.com/images/programming-blog.png" alt="Programming
+Blog"/><h1><a href="https://www.jeremymorgan.com/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Jeremy
+Morgan</a></h1><h2>Mostly
+Coherent
+Ramblings
+of
+a
+Silicon
+Forest
+Software
+Developer</h2><br/><br/><br/>
+<ul><br/>
+<li>RSS</li><br/>
+<br/></ul><br/>
+<br/><br/>
+<br/>
+<br/>
+<br/>
+<br/><br/>
+<br/><ul><li><a href="https://www.jeremymorgan.com/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Home</a></li><li><a href="https://www.jeremymorgan.com/tutorials/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Tutorials</a></li><li><a href="https://www.jeremymorgan.com/blog/programming/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Programming</a></li><li><a href="https://www.jeremymorgan.com/blog/raspberry-pi/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Raspberry
+Pi</a></li><li><a href="https://www.jeremymorgan.com/blog/linux/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Linux</a></li><li><a href="https://www.jeremymorgan.com/blog/dotnet/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">.Net</a></li><li><a href="https://github.com/JeremyMorgan" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">My
+Projects</a></li><li><a href="https://www.jeremymorgan.com/contact/" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">Contact
+Me</a></li></ul><br/><br/>
+<br/>
+<br/>
+<br/><br/>
+<br/>
+<br/>
+<br/>
+<h1>Trying
+Out
+         </html>`,
+			outLines: 6,
+		},
+		"remove with and without whitespace 2": {
+			in: `Daniel
+Tillett</a><br/>
+<br/>
+<br/>
+<br/>
+<br/>
+<br/>
+<br/>
+<br/>
+<br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>
+<br/><br/><br/><br/><br/><br/>
+
+<br/>
+<br/>
+<br/>
+<br/>
+<br/>
+<br/>
+<br/>
+         `,
+			outLines: 1,
+		},
+	}
+
+	for name, c := range cases {
+		out := clearBlankLines(c.in)
+		cnts := []int{
+			strings.Count(out, "<br>"),
+			strings.Count(out, "<br/>"),
+		}
+		if v := cnts[0] + cnts[1]; v != c.outLines {
+			t.Errorf("%v: want %v lines, got %v from %q: %q", name, c.outLines, v, c.in, out)
+		}
+	}
+}

rss/serialize.go

@@ -43,6 +43,7 @@ func WriteFeed(w io.Writer, tag string, items []*Item) error {
 			Title:       item.Title,
 			Link:        &feeds.Link{Href: item.Link},
 			Description: item.Content,
+			Author:      &feeds.Author{Name: item.Copyright},
 			Created:     item.TS,
 		}
 	}

rss/serialize_test.go

@@ -47,6 +47,7 @@ func TestRSSWriteFeed(t *testing.T) {
 	f.ContentFilter = "b"
 	f.Tags = []string{"c"}
 	f.URL = s.URL
+	f.Copyright = "copyright"
 
 	log.SetOutput(bytes.NewBuffer(nil))
 	defer log.SetOutput(os.Stderr)

rss/testdata/rss.xml

@@ -11,6 +11,7 @@
         <itunes:author>Rooster Teeth</itunes:author>
         <itunes:summary>On a weekly basis the Rooster Teeth crew discuss gaming, films, popular culture, the internet, and projects that they're currently working on. Watch LIVE every Monday at 5PM CT on svod.roosterteeth.com. Watch the free video version at YouTube.com/RoosterTeeth.</itunes:summary>
         <description>On a weekly basis the Rooster Teeth crew discuss gaming, films, popular culture, the internet, and projects that they're currently working on. Watch LIVE every Monday at 5PM CT on svod.roosterteeth.com. Watch the free video version at YouTube.com/RoosterTeeth.</description>
+        <content>my content here</content>
         <itunes:owner>
             <itunes:name>Gus Sorola, Rooster Teeth</itunes:name>
             <itunes:email>podcast@roosterteeth.com</itunes:email>
@@ -24,6 +25,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 18 Jun 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Chris Demarais, Blaine Gibson, and Gavin Free as they discuss raditation, toilet paper face, Chris's continued haircuts, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_549.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">549</guid>
             <itunes:duration>01:33:59</itunes:duration>
@@ -35,6 +38,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 11 Jun 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Chris Demarais, Barbara Dunkelman, and Geoff Ramsey as they discuss haircuts, old apartments, reality television, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_548.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">548</guid>
             <itunes:duration>01:31:10</itunes:duration>
@@ -46,6 +51,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 04 Jun 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Miles Luna, Ellie Main, and Becca Frasier as they discuss Apple’s WWDC event, going to space, school, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_547.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">547</guid>
             <itunes:duration>01:36:49</itunes:duration>
@@ -57,6 +64,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 28 May 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Blaine Gibson, and Burnie Burns as they discuss decimation, Chernobyl, recasting the MCU, frumpy Vader and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_546.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">546</guid>
             <itunes:duration>01:36:13</itunes:duration>
@@ -68,6 +77,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 21 May 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Barbara Dunkelman, and Burnie Burns as they discuss steaks, sleep apnea, the Game of Thrones finale (in the final 30 minutes), and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_545.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">545</guid>
             <itunes:duration>01:43:16</itunes:duration>
@@ -79,6 +90,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 14 May 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Barbara Dunkelman, Burnie Burns, and special guest Markiplier as they discuss medical issues, live-streaming, Game of Thrones (in the final 30 minutes), and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_544.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">544</guid>
             <itunes:duration>01:37:03</itunes:duration>
@@ -90,6 +103,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 07 May 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Chris Demarais, Barbara Dunkelman, and Burnie Burns as they discuss VR games, Chris’ car accident, Avengers Endgame (in the final 25 minutes), and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_543.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">543</guid>
             <itunes:duration>01:49:39</itunes:duration>
@@ -101,6 +116,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 30 Apr 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Barbara Dunkelman, and Burnie Burns as they discuss horizontal vs vertical videos, cats, Game of Thrones (in the last 15 minutes), and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_542.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">542</guid>
             <itunes:duration>01:47:22</itunes:duration>
@@ -112,6 +129,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 23 Apr 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, and Jon Risinger as they discuss computer problems, rideshare stories, a weird form of vaping, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_541.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">541</guid>
             <itunes:duration>01:37:42</itunes:duration>
@@ -123,6 +142,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 16 Apr 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Barbara Dunkelman, and Blaine Gibson as they discuss the world’s biggest plane, black holes, getting your car towed, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_540.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">540</guid>
             <itunes:duration>01:35:07</itunes:duration>
@@ -134,6 +155,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 09 Apr 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Barbara Dunkelman, and Burnie Burns as they discuss Game of Thrones, food poisoning, the American flag, Stuber, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_539.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">539</guid>
             <itunes:duration>01:43:39</itunes:duration>
@@ -145,6 +168,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 02 Apr 2019 19:00:00 +0000</pubDate>
             <description>Join Matt Hullum, Gavin Free, Ellie Main, Burnie Burns, and Chad James as they discuss glitter, companies Rooster Teeth has outlasted, timesheets, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_538.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">538</guid>
             <itunes:duration>01:45:00</itunes:duration>
@@ -156,6 +181,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 26 Mar 2019 19:00:00 +0000</pubDate>
             <description>Join Ashley Jenkins, Ellie Main, Barbara Dunkelman, Burnie Burns, and Chad James as they discuss Marvel and DC, stand-up comedy, pregnancy, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_537.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">537</guid>
             <itunes:duration>01:37:44</itunes:duration>
@@ -167,6 +193,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 19 Mar 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Barbara Dunkelman and special guest James Willems as they discuss firework holidays, Benson the dog, failed conventions, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_536.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">536</guid>
             <itunes:duration>01:54:54</itunes:duration>
@@ -178,6 +205,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 12 Mar 2019 19:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Burnie Burns, and special guests Rhett and Link as they discuss kolaches, internetainment, the history of the business, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_535.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">535</guid>
             <itunes:duration>01:36:22</itunes:duration>
@@ -189,6 +217,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 05 Mar 2019 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Barbara Dunkelman, and Burnie Burns as they discuss pancakes, flat-earthers, Fyre Festival, and more on this week's RT Podcast! And be sure to stick around for a special post-show interview with Zachary Levi.</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_534.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">534</guid>
             <itunes:duration>02:10:36</itunes:duration>
@@ -200,6 +229,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 26 Feb 2019 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Ellie Main, and Burnie Burns as they discuss s’mores, Brexit, conspiracy theories, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_533.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">533</guid>
             <itunes:duration>01:41:56</itunes:duration>
@@ -211,6 +241,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 19 Feb 2019 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Barbara Dunkelman, and Burnie Burns as they discuss Apex Legends, printer technology, time zones, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_532.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">532</guid>
             <itunes:duration>01:35:25</itunes:duration>
@@ -222,6 +253,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 12 Feb 2019 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Barbara Dunkelman, and Burnie Burns as they discuss internet outrage, a Vine quiz, YouTube changes, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_531.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">531</guid>
             <itunes:duration>01:39:09</itunes:duration>
@@ -233,6 +265,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 05 Feb 2019 20:00:00 +0000</pubDate>
             <description>Join Blaine Gibson, Gavin Free, Barbara Dunkelman, and Burnie Burns as they discuss movie effects and if they hold up, the Marvel Cinematic Universe, Gavin getting shot with a paintball, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_530.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">530</guid>
             <itunes:duration>01:33:55</itunes:duration>
@@ -244,6 +277,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 29 Jan 2019 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Mariel Salcedo, Barbara Dunkelman, and Burnie Burns as they discuss nutmeg heroin, French trailer parks, Glass, and Mariel crying on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_529.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">529</guid>
             <itunes:duration>01:32:22</itunes:duration>
@@ -255,6 +289,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 22 Jan 2019 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Geoff Ramsey, and Burnie Burns as they discuss Cheez Balls and black coffee, getting old, a bunch of ways to die, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_528.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">528</guid>
             <itunes:duration>01:44:59</itunes:duration>
@@ -266,6 +301,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 15 Jan 2019 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Chad James, Barbara Dunkelman, and Burnie Burns as they discuss toilet paper, customer service, sunburns, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_527.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">527</guid>
             <itunes:duration>01:30:24</itunes:duration>
@@ -277,6 +313,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 08 Jan 2019 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Barbara Dunkelman, and Burnie Burns as they discuss Tik Tok, CES 2019, the government shutdown, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_526.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">526</guid>
             <itunes:duration>01:35:08</itunes:duration>
@@ -288,6 +325,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 01 Jan 2019 21:25:00 +0000</pubDate>
             <description>Join Gus Sorola and Gavin Free and even Jack Patillo, along with other special guests as they wrap up 2018 and look ahead to 2019...and Gus has another lost package story.</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_525.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">525</guid>
             <itunes:duration>01:27:31</itunes:duration>
@@ -299,6 +337,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 25 Dec 2018 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Chris Demarais, Jon Risinger, and Barbara Dunkelman as they discuss Santa Claus, falling down, favorite movies of the year, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_524.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">524</guid>
             <itunes:duration>01:33:41</itunes:duration>
@@ -310,6 +349,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 18 Dec 2018 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Blaine Gibson, Barbara Dunkelman, and Burnie Burns as they discuss the Licki cat brush, the RT holiday party, best movies of the year, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_523.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">523</guid>
             <itunes:duration>01:36:34</itunes:duration>
@@ -321,6 +361,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 11 Dec 2018 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Barbara Dunkelman, and Burnie Burns as they discuss ten years of the podcast, bets, YouTube Rewind, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_522.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">522</guid>
             <itunes:duration>01:35:10</itunes:duration>
@@ -332,6 +373,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 04 Dec 2018 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Barbara Dunkelman, Burnie Burns, and special guest Grace Helbig as they discuss cats, hang gliding gone wrong, Tumblr banning adult content, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_521.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">521</guid>
             <itunes:duration>01:33:55</itunes:duration>
@@ -343,6 +385,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 27 Nov 2018 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Geoff Ramsey, and Burnie Burns as they discuss the post office, coming up with names and titles, ghosts, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_520.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">520</guid>
             <itunes:duration>01:30:27</itunes:duration>
@@ -354,6 +397,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 20 Nov 2018 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Chris Demarais, and Burnie Burns as they discuss the Gus calendar, the Star Wars movies, circumcision, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_519.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">519</guid>
             <itunes:duration>01:34:45</itunes:duration>
@@ -365,6 +409,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 13 Nov 2018 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Gavin Free, Blaine Gibson, Burnie Burns, and special guest Sally Le Page as they discuss Extra Life aftermath, plane stories, cats, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_518.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">518</guid>
             <itunes:duration>01:39:31</itunes:duration>
@@ -376,6 +421,7 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 06 Nov 2018 20:00:00 +0000</pubDate>
             <description>Join Gus Sorola, Becca Frasier, Barbara Dunkelman, and Jon Risinger as they discuss voting, documentaries, mall Santas, and more on this week's RT Podcast!</description>
+            <content>my content here</content>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/roosterteethpodcast/Rooster_Teeth_Podcast_517.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">517</guid>
             <itunes:duration>01:32:43</itunes:duration>
@@ -5997,6 +6043,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Wed, 29 Aug 2012 15:18:32 +0000</pubDate>
             <description>RT farts on Gavin</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/rtpodcast/Rooster_Teeth_Podcast_181.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">181</guid>
             <itunes:duration>00:00:00</itunes:duration>
@@ -6008,6 +6056,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Wed, 05 Sep 2012 22:38:48 +0000</pubDate>
             <description>RT does not know emotion.</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/rtpodcast/Rooster_Teeth_Podcast_182.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">182</guid>
             <itunes:duration>00:00:00</itunes:duration>
@@ -6019,6 +6069,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Wed, 12 Sep 2012 15:59:03 +0000</pubDate>
             <description>RT sits down with Hannah Hart and Shira Lazar!</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/rtpodcast/Rooster_Teeth_Podcast_183.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">183</guid>
             <itunes:duration>00:00:00</itunes:duration>
@@ -6030,6 +6082,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Wed, 19 Sep 2012 15:44:51 +0000</pubDate>
             <description>RT does not wear seat belts in taxis.</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/rtpodcast/Rooster_Teeth_Podcast_184.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">184</guid>
             <itunes:duration>00:00:00</itunes:duration>
@@ -6041,6 +6095,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 19 Nov 2013 22:55:27 +0000</pubDate>
             <description>RT Discusses Wild Videos</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/rtpodcast/Rooster_Teeth_Podcast_245.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">245</guid>
             <itunes:duration>01:35:28</itunes:duration>
@@ -6052,6 +6108,8 @@
             <itunes:author>Rooster Teeth Podcast</itunes:author>
             <pubDate>Tue, 30 Dec 2014 19:54:50 +0000</pubDate>
             <description>RT Discusses ASMR Triggers</description>
+            <content>my content here</content>
+            <copyright>my copyright here</copyright>
             <enclosure url="http://www.podtrac.com/pts/redirect.mp3/traffic.libsyn.com/rtpodcast/Rooster_Teeth_Podcast_304.mp3" type="audio/mpeg" />
             <guid isPermaLink="false">304</guid>
             <itunes:duration>01:40:02</itunes:duration>

server/routes.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"net/http"
 	"regexp"
+	"strconv"
 	"time"
 )
 
@@ -50,6 +51,10 @@ func (s *Server) tag(w http.ResponseWriter, r *http.Request) {
 		s.notFound(w, r)
 		return
 	}
+	limit, err := strconv.Atoi(r.URL.Query().Get("n"))
+	if err != nil {
+		limit = 20
+	}
 	tag := regexp.MustCompile("^.*\\/").ReplaceAllString(r.URL.Path, "")
 	feeds, err := rss.TaggedFeeds(tag)
 	if err != nil {
@@ -58,7 +63,7 @@ func (s *Server) tag(w http.ResponseWriter, r *http.Request) {
 	}
 	items := []*rss.Item{}
 	for _, feed := range feeds {
-		feedItems, err := feed.Items(20)
+		feedItems, err := feed.Items(limit)
 		if err != nil {
 			s.error(w, r, err)
 		}
@@ -76,16 +81,28 @@ func (s *Server) feed(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	var putFeed struct {
-		URL           string   `json:"url"`
-		Interval      string   `json:"refresh"`
-		TitleFilter   string   `json:"items"`
-		ContentFilter string   `json:"content"`
-		Tags          []string `json:"tags"`
+		URL           string            `json:"url"`
+		Interval      string            `json:"refresh"`
+		TitleFilter   string            `json:"items"`
+		Copyright     string            `json:"copyright"`
+		ContentFilter string            `json:"content"`
+		Tags          []string          `json:"tags"`
+		Headers       map[string]string `json:"headers"`
+		URL           string            `json:"url"`
+		Interval      string            `json:"refresh"`
+		TitleFilter   string            `json:"items"`
+		Copyright     string            `json:"copyright"`
+		ContentFilter string            `json:"content"`
+		Tags          []string          `json:"tags"`
 	}
 	if err := json.NewDecoder(r.Body).Decode(&putFeed); err != nil {
 		s.userError(w, r, err)
 		return
 	}
+	if putFeed.Headers == nil {
+		putFeed.Headers = map[string]string{}
+	}
+	putFeedHeaders, _ := json.Marshal(putFeed.Headers)
 	interval, err := time.ParseDuration(putFeed.Interval)
 	if err != nil {
 		s.userError(w, r, err)
@@ -97,6 +114,8 @@ func (s *Server) feed(w http.ResponseWriter, r *http.Request) {
 		TitleFilter:   putFeed.TitleFilter,
 		ContentFilter: putFeed.ContentFilter,
 		Tags:          putFeed.Tags,
+		Copyright:     putFeed.Copyright,
+		Headers:       putFeedHeaders,
 	}
 	if err := rss.SubmitFeed(f); err != nil {
 		s.error(w, r, err)

testdata/add

@@ -0,0 +1,99 @@
+#! /bin/bash
+
+set -e
+
+remote="${remote:-192.168.0.86}"
+
+feeds='
+https://mangadex.org/rss/follows/ryZGFkVgv9DAPqf45EsTHuQcxmteKBN6                manga                      na                                     na "1h"
+https://siftrss.com/f/WXPo6AKQLX                                                 manga                      na                                     na "1h"
+http://yaoi-blcd.tumblr.com/tagged/19-days/rss                                   manga                      "Old.Xian.update.of..19"               na "4h"
+http://www.shanaproject.com/feeds/secure/user/5310/5MUGNU9B3I/                   anime                      na                                     na "24h"
+http://www.manga.megchan.com/blog/?feed=rss2                                     manga                      na                                     na "12h"
+https://siftrss.com/f/6Q8a0ZK0RmL                                                manga                      na                                     na "12h"
+https://dropoutmanga.wordpress.com/feed/                                         manga                      "(Hatsukoi|Tomo.chan)"                 na "6h"
+http://merakiscans.com/manga-rss/solo-leveling/                                  manga                      na                                     na "8h"
+http://www.mangago.me/r/rsslink/a_man_like_you.xml                               manga                      na                                     na "24h"
+http://www.mangago.me/r/rsslink/unriped_expression.xml                           manga                      na                                     na "24h"
+http://www.mangago.me/r/rsslink/heesu_in_class_2.xml                             manga                      na                                     na "24h"
+http://www.mangago.me/r/rsslink/bj_alex.xml                                      manga                      na                                     na "24h"
+https://heroscans.tumblr.com/rss                                                 manga                      "(Saizu)"                              na "24h"
+http://feeds.pinboard.in/rss/popular/                                            news                       ".*"                                   ".*" "24h"
+https://www.vox.com/rss/index.xml                                                news                       "yeahrightnoway"                       ".*" "24h"
+https://www.techdirt.com/techdirt_rss.xml                                        news                       ".*"                                   ".*" "24h"
+http://hotdiggedydemon.com/                                                      comic                      ".*"                                   ".*"  "24h"
+https://nyaa.si/?page=rss&u=Nauti                                                anime                      "(akegurui.*XX)"                       ".*Anime...English.translated.*" "3h"
+https://tapas.io/rss/series/72832                                                manga                      na                                     na "24h"
+https://tapas.io/rss/series/88570                                                manga                      na                                     na "24h"
+https://nyaa.si/?page=rss&q=kakegurui&c=0_0&f=0&u=Nauti                          anime                      "(akegurui.XX)"                        ".*Anime...English.translated.*" "3h"
+https://nyaa.si/?page=rss&q=horriblesubs+mob+psycho+720&c=0_0&f=0                anime                      ".*"                                   ".*Anime...English.translated.*" "3h"
+https://www.youtube.com/feeds/videos.xml?channel_id=UC9CuvdOVfMPvKCiwdGKL3cQ     youtube","gampegrumps      ".*"                                   ".*" "6h"
+https://www.youtube.com/feeds/videos.xml?channel_id=UCbGO1tCyjMlzqIT-tyrJNqQ     youtube","kshaway          ".*"                                   ".*" "6h"
+https://www.youtube.com/feeds/videos.xml?channel_id=UC9YLd0REiXxLqQQH_CpJKZQ     youtube","instalok         ".*"                                   ".*" "6h"
+https://www.youtube.com/feeds/videos.xml?channel_id=UCsvn_Po0SmunchJYOWpOxMg     youtube","videogamedunkey  ".*"                                   ".*" "6h"
+https://www.youtube.com/feeds/videos.xml?channel_id=UCs-w7E2HZWwXmjt9RTvBB_A      youtube","settled         ".*"                                   ".*" "6h"
+https://roosterteeth.com/show/rt-podcast/feed/mp3                                podcast                    ".*"                                   ".*" "6h"
+http://lizclimo.tumblr.com/rss                                                   comic                      na                                     na "24h"
+http://feeds.feedburner.com/blogspot/sBff                                        cooking                    ".*"                                   ".*" "24h"
+https://jaiminisbox.com/reader/feeds/rss                                         manga                      "olo.*eveling"                         ".*" "6h"
+https://xkcd.com/atom.xml                                                        comic                      na                                     na "24h"
+https://matrix.org/blog/feed                                                     news                       ".*"                                   ".*"  "1h"
+https://nyaa.si/?page=rss&f=0&c=0_0&q=shingeki+no+kyojin+final+season+720p       anime","torrent            ".*"                                   ".*"  "1h"     Anime-ShingekiNoKyojin
+https://workchronicles.com/feed/                                                 comic                      na                                     na    "4h"
+https://nyaa.si/?page=rss&f=0&c=0_0&q=re+zero+2nd+part+erai+raws+720             anime","torrent            ".*"                                   ".*"  "2h"     Anime-ReZero
+https://www.webtoons.com/en/challenge/books-of-adam/rss?title_no=136637          comic                      na                                     na    "12h"
+https://www.webtoons.com/en/romance/the-makeup-remover/rss?title_no=2186         comic                      na                                     na    "12h"
+https://www.webtoons.com/en/romance/truebeauty/rss?title_no=1436                 comic                      na                                     na    "12h"
+https://subsplease.org/rss/?r=720                                                anime","torrent            "^jfkaldsjfkldsjafd;sa"                "<a.*?href=.magnet:.*?</a>"              "2h"     Anime
+https://eztv.ag/ezrss.xml?h=h                                                    torrent                    "^([Rr]ick.[Aa]nd.[Mm]orty)"                 "<a.*?href=.magnet:.*?</a>"  "2h"     RickAndMorty
+https://eztv.ag/ezrss.xml?c=c                                                    momma","torrent            "^NOT(d.Rush.*720.*WEB|Gold.Rush.*S.*E.*720.*WEB)"  "<a.*?href=.magnet:.*?</a>"                                                     "2h"     GoldRush
+https://eztv.ag/ezrss.xml?e=e                                                    momma","torrent            "^(Archer|The.Handmaids.Tale.*720.*WEB|Australian.Ninja.Warrior.*HDTV)"  "<a.*?href=.magnet:.*?</a>"                                "2h"
+https://eztv.ag/ezrss.xml?b=b                                                    momma","torrent            "^(Survivor.S.*720p|[Ss]urvivor.*[AaSs][UuAa])"  "<a.*?href=.magnet:.*?</a>"                                                         "2h"     Survivor
+https://eztv.ag/ezrss.xml?a=a                                                    momma","torrent            "^abc123(Wheel.[oO]f.Fortune.[S2]|Jeopardy.20|[tT]he.[pP]rice.[iI]s.[rR]ight|[lL]et.*[mM]ake.[aA].[dD]eal)"  "<a.*?href=.magnet:.*?</a>"  "2h"     GameShows
+https://eztv.ag/ezrss.xml?d=d                                                    momma","torrent            "([Mm]andalorian)"                     "<a.*?href=.magnet:.*?</a>"                                              "2h"     Mandalorian
+https://eztv.ag/ezrss.xml?f=f                                                    momma","torrent            "^([Vv]ikings.S)"                                  "<a.*?href=.magnet:.*?</a>"  "2h"     Vikings
+http://rss.slashdot.org/Slashdot/slashdotMain?format=usm                         news                       ".*"                                   ".*"  "6h"
+https://netflixtechblog.com/feed                                                 news                       ".*"                                   ".*"  "6h"
+https://www.backblaze.com/blog/feed                                              news                       "zzzzzzzzzzzzzz.*"                                   "zzzzzzzzzzzzzzzzzz.*"  "6h"
+https://hackernoon.com/feed                                                      news                       "zzzzzzzzzzzzzz.*"                                   "zzzzzzzzzzzzzzzzzz.*"  "6h"
+https://github.com/jpochyla/psst/commits.atom                                    news                       ".*"                                   ".*"  "6h"
+https://eztv.re/ezrss.xml?b=b                                                    momma","torrent            "^(Survivor.S.*720p|[Ss]urvivor.*[AaSs][UuAa])"  "<a.*?href=.magnet:.*?</a>"                                                         "2h"     Survivor
+https://gitlab.com/famedly/conduit.atom                                          news                       "loljkcancelthisone"                   "loljkcancelpls"  "1288h"
+https://gitlab.com/famedly/conduit/-/tags?format=atom                            news                       ".*"                                   ".*"  "6h"
+https://eztv.re/ezrss.xml?g=g                                                    momma","torrent            ".*awn.*tars.S.*WEB.*"                 "<a.*?href=.magnet:.*?</a>"                                                                   "2h"     PawnStars
+https://ludic.mataroa.blog/rss/                                                  news                       ".*"                                   ".*"  "6h"
+https://eztv.ag/ezrss.xml?d=d                                                    cooking","torrent          "^noooooooooooooooooooooooomore([Mm]aster[cC]hef.*720p.WEB|[Mm]aster[cC]hef.Aus.*)"  "<a.*?href=.nooooooooooooomagnet:.*?</a>"                                              "2h"     MasterChef
+https://eztv.ag/ezrss.xml?g=g                                                    torrent","news             "^nooooooooo([Ll]ast..eek..onight..ith..ohn..liver.*WEB)"                                        "<a.*?href=.nooooooooomagnet:.*?</a>"  "2h"     LastWeekTonight
+https://memorycache.ai/feed.xml                                                  news                       ".*"                                   ".*"  "2h"
+'
+#https://eztv.ag/ezrss.xml                                                        momma","torrent            "^(Wheel.[oO]f.Fortune.[S2]|Jeopardy.20|Gold.Rush.*720.*WEB|Survivor.S.*720p|Gold.Rush.*S.*E.*720.*WEB|Archer|[Mm]aster[cC]hef.US.*720p.WEB|[Mm]aster[cC]hef.Aus.*|[tT]he.[pP]rice.[iI]s.[rR]ight|[lL]et.*[mM]ake.[aA].[dD]eal|The.Handmaids.Tale.*720.*WEB|[Ss]urvivor.*[AaSs][UuAa]|Australian.Ninja.Warrior.*HDTV)"  "<a.*?href=.magnet:.*?</a>" "2h"
+feedsB="$(printf "%s\n" "$feeds" | grep -v '^$' | tail -n 1)"
+feeds="$feedsB"
+
+defaultIF='""'
+defaultCF='"<img.*?/(img)?>"'
+defaultIN='"6h"'
+
+feeds="${feeds// /,,}"
+
+for feed in $feeds; do
+   feed="${feed//,,/ }"
+   link="$(echo "$feed" | awk '{print $1}')"
+   tag="$(echo "$feed" | awk '{print $2}')"
+   if="$(echo "$feed" | awk '{print $3}')"
+   cf="$(echo "$feed" | awk '{print $4}')"
+   in="$(echo "$feed" | awk '{print $5}')"
+   copyright="$(echo "$feed" | awk '{print $6}')"
+   if [ "$if" == "na" ]; then
+      if="$defaultIF"
+   fi
+   if [ "$cf" == "na" ]; then
+      cf="$defaultCF"
+   fi
+   if [ "$in" == "na" ]; then
+      in="$defaultIN"
+   fi
+   cmd="curl ${PROXY:-} -i ${remote}:33419/api/feed -X PUT --data '{\"url\":\"$link\", \"refresh\":$in, \"items\":$if, \"content\":$cf, \"tags\":[\"$tag\"], \"copyright\":\"$copyright\", \"headers\": {\"Cookie\": \"cf_chl_rc_ni=1; cf_clearance=fd4ce60270476e0a520dcde90c81d3d478fcecf9-1623375837-0-150\", \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0\"}}'"
+   echo "$cmd" >&2
+   eval "$cmd"
+done

testdata/do

@@ -0,0 +1,7 @@
+*  implement torret+podcast handlers
+*  fetch <img> and embed?
+
+x  render tags -> feeds for server
+x  load feed jobs on startup and queue
+x  push new jobs/queue items from server
+x  implement endpoints for server

vendor/vendor.json

@@ -1,6 +0,0 @@
-{
-	"comment": "",
-	"ignore": "test",
-	"package": [],
-	"rootPath": "local/rssmon3"
-}