can fetch zenshuu with tag

main
Bel LaPointe 2025-04-27 12:56:33 -06:00
parent ba429f6028
commit f95563e849
5 changed files with 26 additions and 9 deletions

View File

@ -30,6 +30,7 @@ type (
URL string URL string
Cron string Cron string
Pattern string Pattern string
Tag string
} }
Execution struct { Execution struct {
@ -97,6 +98,7 @@ func Get(ctx context.Context, id string) (Feed, error) {
versions.url AS "Version.URL", versions.url AS "Version.URL",
versions.cron AS "Version.Cron", versions.cron AS "Version.Cron",
versions.pattern AS "Version.Pattern", versions.pattern AS "Version.Pattern",
versions.tag AS "Version.Tag",
( (
SELECT executed_at SELECT executed_at
FROM "feed.executions" FROM "feed.executions"
@ -134,7 +136,8 @@ func oldGet(ctx context.Context, id string) (Feed, error) {
"feed.current_versions".versions_created_at AS Created, "feed.current_versions".versions_created_at AS Created,
"feed.current_versions".url AS URL, "feed.current_versions".url AS URL,
"feed.current_versions".cron AS Cron, "feed.current_versions".cron AS Cron,
"feed.current_versions".pattern AS Pattern "feed.current_versions".pattern AS Pattern,
"feed.current_versions".tag AS Tag
FROM FROM
"feed.current_versions" "feed.current_versions"
JOIN JOIN
@ -187,7 +190,7 @@ func (feed Feed) Executed(ctx context.Context) error {
) )
} }
func Insert(ctx context.Context, url, cron, pattern string) (string, error) { func Insert(ctx context.Context, url, cron, pattern, tag string) (string, error) {
if err := initDB(ctx); err != nil { if err := initDB(ctx); err != nil {
return "", err return "", err
} }
@ -206,16 +209,17 @@ func Insert(ctx context.Context, url, cron, pattern string) (string, error) {
created_at, created_at,
url, url,
cron, cron,
pattern pattern,
) VALUES ($4, $5, $6, $7, $8); tag
) VALUES ($4, $5, $6, $7, $8, $9);
COMMIT; COMMIT;
`, `,
id, now, now, id, now, now,
id, now, url, cron, pattern, id, now, url, cron, pattern, tag,
) )
} }
func (feed Feed) Update(ctx context.Context, url, cron, pattern *string) error { func (feed Feed) Update(ctx context.Context, url, cron, pattern, tag *string) error {
return io.EOF return io.EOF
} }
@ -276,6 +280,7 @@ func initDB(ctx context.Context) error {
`ALTER TABLE "feed.versions" ADD COLUMN url TEXT NOT NULL`, `ALTER TABLE "feed.versions" ADD COLUMN url TEXT NOT NULL`,
`ALTER TABLE "feed.versions" ADD COLUMN cron TEXT NOT NULL DEFAULT '0 0 * * *'`, `ALTER TABLE "feed.versions" ADD COLUMN cron TEXT NOT NULL DEFAULT '0 0 * * *'`,
`ALTER TABLE "feed.versions" ADD COLUMN pattern TEXT NOT NULL DEFAULT ''`, `ALTER TABLE "feed.versions" ADD COLUMN pattern TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE "feed.versions" ADD COLUMN tag TEXT NOT NULL DEFAULT ''`,
`CREATE TABLE "feed.executions" ( `CREATE TABLE "feed.executions" (
entries_id TEXT, entries_id TEXT,

View File

@ -15,7 +15,7 @@ func TestFeeds(t *testing.T) {
t.Run("crud", func(t *testing.T) { t.Run("crud", func(t *testing.T) {
ctx := db.Test(t, ctx) ctx := db.Test(t, ctx)
id, err := feeds.Insert(ctx, "url", "cron", "pattern") id, err := feeds.Insert(ctx, "url", "cron", "pattern", "tag")
if err != nil { if err != nil {
t.Fatal("cannot insert:", err) t.Fatal("cannot insert:", err)
} }
@ -51,6 +51,9 @@ func TestFeeds(t *testing.T) {
if got.Version.Pattern != "pattern" { if got.Version.Pattern != "pattern" {
t.Error("bad version.pattern") t.Error("bad version.pattern")
} }
if got.Version.Tag != "tag" {
t.Error("bad version.tag")
}
if !got.Execution.Executed.IsZero() { if !got.Execution.Executed.IsZero() {
t.Error("execution.executed") t.Error("execution.executed")

View File

@ -64,6 +64,9 @@ func (feed Feed) Fetch(ctx context.Context) (Items, error) {
result := make(Items, 0, len(gfeed.Items)) result := make(Items, 0, len(gfeed.Items))
for _, gitem := range gfeed.Items { for _, gitem := range gfeed.Items {
if gitem.Author == nil {
gitem.Author = &gofeed.Person{}
}
if matches := slices.DeleteFunc(append([]string{ if matches := slices.DeleteFunc(append([]string{
gitem.Title, gitem.Title,
gitem.Description, gitem.Description,
@ -97,6 +100,7 @@ func (feed Feed) Fetch(ctx context.Context) (Items, error) {
Links: links, Links: links,
Preview: preview, Preview: preview,
Body: body, Body: body,
Tag: feed.Version.Tag,
}) })
} }

View File

@ -28,9 +28,10 @@ func TestIntegrationFeedFetchProxy(t *testing.T) {
}, },
Version: feeds.Version{ Version: feeds.Version{
Created: created, Created: created,
URL: "http://rss.slashdot.org/Slashdot/slashdotMain?format=usm", URL: "https://nyaa.si/?f=0&c=0_0&q=Zenshuu.+WEB+ToonsHub+AMZN&page=rss",
Cron: "* * * * *", Cron: "* * * * *",
Pattern: ".*", Pattern: `\.torrent$`,
Tag: `outdir:/data/completed-rss/Zenshuu`,
}, },
Execution: feeds.Execution{ Execution: feeds.Execution{
Executed: created.Add(-2 * time.Second), Executed: created.Add(-2 * time.Second),
@ -44,5 +45,8 @@ func TestIntegrationFeedFetchProxy(t *testing.T) {
} }
for i, item := range items { for i, item := range items {
t.Logf("[%d] %+v", i, item) t.Logf("[%d] %+v", i, item)
if item.Tag != "outdir:/data/completed-rss/Zenshuu" {
t.Errorf("[%d] wrong tag: %s", i, item.Tag)
}
} }
} }

View File

@ -7,4 +7,5 @@ type Item struct {
Links []string Links []string
Preview string Preview string
Body string Body string
Tag string
} }