render
/
spoc-bot-vr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

oops drop bad log

main
Bel LaPointe 2024-04-18 15:05:25 -06:00
parent f8861a73b5
commit 894536d209
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
slackscrape.go
View File

@ -39,7 +39,6 @@ func NewSlackScrapePipeline(ctx context.Context, cfg Config) (Pipeline, error) {
func newSlackScrapeProcess(cfg Config) processFunc { func newSlackScrapeProcess(cfg Config) processFunc {
limiter := rate.NewLimiter(0.5, 1) limiter := rate.NewLimiter(0.5, 1)
return func(ctx context.Context, jobb []byte) ([]byte, error) { return func(ctx context.Context, jobb []byte) ([]byte, error) {
log.Printf("newSlackScrapeProcess(%s)", jobb)
if err := limiter.Wait(ctx); err != nil { if err := limiter.Wait(ctx); err != nil {
return nil, err return nil, err
} }
@ -128,7 +127,7 @@ func newSlackScrapeProcess(cfg Config) processFunc {
if err := cfg.slackScrapePipeline.reader.Enqueue(ctx, b); err != nil { if err := cfg.slackScrapePipeline.reader.Enqueue(ctx, b); err != nil {
return nil, err return nil, err
} }
log.Printf("enqueue thread scrape for %s/%s", job.Channel, peek.ThreadTS) log.Printf("fanout thread scrape for %s/%s", job.Channel, peek.ThreadTS)
} }
} }
} }
@ -139,7 +138,7 @@ func newSlackScrapeProcess(cfg Config) processFunc {
if err := cfg.slackScrapePipeline.reader.Enqueue(ctx, b); err != nil { if err := cfg.slackScrapePipeline.reader.Enqueue(ctx, b); err != nil {
return nil, err return nil, err
} }
log.Printf("enqueue page scrape for %s up to %v", job.Channel, clone.Latest) log.Printf("fanout page scrape for %s up to %v", job.Channel, clone.Latest)
} }
log.Printf("scraped %v from %s", len(page.Messages), url) log.Printf("scraped %v from %s", len(page.Messages), url)