package auction import ( "context" "local/sandbox/selenium/copart/copart/browser" "local/sandbox/selenium/copart/copart/config" "local/storage" "log" "os" "strings" "syscall" "time" "golang.org/x/time/rate" ) type Today struct { today *browser.Browser Auctions []*Auction ctx context.Context can context.CancelFunc routines chan struct{} sigc chan os.Signal } func NewToday(sigc chan os.Signal) (*Today, error) { today, err := today() if err != nil { today.Close() return nil, err } if _, err := config.Values().DB.Get("LIST"); err == storage.ErrNotFound { if err := config.Values().DB.Set("LIST", []byte("[]")); err != nil { today.Close() return nil, err } } else if err != nil { today.Close() return nil, err } deadline := time.Now() deadline = deadline.Add(time.Hour * 24) deadline = deadline.Add(time.Duration(-1*time.Now().Hour()) * time.Hour) ctx, can := context.WithDeadline(context.Background(), deadline) return &Today{ today: today, Auctions: make([]*Auction, 0), ctx: ctx, can: can, routines: make(chan struct{}, 10), sigc: sigc, }, nil } func today() (*browser.Browser, error) { today, err := browser.New() if err != nil { return today, err } if err := today.Get("https://www.copart.com/todaysAuction/"); err != nil { return today, err } return today, nil } func (t *Today) Start() error { ch := make(chan string) go t.watchAuctionList(ch) t.routines <- struct{}{} go t.startAuctions(ch) t.routines <- struct{}{} return t.ctx.Err() } func (t *Today) watchAuctionList(ch chan string) { log.Printf("[watchAuctionList] starting") defer func() { if err := recover(); err != nil { log.Println("[watchAuctionList] recover:", err) } log.Printf("[watchAuctionList] stopping") t.routines <- struct{}{} t.Stop() }() found := make(map[string]struct{}) retryCD := 3 for err := t.ctx.Err(); err == nil; err = t.ctx.Err() { elems, err := t.today.Driver.FindElements("link text", "Join Auction") if err != nil { panic(err) } log.Printf("[watchAuctionList] found %v", len(elems)) for _, elem := range elems { url, err := elem.GetAttribute("href") if err != nil { panic(err) } key := time.Now().Format("2006-01-02") + url if _, ok := found[key]; ok { log.Printf("[watchAuctionList] skipping %v", url) continue } found[key] = struct{}{} log.Printf("[watchAuctionList] sending %v", len(url)) ch <- url } log.Printf("[watchAuctionList] blocking 5 minutes") select { case <-t.ctx.Done(): panic(t.ctx.Err()) case <-time.After(time.Minute * 5): retryCD -= 1 if retryCD == 0 { retryCD = 3 for { if t.ctx.Err() != nil { panic(t.ctx.Err()) } err := t.today.Driver.Refresh() if err == nil { break } if strings.Contains(err.Error(), "timeout") { select { case <-time.After(time.Second * 15): case <-t.ctx.Done(): panic(t.ctx.Err()) } } } } } } } func (t *Today) startAuctions(ch chan string) { log.Printf("[startAuctions] starting") defer func() { if err := recover(); err != nil { log.Println("[startAuctions] recover:", err) } log.Printf("[startAuctions] stopping") t.routines <- struct{}{} t.Stop() }() limiter := rate.NewLimiter(rate.Every(time.Second*15), 1) for { select { case <-t.ctx.Done(): panic(t.ctx.Err()) case url := <-ch: if err := limiter.Wait(t.ctx); err != nil { panic(err) } log.Printf("[startAuctions] spawning %v", url) b, err := browser.New() if err != nil { time.Sleep(time.Second * 15) if err := limiter.Wait(t.ctx); err != nil { panic(err) } b, err = browser.New() if err != nil { panic(err) } } a, err := New(b, url) if a != nil { t.Auctions = append(t.Auctions, a) } if err != nil { log.Println("[startAuctions] failed to New:", err) } else if err := a.Start(); err != nil { log.Println("[startAuctions] failed to New().Start:", err) } else { log.Printf("[startAuctions] spawned %v", url) } } } } func (t *Today) Stop() error { last := make(chan error) if t == nil { return nil } l := len(t.routines) t.can() ctx, can := context.WithTimeout(context.Background(), time.Second*10) defer can() for i := 0; i < l*2; i++ { select { case <-t.routines: case <-ctx.Done(): select { case last <- ctx.Err(): default: } } } for _, a := range t.Auctions { go func() { if err := a.Stop(); err != nil { select { case last <- err: case <-time.After(time.Second * 5): } } }() } if err := t.today.Driver.Close(); err != nil { select { case last <- err: default: } } var err error select { case err = <-last: case <-time.After(time.Second * 5): } select { case t.sigc <- syscall.SIGHUP: default: } return err }