rssmon3/copart/auction/today.go

package auction

import (
	"context"
	"local/sandbox/selenium/copart/copart/browser"
	"local/sandbox/selenium/copart/copart/config"
	"local/storage"
	"log"
	"os"
	"strings"
	"syscall"
	"time"

	"golang.org/x/time/rate"
)

type Today struct {
	today    *browser.Browser
	Auctions []*Auction
	ctx      context.Context
	can      context.CancelFunc
	routines chan struct{}
	sigc     chan os.Signal
}

func NewToday(sigc chan os.Signal) (*Today, error) {
	today, err := today()
	if err != nil {
		today.Close()
		return nil, err
	}

	if _, err := config.Values().DB.Get("LIST"); err == storage.ErrNotFound {
		if err := config.Values().DB.Set("LIST", []byte("[]")); err != nil {
			today.Close()
			return nil, err
		}
	} else if err != nil {
		today.Close()
		return nil, err
	}

	deadline := time.Now()
	deadline = deadline.Add(time.Hour * 24)
	deadline = deadline.Add(time.Duration(-1*time.Now().Hour()) * time.Hour)
	ctx, can := context.WithDeadline(context.Background(), deadline)
	return &Today{
		today:    today,
		Auctions: make([]*Auction, 0),
		ctx:      ctx,
		can:      can,
		routines: make(chan struct{}, 10),
		sigc:     sigc,
	}, nil
}

func today() (*browser.Browser, error) {
	today, err := browser.New()
	if err != nil {
		return today, err
	}
	if err := today.Get("https://www.copart.com/todaysAuction/"); err != nil {
		return today, err
	}
	return today, nil
}

func (t *Today) Start() error {
	ch := make(chan string)
	go t.watchAuctionList(ch)
	t.routines <- struct{}{}
	go t.startAuctions(ch)
	t.routines <- struct{}{}
	return t.ctx.Err()
}

func (t *Today) watchAuctionList(ch chan string) {
	log.Printf("[watchAuctionList] starting")
	defer func() {
		if err := recover(); err != nil {
			log.Println("[watchAuctionList] recover:", err)
		}
		log.Printf("[watchAuctionList] stopping")
		t.routines <- struct{}{}
		t.Stop()
	}()
	found := make(map[string]struct{})
	retryCD := 3
	for err := t.ctx.Err(); err == nil; err = t.ctx.Err() {
		elems, err := t.today.Driver.FindElements("link text", "Join Auction")
		if err != nil {
			panic(err)
		}
		log.Printf("[watchAuctionList] found %v", len(elems))

		for _, elem := range elems {
			url, err := elem.GetAttribute("href")
			if err != nil {
				panic(err)
			}
			key := time.Now().Format("2006-01-02") + url
			if _, ok := found[key]; ok {
				log.Printf("[watchAuctionList] skipping %v", url)
				continue
			}
			found[key] = struct{}{}
			log.Printf("[watchAuctionList] sending %v", len(url))
			ch <- url
		}

		log.Printf("[watchAuctionList] blocking 5 minutes")
		select {
		case <-t.ctx.Done():
			panic(t.ctx.Err())
		case <-time.After(time.Minute * 5):
			retryCD -= 1
			if retryCD == 0 {
				retryCD = 3
				for {
					if t.ctx.Err() != nil {
						panic(t.ctx.Err())
					}
					err := t.today.Driver.Refresh()
					if err == nil {
						break
					}
					if strings.Contains(err.Error(), "timeout") {
						select {
						case <-time.After(time.Second * 15):
						case <-t.ctx.Done():
							panic(t.ctx.Err())
						}
					}
				}
			}
		}
	}
}

func (t *Today) startAuctions(ch chan string) {
	log.Printf("[startAuctions] starting")
	defer func() {
		if err := recover(); err != nil {
			log.Println("[startAuctions] recover:", err)
		}
		log.Printf("[startAuctions] stopping")
		t.routines <- struct{}{}
		t.Stop()
	}()
	limiter := rate.NewLimiter(rate.Every(time.Second*15), 1)
	for {
		select {
		case <-t.ctx.Done():
			panic(t.ctx.Err())
		case url := <-ch:
			if err := limiter.Wait(t.ctx); err != nil {
				panic(err)
			}
			log.Printf("[startAuctions] spawning %v", url)
			b, err := browser.New()
			if err != nil {
				time.Sleep(time.Second * 15)
				if err := limiter.Wait(t.ctx); err != nil {
					panic(err)
				}
				b, err = browser.New()
				if err != nil {
					panic(err)
				}
			}
			a, err := New(b, url)
			if a != nil {
				t.Auctions = append(t.Auctions, a)
			}
			if err != nil {
				log.Println("[startAuctions] failed to New:", err)
			} else if err := a.Start(); err != nil {
				log.Println("[startAuctions] failed to New().Start:", err)
			} else {
				log.Printf("[startAuctions] spawned %v", url)
			}
		}
	}
}

func (t *Today) Stop() error {
	last := make(chan error)
	if t == nil {
		return nil
	}
	l := len(t.routines)
	t.can()
	ctx, can := context.WithTimeout(context.Background(), time.Second*10)
	defer can()
	for i := 0; i < l*2; i++ {
		select {
		case <-t.routines:
		case <-ctx.Done():
			select {
			case last <- ctx.Err():
			default:
			}
		}
	}
	for _, a := range t.Auctions {
		go func() {
			if err := a.Stop(); err != nil {
				select {
				case last <- err:
				case <-time.After(time.Second * 5):
				}
			}
		}()
	}
	if err := t.today.Driver.Close(); err != nil {
		select {
		case last <- err:
		default:
		}
	}
	var err error
	select {
	case err = <-last:
	case <-time.After(time.Second * 5):
	}
	select {
	case t.sigc <- syscall.SIGHUP:
	default:
	}
	return err
}