package auction import ( "context" "encoding/json" "errors" "local/sandbox/selenium/copart/copart/browser" "local/sandbox/selenium/copart/copart/config" "local/storage" "log" "strconv" "strings" "time" ) type Auction struct { browser *browser.Browser cars chan *Car ctx context.Context can context.CancelFunc routines chan struct{} } func New(b *browser.Browser, url string) (*Auction, error) { if err := b.Get(url); err != nil { b.Close() return nil, err } ctx, can := context.WithTimeout(context.Background(), time.Second*10) defer can() var elem WebElement for err := ctx.Err(); err == nil; err = ctx.Err() { elem, err = b.Driver.FindElement("xpath", "//iframe") if err == nil { break } time.Sleep(time.Second) } if err := ctx.Err(); err != nil { b.Close() return nil, err } iframeURL, err := elem.GetAttribute("src") if err != nil { b.Close() return nil, err } else if iframeURL == "" { b.Close() return nil, errors.New("auction iframe has no src") } if err := b.Get(iframeURL); err != nil { b.Close() return nil, err } ctx, can = context.WithCancel(context.Background()) return &Auction{ browser: b, cars: make(chan *Car), ctx: ctx, can: can, routines: make(chan struct{}, 5), }, nil } func (a *Auction) Start() error { go a.parseCars() a.routines <- struct{}{} go a.saveCars() a.routines <- struct{}{} return nil } func (a *Auction) parseCars() error { log.Printf("[parseCars] starting") defer func() { if err := recover(); err != nil { log.Println("[parseCars] recover:", err) } log.Printf("[parseCars] stopping") close(a.cars) a.can() a.routines <- struct{}{} }() for err := a.ctx.Err(); err == nil; err = a.ctx.Err() { c, err := a.nextCar() if err != nil { panic(err) } log.Printf("[parseCars] found car %v", c.String()) if !strings.Contains(c.Bid, "$") { continue } if b, _ := c.MarshalJSON(); strings.Contains(strings.ToLower(string(b)), ", hi") { continue } else if strings.Contains(strings.ToLower(string(b)), "guam") { continue } else if len(c.Title) < 4 { continue } else if v, err := strconv.Atoi(c.Title[:4]); err != nil { continue } else if v+5 < time.Now().Year() { continue } select { case a.cars <- c: case <-a.ctx.Done(): panic(a.ctx.Err()) } } return a.ctx.Err() } func (a *Auction) nextCar() (*Car, error) { c := NewCar() if err := c.Parse(a.browser.Driver); err != nil { if !strings.Contains(err.Error(), "stale element") { return nil, err } else { return a.nextCar() } } for err := a.ctx.Err(); err == nil; err = a.ctx.Err() { time.Sleep(time.Second * 2) d := NewCar() if err := d.Parse(a.browser.Driver); err != nil { if !strings.Contains(err.Error(), "stale element") { return nil, err } } if !c.Equals(d) { return c, nil } if d.Bid != "" { c = d } } return nil, a.ctx.Err() } func (a *Auction) saveCars() error { log.Printf("[saveCars] starting") defer func() { if err := recover(); err != nil { log.Println("[saveCars] recover:", err) } log.Printf("[saveCars] stopping") a.can() a.routines <- struct{}{} }() for err := a.ctx.Err(); err == nil; err = a.ctx.Err() { select { case <-a.ctx.Done(): panic(a.ctx.Err()) case c := <-a.cars: go func(d *Car) { if d == nil { return } log.Printf("[saveCars] saving %v", d.String()) if err := a.saveCar(d); err != nil { log.Printf("[saveCars] err: %v", err) } }(c) } } return a.ctx.Err() } func (a *Auction) saveCar(c *Car) error { db := config.Values().DB b, err := c.Encode() if err != nil { return err } if err := db.Set(c.String(), b); err != nil { return err } var list []string b, err = db.Get("LIST") if err == storage.ErrNotFound { b = []byte("[]") } else if err != nil { return err } if err := json.Unmarshal(b, &list); err != nil { return err } list = append([]string{c.String()}, list...) if b, err := json.Marshal(list); err != nil { return err } else if err := db.Set("LIST", b); err != nil { return err } return nil } func (a *Auction) Stop() error { l := len(a.routines) a.can() ctx, can := context.WithTimeout(context.Background(), time.Second*5) defer can() for i := 0; i < 2*l; i++ { select { case <-a.routines: case <-ctx.Done(): } } if len(a.routines) > 0 { return errors.New("not all routines exited") } return nil }