package broker import ( "bytes" "compress/gzip" "errors" "fmt" "io" "io/ioutil" "local/storage" "local/truckstop/config" "local/truckstop/logtr" "net/http" "sort" "strconv" "strings" "time" "github.com/PuerkitoBio/goquery" ) type FastExact struct { doer interface { doRequest(*http.Request) (*http.Response, error) } } func NewFastExact() FastExact { fe := FastExact{} fe.doer = fe if config.Get().Brokers.FastExact.Mock { fe = fe.WithMock() } return fe } func (fe FastExact) WithMock() FastExact { fe.doer = mockFastExactDoer{} return fe } func (fe FastExact) SearchZips(zips []string) ([]Job, error) { jobs, err := fe.searchZips(zips) if err == ErrNoAuth { if err := fe.login(); err != nil { return nil, err } jobs, err = fe.searchZips(zips) } return jobs, err } func (fe FastExact) SearchStates(states []config.State) ([]Job, error) { jobs, err := fe.searchStates(states) if err == ErrNoAuth { if err := fe.login(); err != nil { return nil, err } jobs, err = fe.searchStates(states) } return jobs, err } func (fe FastExact) login() error { conf := config.Get() return fe._login(conf.Brokers.FastExact.Username, conf.Brokers.FastExact.Password, conf.DB()) } func (fe FastExact) _login(username, password string, db storage.DB) error { req, err := http.NewRequest( http.MethodPost, `https://www.fastexact.com/secure/index.php?page=userLogin`, strings.NewReader(fmt.Sprintf( `user_name=%s&user_password=%s&buttonSubmit=Login`, username, password, )), ) if err != nil { return err } fe.setHeaders(req) db.Set("cookies_"+req.URL.Host, nil) resp, err := fe.doer.doRequest(req) if err != nil { return err } b, _ := ioutil.ReadAll(resp.Body) defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("bad status logging into fast exact: %d: %s", resp.StatusCode, b) } return nil } func (fe FastExact) setHeaders(req *http.Request) { req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.5") req.Header.Set("Accept-Encoding", "gzip, deflate, br") req.Header.Set("Content-Type", "application/x-www-form-urlencoded") } func (fe FastExact) searchZips(zips []string) ([]Job, error) { var jobs []Job for _, zip := range zips { subjobs, err := fe.searchOneZip(zip) if err != nil { return nil, err } jobs = append(jobs, subjobs...) } return fe.dedupeJobs(jobs), nil } func (fe FastExact) searchStates(states []config.State) ([]Job, error) { var jobs []Job for _, state := range states { subjobs, err := fe.searchOneState(state) if err != nil { return nil, err } jobs = append(jobs, subjobs...) } return fe.dedupeJobs(jobs), nil } func (fe FastExact) dedupeJobs(jobs []Job) []Job { sort.Slice(jobs, func(i, j int) bool { return jobs[i].UID() < jobs[j].UID() }) dedupeJobs := map[string]Job{} for _, job := range jobs { dedupeJobs[strings.ReplaceAll(job.UID(), job.ID, "")] = job } result := []Job{} for _, job := range dedupeJobs { result = append(result, job) } return result } func (fe FastExact) searchOneZip(zip string) ([]Job, error) { req, err := fe.newRequest(zip) if err != nil { return nil, err } resp, err := fe.doer.doRequest(req) logtr.Verbosef("req: %+v => resp: %+v", req, resp) if err != nil { return nil, err } defer resp.Body.Close() return fe.parse(resp) } func (fe FastExact) searchOneState(state config.State) ([]Job, error) { req, err := fe.newRequestWithState(state) if err != nil { return nil, err } resp, err := fe.doer.doRequest(req) logtr.Verbosef("req: %+v => resp: %+v", req, resp) if err != nil { return nil, err } defer resp.Body.Close() return fe.parse(resp) } func (fe FastExact) newRequestWithState(state config.State) (*http.Request, error) { zip, ok := config.States[state] if !ok { return nil, fmt.Errorf("no configured zip for %s", state) } return fe.newRequest(zip) } func (fe FastExact) newRequest(zip string) (*http.Request, error) { req, err := http.NewRequest( http.MethodGet, "https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance="+strconv.Itoa(config.Get().Brokers.RadiusMiles)+"&st_loc_zip=8", nil, ) if err != nil { return nil, err } fe.setHeaders(req) return req, nil } func (fe FastExact) doRequest(req *http.Request) (*http.Response, error) { resp, err := do(req) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { b, _ := ioutil.ReadAll(resp.Body) logtr.Errorf("fastExact bad status: %d: %s", resp.StatusCode, b) if resp.StatusCode > 400 && resp.StatusCode < 404 { return nil, ErrNoAuth } return nil, fmt.Errorf("bad status from FastExact: %d: %s", resp.StatusCode, b) } return resp, nil } func (fe FastExact) parse(resp *http.Response) ([]Job, error) { b, _ := ioutil.ReadAll(resp.Body) if !bytes.HasPrefix(bytes.TrimSpace(b), []byte("<")) { gzip, err := gzip.NewReader(bytes.NewReader(b)) if err != nil { return nil, err } b, _ = ioutil.ReadAll(gzip) } if bytes.Contains(b, []byte(``)) { return nil, ErrNoAuth } logtr.Verbosef("fe.parse %s", b) resp.Body = io.NopCloser(bytes.NewReader(b)) doc, err := goquery.NewDocumentFromReader(resp.Body) if err != nil { return nil, err } result := make([]Job, 0) doc.Find("#list table tr").Each(func(i int, s *goquery.Selection) { columns := []string{} s.Find("td").Each(func(i int, s *goquery.Selection) { if s.Nodes[0].LastChild != nil && len(s.Nodes[0].LastChild.Attr) > 0 { attrs := s.Nodes[0].LastChild.Attr columns = append(columns, attrs[len(attrs)-1].Val) } else { columns = append(columns, s.Text()) } }) if len(columns) < 9 { return } job := Job{ ID: columns[0], URI: columns[8], } job.Pickup.Date, _ = time.ParseInLocation("02-Jan-2006 15:04:05", columns[7], time.Local) job.Pickup.City = strings.Title(strings.ToLower(strings.Split(columns[1], ",")[0])) if strings.Contains(columns[1], ",") { job.Pickup.State = strings.Title(strings.Split(strings.Split(columns[1], ",")[1], " ")[1]) } job.Dropoff.Date = job.Pickup.Date job.Dropoff.City = strings.Title(strings.ToLower(strings.Split(columns[2], ",")[0])) if strings.Contains(columns[2], ",") { job.Dropoff.State = strings.Title(strings.Split(strings.Split(columns[2], ",")[1], " ")[1]) } job.Miles, _ = strconv.Atoi(columns[3]) if strings.Contains(columns[4], "/") { weight, _ := strconv.ParseFloat(strings.TrimSpace(strings.Split(columns[4], "/")[1]), 32) job.Weight = int(weight) } job.Meta = fmt.Sprintf(`dimensions:%s`, strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(columns[5], " ", ""), "\n", ""))) logtr.Verbosef("fe.parse %+v => %+v", columns, job) result = append(result, job) }) return result, nil } type mockFastExactDoer struct{} func (mock mockFastExactDoer) doRequest(req *http.Request) (*http.Response, error) { if req.URL.Path != "/secure/index.php" { return nil, errors.New("bad path") } switch req.URL.Query().Get("page") { case "userLogin": if b, _ := ioutil.ReadAll(req.Body); !bytes.Equal(b, []byte(`user_name=u&user_password=p&buttonSubmit=Login`)) { return nil, errors.New("bad req body") } return &http.Response{ Status: http.StatusText(http.StatusOK), StatusCode: http.StatusOK, Header: http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromLogin; path=/"}}, Body: io.NopCloser(bytes.NewReader([]byte{})), }, nil case "ajaxListJobs": if req.URL.Query().Get("action") != "ajax" { return nil, errors.New("bad query: action should be ajax") } if req.URL.Query().Get("records_per_page") != "50" { return nil, errors.New("bad query: records_per_page should be 50") } if req.URL.Query().Get("distance") != strconv.Itoa(config.Get().Brokers.RadiusMiles) { return nil, errors.New("bad query: distance should be as configured") } if req.URL.Query().Get("zipcode") == "" { return nil, errors.New("bad query: zip code empty") } b, err := ioutil.ReadFile("./testdata/fastexact_search.xml") if err != nil { b, err = ioutil.ReadFile("./broker/testdata/fastexact_search.xml") } if err != nil { return nil, err } return &http.Response{ Status: http.StatusText(http.StatusOK), StatusCode: http.StatusOK, Header: http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromSearch; path=/"}}, Body: io.NopCloser(bytes.NewReader(b)), }, nil } return nil, errors.New("bad query") }