package broker import ( "bytes" "compress/gzip" "errors" "fmt" "io" "io/ioutil" "local/storage" "local/truckstop/config" "local/truckstop/logtr" "net/http" "strconv" "strings" "time" "github.com/PuerkitoBio/goquery" ) type FastExact struct { doer interface { doRequest(*http.Request) (*http.Response, error) } } func NewFastExact() FastExact { fe := FastExact{} fe.doer = fe if config.Get().Brokers.FastExact.Mock { fe = fe.WithMock() } return fe } func (fe FastExact) WithMock() FastExact { fe.doer = mockFastExactDoer{} return fe } func (fe FastExact) Search(states []config.State) ([]Job, error) { jobs, err := fe.search(states) if err == ErrNoAuth { if err := fe.login(); err != nil { return nil, err } jobs, err = fe.search(states) } return jobs, err } func (fe FastExact) login() error { conf := config.Get() return fe._login(conf.Brokers.FastExact.Username, conf.Brokers.FastExact.Password, conf.DB()) } func (fe FastExact) _login(username, password string, db storage.DB) error { req, err := http.NewRequest( http.MethodPost, `https://www.fastexact.com/secure/index.php?page=userLogin`, strings.NewReader(fmt.Sprintf( `user_name=%s&user_password=%s&buttonSubmit=Login`, username, password, )), ) if err != nil { return err } fe.setHeaders(req) db.Set("cookies_"+req.URL.Host, nil) resp, err := fe.doer.doRequest(req) if err != nil { return err } b, _ := ioutil.ReadAll(resp.Body) resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("bad status logging into fast exact: %d: %s", resp.StatusCode, b) } return nil } func (fe FastExact) setHeaders(req *http.Request) { req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.5") req.Header.Set("Accept-Encoding", "gzip, deflate, br") req.Header.Set("Content-Type", "application/x-www-form-urlencoded") } func (fe FastExact) search(states []config.State) ([]Job, error) { var result []Job for _, state := range states { jobs, err := fe.searchOne(state) if err != nil { return nil, err } result = append(result, jobs...) } return result, nil } func (fe FastExact) searchOne(state config.State) ([]Job, error) { req, err := fe.newRequest(state) if err != nil { return nil, err } resp, err := fe.doer.doRequest(req) logtr.Verbosef("req: %+v => resp: %+v", req, resp) if err != nil { return nil, err } return fe.parse(resp) } func (fe FastExact) newRequest(state config.State) (*http.Request, error) { zip, ok := config.States[state] if !ok { return nil, fmt.Errorf("no configured zip for %s", state) } req, err := http.NewRequest( http.MethodGet, "https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance=300&st_loc_zip=8", nil, ) if err != nil { return nil, err } fe.setHeaders(req) return req, nil } func (fe FastExact) doRequest(req *http.Request) (*http.Response, error) { return do(req) } func (fe FastExact) parse(resp *http.Response) ([]Job, error) { b, _ := ioutil.ReadAll(resp.Body) if !bytes.HasPrefix(bytes.TrimSpace(b), []byte("<")) { gzip, err := gzip.NewReader(bytes.NewReader(b)) if err != nil { return nil, err } b, _ = ioutil.ReadAll(gzip) } logtr.Verbosef("fe.parse %s", b) resp.Body = io.NopCloser(bytes.NewReader(b)) doc, err := goquery.NewDocumentFromReader(resp.Body) if err != nil { return nil, err } result := make([]Job, 0) doc.Find("#list table tr").Each(func(i int, s *goquery.Selection) { columns := []string{} s.Find("td").Each(func(i int, s *goquery.Selection) { if s.Nodes[0].LastChild != nil && len(s.Nodes[0].LastChild.Attr) > 0 { attrs := s.Nodes[0].LastChild.Attr columns = append(columns, attrs[len(attrs)-1].Val) } else { columns = append(columns, s.Text()) } }) if len(columns) < 9 { return } job := Job{ ID: columns[0], URI: columns[8], } job.Pickup.Date, _ = time.ParseInLocation("02-Jan-2006 15:04:05", columns[7], time.Local) job.Pickup.City = strings.Title(strings.ToLower(strings.Split(columns[1], ",")[0])) if strings.Contains(columns[1], ",") { job.Pickup.State = strings.Title(strings.Split(strings.Split(columns[1], ",")[1], " ")[1]) } job.Dropoff.Date = job.Pickup.Date job.Dropoff.City = strings.Title(strings.ToLower(strings.Split(columns[2], ",")[0])) if strings.Contains(columns[2], ",") { job.Dropoff.State = strings.Title(strings.Split(strings.Split(columns[2], ",")[1], " ")[1]) } job.Miles, _ = strconv.Atoi(columns[3]) if strings.Contains(columns[4], "/") { weight, _ := strconv.ParseFloat(strings.TrimSpace(strings.Split(columns[4], "/")[1]), 32) job.Weight = int(weight) } job.Meta = fmt.Sprintf(`dimensions:%s`, strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(columns[5], " ", ""), "\n", ""))) logtr.Verbosef("fe.parse %+v => %+v", columns, job) result = append(result, job) }) return result, nil } type mockFastExactDoer struct{} func (mock mockFastExactDoer) doRequest(req *http.Request) (*http.Response, error) { if req.URL.Path != "/secure/index.php" { return nil, errors.New("bad path") } switch req.URL.Query().Get("page") { case "userLogin": if b, _ := ioutil.ReadAll(req.Body); !bytes.Equal(b, []byte(`user_name=u&user_password=p&buttonSubmit=Login`)) { return nil, errors.New("bad req body") } return &http.Response{ Status: http.StatusText(http.StatusOK), StatusCode: http.StatusOK, Header: http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromLogin; path=/"}}, Body: io.NopCloser(bytes.NewReader([]byte{})), }, nil case "ajaxListJobs": if req.URL.Query().Get("action") != "ajax" { return nil, errors.New("bad query: action should be ajax") } if req.URL.Query().Get("records_per_page") != "50" { return nil, errors.New("bad query: records_per_page should be 50") } if req.URL.Query().Get("distance") != "300" { return nil, errors.New("bad query: distance should be 300") } if req.URL.Query().Get("zipcode") == "" { return nil, errors.New("bad query: zip code empty") } b, err := ioutil.ReadFile("./testdata/fastexact_search.xml") if err != nil { b, err = ioutil.ReadFile("./broker/testdata/fastexact_search.xml") } if err != nil { return nil, err } return &http.Response{ Status: http.StatusText(http.StatusOK), StatusCode: http.StatusOK, Header: http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromSearch; path=/"}}, Body: io.NopCloser(bytes.NewReader(b)), }, nil } return nil, errors.New("bad query") }