package broker import ( "bytes" "errors" "fmt" "io" "io/ioutil" "local/storage" "local/truckstop/config" "net/http" "strings" ) type FastExact struct { doer interface { doRequest(*http.Request) (*http.Response, error) } } func NewFastExact() FastExact { fe := FastExact{} fe.doer = fe return fe } func (fe FastExact) WithMock() FastExact { fe.doer = mockFastExactDoer{} return fe } func (fe FastExact) Search(states []config.State) ([]Job, error) { jobs, err := fe.search(states) if err == ErrNoAuth { if err := fe.login(); err != nil { return nil, err } jobs, err = fe.search(states) } return jobs, err } func (fe FastExact) login() error { conf := config.Get() return fe._login(conf.Brokers.FastExact.Username, conf.Brokers.FastExact.Password, conf.DB()) } func (fe FastExact) _login(username, password string, db storage.DB) error { req, err := http.NewRequest( http.MethodPost, `https://www.fastexact.com/secure/index.php?page=userLogin`, strings.NewReader(fmt.Sprintf( `user_name=%s&user_password=%s&buttonSubmit=Login`, username, password, )), ) if err != nil { return err } fe.setHeaders(req) db.Set("cookies_"+req.URL.Host, nil) resp, err := fe.doer.doRequest(req) if err != nil { return err } b, _ := ioutil.ReadAll(resp.Body) resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("bad status logging into fast exact: %d: %s", resp.StatusCode, b) } return nil } func (fe FastExact) setHeaders(req *http.Request) { req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.5") req.Header.Set("Accept-Encoding", "gzip, deflate, br") req.Header.Set("Content-Type", "application/x-www-form-urlencoded") } func (fe FastExact) search(states []config.State) ([]Job, error) { var result []Job for _, state := range states { jobs, err := fe.searchOne(state) if err != nil { return nil, err } result = append(result, jobs...) } return result, nil } func (fe FastExact) searchOne(state config.State) ([]Job, error) { req, err := fe.newRequest(state) if err != nil { return nil, err } resp, err := fe.doer.doRequest(req) if err != nil { return nil, err } return fe.parse(resp) } func (fe FastExact) newRequest(state config.State) (*http.Request, error) { zip, ok := config.States[state] if !ok { return nil, fmt.Errorf("no configured zip for %s", state) } req, err := http.NewRequest( http.MethodGet, "https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance=300&st_loc_zip=8", nil, ) if err != nil { return nil, err } fe.setHeaders(req) return req, nil } func (fe FastExact) doRequest(req *http.Request) (*http.Response, error) { return nil, errors.New("not impl: fe.doreq") } func (fe FastExact) parse(resp *http.Response) ([]Job, error) { return nil, errors.New("not impl: fe.parse") } type mockFastExactDoer struct{} func (mock mockFastExactDoer) doRequest(req *http.Request) (*http.Response, error) { if req.URL.Path != "/secure/index.php" { return nil, errors.New("bad path") } switch req.URL.Query().Get("page") { case "userLogin": if b, _ := ioutil.ReadAll(req.Body); !bytes.Equal(b, []byte(`user_name=u&user_password=p&buttonSubmit=Login`)) { return nil, errors.New("bad req body") } return &http.Response{ Status: http.StatusText(http.StatusOK), StatusCode: http.StatusOK, Header: http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromLogin; path=/"}}, Body: io.NopCloser(bytes.NewReader([]byte{})), }, nil case "ajaxListJobs": if req.URL.Query().Get("action") != "ajax" { return nil, errors.New("bad query: action should be ajax") } if req.URL.Query().Get("records_per_page") != "50" { return nil, errors.New("bad query: records_per_page should be 50") } if req.URL.Query().Get("distance") != "300" { return nil, errors.New("bad query: distance should be 300") } if req.URL.Query().Get("zipcode") == "" { return nil, errors.New("bad query: zip code empty") } b, err := ioutil.ReadFile("./testdata/fastexact_search.xml") if err != nil { return nil, err } return &http.Response{ Status: http.StatusText(http.StatusOK), StatusCode: http.StatusOK, Header: http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromSearch; path=/"}}, Body: io.NopCloser(bytes.NewReader(b)), }, nil } return nil, errors.New("bad query") }