truckstop/broker/fastexact.go

package broker

import (
	"bytes"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"gogs.inhome.blapointe.com/local/storage"
	"gogs.inhome.blapointe.com/local/truckstop/config"
	"gogs.inhome.blapointe.com/local/truckstop/logtr"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

type FastExact struct {
	doer interface {
		doRequest(*http.Request) (*http.Response, error)
	}
}

func NewFastExact() FastExact {
	fe := FastExact{}
	fe.doer = fe
	if config.Get().Brokers.FastExact.Mock {
		fe = fe.WithMock()
	}
	return fe
}

func (fe FastExact) WithMock() FastExact {
	fe.doer = mockFastExactDoer{}
	return fe
}

func (fe FastExact) SearchZips(zips []string) ([]Job, error) {
	jobs, err := fe.searchZips(zips)
	if err == ErrNoAuth {
		if err := fe.login(); err != nil {
			return nil, err
		}
		jobs, err = fe.searchZips(zips)
	}
	return jobs, err
}

func (fe FastExact) SearchStates(states []config.State) ([]Job, error) {
	jobs, err := fe.searchStates(states)
	if err == ErrNoAuth {
		if err := fe.login(); err != nil {
			return nil, err
		}
		jobs, err = fe.searchStates(states)
	}
	return jobs, err
}

func (fe FastExact) login() error {
	conf := config.Get()
	return fe._login(conf.Brokers.FastExact.Username, conf.Brokers.FastExact.Password, conf.DB())
}

func (fe FastExact) _login(username, password string, db storage.DB) error {
	req, err := http.NewRequest(
		http.MethodPost,
		`https://www.fastexact.com/secure/index.php?page=userLogin`,
		strings.NewReader(fmt.Sprintf(
			`user_name=%s&user_password=%s&buttonSubmit=Login`,
			username,
			password,
		)),
	)
	if err != nil {
		return err
	}
	fe.setHeaders(req)

	db.Set("cookies_"+req.URL.Host, nil)
	resp, err := fe.doer.doRequest(req)
	if err != nil {
		return err
	}
	b, _ := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("bad status logging into fast exact: %d: %s", resp.StatusCode, b)
	}
	return nil
}

func (fe FastExact) setHeaders(req *http.Request) {
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0")
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
	req.Header.Set("Accept-Language", "en-US,en;q=0.5")
	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}

func (fe FastExact) searchZips(zips []string) ([]Job, error) {
	var jobs []Job
	for _, zip := range zips {
		subjobs, err := fe.searchOneZip(zip)
		if err != nil {
			return nil, err
		}
		jobs = append(jobs, subjobs...)
	}
	return fe.dedupeJobs(jobs), nil
}

func (fe FastExact) searchStates(states []config.State) ([]Job, error) {
	var jobs []Job
	for _, state := range states {
		subjobs, err := fe.searchOneState(state)
		if err != nil {
			return nil, err
		}
		jobs = append(jobs, subjobs...)
	}
	return fe.dedupeJobs(jobs), nil
}

func (fe FastExact) dedupeJobs(jobs []Job) []Job {
	sort.Slice(jobs, func(i, j int) bool {
		return jobs[i].UID() < jobs[j].UID()
	})
	dedupeJobs := map[string]Job{}
	for _, job := range jobs {
		dedupeJobs[strings.ReplaceAll(job.UID(), job.ID, "")] = job
	}
	result := []Job{}
	for _, job := range dedupeJobs {
		result = append(result, job)
	}
	return result
}

func (fe FastExact) searchOneZip(zip string) ([]Job, error) {
	req, err := fe.newRequest(zip)
	if err != nil {
		return nil, err
	}
	resp, err := fe.doer.doRequest(req)
	logtr.Verbosef("req: %+v => resp: %+v", req, resp)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return fe.parse(resp)
}

func (fe FastExact) searchOneState(state config.State) ([]Job, error) {
	req, err := fe.newRequestWithState(state)
	if err != nil {
		return nil, err
	}
	resp, err := fe.doer.doRequest(req)
	logtr.Verbosef("req: %+v => resp: %+v", req, resp)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return fe.parse(resp)
}

func (fe FastExact) newRequestWithState(state config.State) (*http.Request, error) {
	zip, ok := config.States[state]
	if !ok {
		return nil, fmt.Errorf("no configured zip for %s", state)
	}
	return fe.newRequest(zip)
}

func (fe FastExact) newRequest(zip string) (*http.Request, error) {
	req, err := http.NewRequest(
		http.MethodGet,
		"https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance="+strconv.Itoa(config.Get().Brokers.RadiusMiles)+"&st_loc_zip=8",
		nil,
	)
	if err != nil {
		return nil, err
	}
	fe.setHeaders(req)
	return req, nil
}

func (fe FastExact) doRequest(req *http.Request) (*http.Response, error) {
	resp, err := do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		b, _ := ioutil.ReadAll(resp.Body)
		logtr.Errorf("fastExact bad status: %d: %s", resp.StatusCode, b)
		if resp.StatusCode > 400 && resp.StatusCode < 404 {
			return nil, ErrNoAuth
		}
		return nil, fmt.Errorf("bad status from FastExact: %d: %s", resp.StatusCode, b)
	}
	return resp, nil
}

func (fe FastExact) parse(resp *http.Response) ([]Job, error) {
	b, _ := ioutil.ReadAll(resp.Body)
	if !bytes.HasPrefix(bytes.TrimSpace(b), []byte("<")) {
		gzip, err := gzip.NewReader(bytes.NewReader(b))
		if err != nil {
			return nil, err
		}
		b, _ = ioutil.ReadAll(gzip)
	}
	if bytes.Contains(b, []byte(`<script>window.location.href='index.php'</script>`)) {
		return nil, ErrNoAuth
	}
	logtr.Verbosef("fe.parse %s", b)
	resp.Body = io.NopCloser(bytes.NewReader(b))
	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return nil, err
	}
	result := make([]Job, 0)
	doc.Find("#list table tr").Each(func(i int, s *goquery.Selection) {
		columns := []string{}
		s.Find("td").Each(func(i int, s *goquery.Selection) {
			if s.Nodes[0].LastChild != nil && len(s.Nodes[0].LastChild.Attr) > 0 {
				attrs := s.Nodes[0].LastChild.Attr
				columns = append(columns, attrs[len(attrs)-1].Val)
			} else {
				columns = append(columns, s.Text())
			}
		})
		if len(columns) < 9 {
			return
		}
		job := Job{
			ID:  columns[0],
			URI: columns[8],
		}
		job.Pickup.Date, _ = time.ParseInLocation("02-Jan-2006  15:04:05", columns[7], time.Local)
		job.Pickup.City = strings.Title(strings.ToLower(strings.Split(columns[1], ",")[0]))
		if strings.Contains(columns[1], ",") {
			job.Pickup.State = strings.Title(strings.Split(strings.Split(columns[1], ",")[1], " ")[1])
		}

		job.Dropoff.Date = job.Pickup.Date
		job.Dropoff.City = strings.Title(strings.ToLower(strings.Split(columns[2], ",")[0]))
		if strings.Contains(columns[2], ",") {
			job.Dropoff.State = strings.Title(strings.Split(strings.Split(columns[2], ",")[1], " ")[1])
		}

		job.Miles, _ = strconv.Atoi(columns[3])
		if strings.Contains(columns[4], "/") {
			weight, _ := strconv.ParseFloat(strings.TrimSpace(strings.Split(columns[4], "/")[1]), 32)
			job.Weight = int(weight)
		}

		job.Meta = fmt.Sprintf(`dimensions:%s`, strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(columns[5], " ", ""), "\n", "")))

		logtr.Verbosef("fe.parse %+v => %+v", columns, job)

		result = append(result, job)
	})
	return result, nil
}

type mockFastExactDoer struct{}

func (mock mockFastExactDoer) doRequest(req *http.Request) (*http.Response, error) {
	if req.URL.Path != "/secure/index.php" {
		return nil, errors.New("bad path")
	}
	switch req.URL.Query().Get("page") {
	case "userLogin":
		if b, _ := ioutil.ReadAll(req.Body); !bytes.Equal(b, []byte(`user_name=u&user_password=p&buttonSubmit=Login`)) {
			return nil, errors.New("bad req body")
		}
		return &http.Response{
			Status:     http.StatusText(http.StatusOK),
			StatusCode: http.StatusOK,
			Header:     http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromLogin; path=/"}},
			Body:       io.NopCloser(bytes.NewReader([]byte{})),
		}, nil
	case "ajaxListJobs":
		if req.URL.Query().Get("action") != "ajax" {
			return nil, errors.New("bad query: action should be ajax")
		}
		if req.URL.Query().Get("records_per_page") != "50" {
			return nil, errors.New("bad query: records_per_page should be 50")
		}
		if req.URL.Query().Get("distance") != strconv.Itoa(config.Get().Brokers.RadiusMiles) {
			return nil, errors.New("bad query: distance should be as configured")
		}
		if req.URL.Query().Get("zipcode") == "" {
			return nil, errors.New("bad query: zip code empty")
		}
		b, err := ioutil.ReadFile("./testdata/fastexact_search.xml")
		if err != nil {
			b, err = ioutil.ReadFile("./broker/testdata/fastexact_search.xml")
		}
		if err != nil {
			return nil, err
		}
		return &http.Response{
			Status:     http.StatusText(http.StatusOK),
			StatusCode: http.StatusOK,
			Header:     http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromSearch; path=/"}},
			Body:       io.NopCloser(bytes.NewReader(b)),
		}, nil
	}
	return nil, errors.New("bad query")
}