impl fastexact SearchZips and dedupe diff-id same-job for fastexact

broker/broker.go

@@ -21,6 +21,7 @@ var limiter = rate.NewLimiter(rate.Limit(1.0/20.0), 1)
 
 type Broker interface {
 	SearchStates([]config.State) ([]Job, error)
+	SearchZips([]string) ([]Job, error)
 }
 
 func do(r *http.Request) (*http.Response, error) {

broker/fastexact.go

@@ -11,6 +11,7 @@ import (
 	"local/truckstop/config"
 	"local/truckstop/logtr"
 	"net/http"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -38,6 +39,17 @@ func (fe FastExact) WithMock() FastExact {
 	return fe
 }
 
+func (fe FastExact) SearchZips(zips []string) ([]Job, error) {
+	jobs, err := fe.searchZips(zips)
+	if err == ErrNoAuth {
+		if err := fe.login(); err != nil {
+			return nil, err
+		}
+		jobs, err = fe.searchZips(zips)
+	}
+	return jobs, err
+}
+
 func (fe FastExact) SearchStates(states []config.State) ([]Job, error) {
 	jobs, err := fe.searchStates(states)
 	if err == ErrNoAuth {
@@ -91,28 +103,47 @@ func (fe FastExact) setHeaders(req *http.Request) {
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 }
 
-func (fe FastExact) searchStates(states []config.State) ([]Job, error) {
+func (fe FastExact) searchZips(zips []string) ([]Job, error) {
 	var jobs []Job
-	for _, state := range states {
-		subjobs, err := fe.searchOne(state)
+	for _, zip := range zips {
+		subjobs, err := fe.searchOneZip(zip)
 		if err != nil {
 			return nil, err
 		}
 		jobs = append(jobs, subjobs...)
 	}
+	return fe.dedupeJobs(jobs), nil
+}
+
+func (fe FastExact) searchStates(states []config.State) ([]Job, error) {
+	var jobs []Job
+	for _, state := range states {
+		subjobs, err := fe.searchOneState(state)
+		if err != nil {
+			return nil, err
+		}
+		jobs = append(jobs, subjobs...)
+	}
+	return fe.dedupeJobs(jobs), nil
+}
+
+func (fe FastExact) dedupeJobs(jobs []Job) []Job {
+	sort.Slice(jobs, func(i, j int) bool {
+		return jobs[i].UID() < jobs[j].UID()
+	})
 	dedupeJobs := map[string]Job{}
 	for _, job := range jobs {
-		dedupeJobs[job.UID()] = job
+		dedupeJobs[strings.ReplaceAll(job.UID(), job.ID, "")] = job
 	}
 	result := []Job{}
 	for _, job := range dedupeJobs {
 		result = append(result, job)
 	}
-	return result, nil
+	return result
 }
 
-func (fe FastExact) searchOne(state config.State) ([]Job, error) {
-	req, err := fe.newRequest(state)
+func (fe FastExact) searchOneZip(zip string) ([]Job, error) {
+	req, err := fe.newRequest(zip)
 	if err != nil {
 		return nil, err
 	}
@@ -124,11 +155,28 @@ func (fe FastExact) searchOne(state config.State) ([]Job, error) {
 	return fe.parse(resp)
 }
 
-func (fe FastExact) newRequest(state config.State) (*http.Request, error) {
+func (fe FastExact) searchOneState(state config.State) ([]Job, error) {
+	req, err := fe.newRequestWithState(state)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := fe.doer.doRequest(req)
+	logtr.Verbosef("req: %+v => resp: %+v", req, resp)
+	if err != nil {
+		return nil, err
+	}
+	return fe.parse(resp)
+}
+
+func (fe FastExact) newRequestWithState(state config.State) (*http.Request, error) {
 	zip, ok := config.States[state]
 	if !ok {
 		return nil, fmt.Errorf("no configured zip for %s", state)
 	}
+	return fe.newRequest(zip)
+}
+
+func (fe FastExact) newRequest(zip string) (*http.Request, error) {
 	req, err := http.NewRequest(
 		http.MethodGet,
 		"https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance="+strconv.Itoa(config.Get().Brokers.FastExact.RadiusMiles)+"&st_loc_zip=8",

broker/fastexact_test.go

@@ -65,7 +65,7 @@ func TestFastExactSearchStates(t *testing.T) {
 	_ = db
 	if jobs, err := fe.searchStates([]config.State{config.State("NC"), config.State("SC")}); err != nil {
 		t.Fatal(err)
-	} else if len(jobs) != 10 {
+	} else if len(jobs) != 9 {
 		t.Fatal(len(jobs))
 	} else {
 		for _, job := range jobs {

broker/ntgvision.go

@@ -187,6 +187,10 @@ func (ntg NTGVision) searchJob(id int64) (ntgVisionJobInfo, error) {
 	return result, err
 }
 
+func (ntg NTGVision) SearchZips(zips []string) ([]Job, error) {
+	return nil, errors.New("not impl: ntg search zips")
+}
+
 func (ntg NTGVision) SearchStates(states []config.State) ([]Job, error) {
 	rc, err := ntg.searcher.searchStates(states)
 	if err != nil {