unit tests are good

broker/fastexact.go

@@ -92,13 +92,21 @@ func (fe FastExact) setHeaders(req *http.Request) {
 }
 
 func (fe FastExact) search(states []config.State) ([]Job, error) {
-	var result []Job
+	var jobs []Job
 	for _, state := range states {
-		jobs, err := fe.searchOne(state)
+		subjobs, err := fe.searchOne(state)
 		if err != nil {
 			return nil, err
 		}
-		result = append(result, jobs...)
+		jobs = append(jobs, subjobs...)
+	}
+	dedupeJobs := map[string]Job{}
+	for _, job := range jobs {
+		dedupeJobs[job.UID()] = job
+	}
+	result := []Job{}
+	for _, job := range dedupeJobs {
+		result = append(result, job)
 	}
 	return result, nil
 }
@@ -123,7 +131,7 @@ func (fe FastExact) newRequest(state config.State) (*http.Request, error) {
 	}
 	req, err := http.NewRequest(
 		http.MethodGet,
-		"https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance=300&st_loc_zip=8",
+		"https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance="+strconv.Itoa(config.Get().Brokers.FastExact.RadiusMiles)+"&st_loc_zip=8",
 		nil,
 	)
 	if err != nil {
@@ -158,6 +166,9 @@ func (fe FastExact) parse(resp *http.Response) ([]Job, error) {
 		}
 		b, _ = ioutil.ReadAll(gzip)
 	}
+	if bytes.Contains(b, []byte(`<script>window.location.href='index.php'</script>`)) {
+		return nil, ErrNoAuth
+	}
 	logtr.Verbosef("fe.parse %s", b)
 	resp.Body = io.NopCloser(bytes.NewReader(b))
 	doc, err := goquery.NewDocumentFromReader(resp.Body)
@@ -233,8 +244,8 @@ func (mock mockFastExactDoer) doRequest(req *http.Request) (*http.Response, erro
 		if req.URL.Query().Get("records_per_page") != "50" {
 			return nil, errors.New("bad query: records_per_page should be 50")
 		}
-		if req.URL.Query().Get("distance") != "300" {
-			return nil, errors.New("bad query: distance should be 300")
+		if req.URL.Query().Get("distance") != strconv.Itoa(config.Get().Brokers.FastExact.RadiusMiles) {
+			return nil, errors.New("bad query: distance should be as configured")
 		}
 		if req.URL.Query().Get("zipcode") == "" {
 			return nil, errors.New("bad query: zip code empty")

broker/fastexact_test.go

@@ -65,8 +65,8 @@ func TestFastExactSearch(t *testing.T) {
 	_ = db
 	if jobs, err := fe.search([]config.State{config.State("NC"), config.State("SC")}); err != nil {
 		t.Fatal(err)
-	} else if len(jobs) != 20 {
-		t.Fatal(jobs)
+	} else if len(jobs) != 10 {
+		t.Fatal(len(jobs))
 	} else {
 		for _, job := range jobs {
 			if job.ID == "" {

config.json

@@ -72,11 +72,14 @@
    "Once": true,
    "Brokers": {
       "FastExact": {
-         "Mock": false,
-         "Username": "birdman",
-         "Password": "166647"
+			"RadiusMiles": 100,
+         "Enabled": true,
+         "Mock": true,
+         "Username": "u",
+         "Password": "p"
       },
       "NTG": {
+         "Enabled": true,
          "JobInfo": true,
          "Mock": true,
          "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",

config.main_test.json

@@ -72,11 +72,14 @@
    "Once": true,
    "Brokers": {
       "FastExact": {
+			"RadiusMiles": 100,
+         "Enabled": true,
          "Mock": true,
          "Username": "u",
          "Password": "p"
       },
       "NTG": {
+         "Enabled": true,
          "JobInfo": true,
          "Mock": true,
          "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",

config/config.go

@@ -65,6 +65,7 @@ type Config struct {
 	Once    bool
 	Brokers struct {
 		NTG struct {
+			Enabled              bool
 			JobInfo              bool
 			Mock                 bool
 			LoadPageURIFormat    string
@@ -73,9 +74,11 @@ type Config struct {
 			Password             string
 		}
 		FastExact struct {
-			Mock     bool
-			Username string
-			Password string
+			Enabled     bool
+			RadiusMiles int
+			Mock        bool
+			Username    string
+			Password    string
 		}
 	}
 

main.go

@@ -314,9 +314,13 @@ func once() error {
 
 func getJobs() ([]broker.Job, error) {
 	states := config.AllStates()
-	ntg := broker.NewNTGVision()
-	fe := broker.NewFastExact()
-	brokers := []broker.Broker{ntg, fe}
+	brokers := []broker.Broker{}
+	if config.Get().Brokers.NTG.Enabled {
+		brokers = append(brokers, broker.NewNTGVision())
+	}
+	if config.Get().Brokers.FastExact.Enabled {
+		brokers = append(brokers, broker.NewFastExact())
+	}
 	jobs := []broker.Job{}
 	for _, broker := range brokers {
 		somejobs, err := broker.Search(states)