diff --git a/broker/fastexact.go b/broker/fastexact.go index ec68efb..4e955f2 100644 --- a/broker/fastexact.go +++ b/broker/fastexact.go @@ -92,13 +92,21 @@ func (fe FastExact) setHeaders(req *http.Request) { } func (fe FastExact) search(states []config.State) ([]Job, error) { - var result []Job + var jobs []Job for _, state := range states { - jobs, err := fe.searchOne(state) + subjobs, err := fe.searchOne(state) if err != nil { return nil, err } - result = append(result, jobs...) + jobs = append(jobs, subjobs...) + } + dedupeJobs := map[string]Job{} + for _, job := range jobs { + dedupeJobs[job.UID()] = job + } + result := []Job{} + for _, job := range dedupeJobs { + result = append(result, job) } return result, nil } @@ -123,7 +131,7 @@ func (fe FastExact) newRequest(state config.State) (*http.Request, error) { } req, err := http.NewRequest( http.MethodGet, - "https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance=300&st_loc_zip=8", + "https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance="+strconv.Itoa(config.Get().Brokers.FastExact.RadiusMiles)+"&st_loc_zip=8", nil, ) if err != nil { @@ -236,8 +244,8 @@ func (mock mockFastExactDoer) doRequest(req *http.Request) (*http.Response, erro if req.URL.Query().Get("records_per_page") != "50" { return nil, errors.New("bad query: records_per_page should be 50") } - if req.URL.Query().Get("distance") != "300" { - return nil, errors.New("bad query: distance should be 300") + if req.URL.Query().Get("distance") != strconv.Itoa(config.Get().Brokers.FastExact.RadiusMiles) { + return nil, errors.New("bad query: distance should be as configured") } if req.URL.Query().Get("zipcode") == "" { return nil, errors.New("bad query: zip code empty") diff --git a/broker/fastexact_test.go b/broker/fastexact_test.go index e67a2d0..add5dca 100644 --- a/broker/fastexact_test.go +++ b/broker/fastexact_test.go @@ -65,8 +65,8 @@ func TestFastExactSearch(t *testing.T) { _ = db if jobs, err := fe.search([]config.State{config.State("NC"), config.State("SC")}); err != nil { t.Fatal(err) - } else if len(jobs) != 20 { - t.Fatal(jobs) + } else if len(jobs) != 10 { + t.Fatal(len(jobs)) } else { for _, job := range jobs { if job.ID == "" { diff --git a/config.json b/config.json index c94f84d..a1149ac 100644 --- a/config.json +++ b/config.json @@ -72,11 +72,14 @@ "Once": true, "Brokers": { "FastExact": { - "Mock": false, - "Username": "birdman", - "Password": "166647" + "RadiusMiles": 100, + "Enabled": true, + "Mock": true, + "Username": "u", + "Password": "p" }, "NTG": { + "Enabled": true, "JobInfo": true, "Mock": true, "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d", diff --git a/config.main_test.json b/config.main_test.json index b08b6f3..0af08fe 100644 --- a/config.main_test.json +++ b/config.main_test.json @@ -72,11 +72,14 @@ "Once": true, "Brokers": { "FastExact": { + "RadiusMiles": 100, + "Enabled": true, "Mock": true, "Username": "u", "Password": "p" }, "NTG": { + "Enabled": true, "JobInfo": true, "Mock": true, "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d", diff --git a/config/config.go b/config/config.go index 6019fcb..31c6a34 100644 --- a/config/config.go +++ b/config/config.go @@ -65,6 +65,7 @@ type Config struct { Once bool Brokers struct { NTG struct { + Enabled bool JobInfo bool Mock bool LoadPageURIFormat string @@ -73,9 +74,11 @@ type Config struct { Password string } FastExact struct { - Mock bool - Username string - Password string + Enabled bool + RadiusMiles int + Mock bool + Username string + Password string } } diff --git a/main.go b/main.go index e7717c5..9348877 100644 --- a/main.go +++ b/main.go @@ -314,9 +314,16 @@ func once() error { func getJobs() ([]broker.Job, error) { states := config.AllStates() - ntg := broker.NewNTGVision() - fe := broker.NewFastExact() - brokers := []broker.Broker{ntg, fe} + brokers := []broker.Broker{} + if config.Get().Brokers.NTG.Enabled { + logtr.Debugf("NTG enabled") + brokers = append(brokers, broker.NewNTGVision()) + } + if config.Get().Brokers.FastExact.Enabled { + logtr.Debugf("FastExact enabled") + brokers = append(brokers, broker.NewFastExact()) + } + logtr.Debugf("brokers=%+v", brokers) jobs := []broker.Job{} for _, broker := range brokers { somejobs, err := broker.Search(states)