Merge branch 'master' of http://gogs.scratch.com:59515/bel/truckstop

broker/fastexact.go

@@ -92,13 +92,21 @@ func (fe FastExact) setHeaders(req *http.Request) {
 }
 
 func (fe FastExact) search(states []config.State) ([]Job, error) {
-	var result []Job
+	var jobs []Job
 	for _, state := range states {
-		jobs, err := fe.searchOne(state)
+		subjobs, err := fe.searchOne(state)
 		if err != nil {
 			return nil, err
 		}
-		result = append(result, jobs...)
+		jobs = append(jobs, subjobs...)
+	}
+	dedupeJobs := map[string]Job{}
+	for _, job := range jobs {
+		dedupeJobs[job.UID()] = job
+	}
+	result := []Job{}
+	for _, job := range dedupeJobs {
+		result = append(result, job)
 	}
 	return result, nil
 }
@@ -123,7 +131,7 @@ func (fe FastExact) newRequest(state config.State) (*http.Request, error) {
 	}
 	req, err := http.NewRequest(
 		http.MethodGet,
-		"https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance=300&st_loc_zip=8",
+		"https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance="+strconv.Itoa(config.Get().Brokers.FastExact.RadiusMiles)+"&st_loc_zip=8",
 		nil,
 	)
 	if err != nil {
@@ -158,6 +166,9 @@ func (fe FastExact) parse(resp *http.Response) ([]Job, error) {
 		}
 		b, _ = ioutil.ReadAll(gzip)
 	}
+	if bytes.Contains(b, []byte(`<script>window.location.href='index.php'</script>`)) {
+		return nil, ErrNoAuth
+	}
 	logtr.Verbosef("fe.parse %s", b)
 	resp.Body = io.NopCloser(bytes.NewReader(b))
 	doc, err := goquery.NewDocumentFromReader(resp.Body)
@@ -233,8 +244,8 @@ func (mock mockFastExactDoer) doRequest(req *http.Request) (*http.Response, erro
 		if req.URL.Query().Get("records_per_page") != "50" {
 			return nil, errors.New("bad query: records_per_page should be 50")
 		}
-		if req.URL.Query().Get("distance") != "300" {
-			return nil, errors.New("bad query: distance should be 300")
+		if req.URL.Query().Get("distance") != strconv.Itoa(config.Get().Brokers.FastExact.RadiusMiles) {
+			return nil, errors.New("bad query: distance should be as configured")
 		}
 		if req.URL.Query().Get("zipcode") == "" {
 			return nil, errors.New("bad query: zip code empty")

broker/fastexact_test.go

@@ -65,8 +65,8 @@ func TestFastExactSearch(t *testing.T) {
 	_ = db
 	if jobs, err := fe.search([]config.State{config.State("NC"), config.State("SC")}); err != nil {
 		t.Fatal(err)
-	} else if len(jobs) != 20 {
-		t.Fatal(jobs)
+	} else if len(jobs) != 10 {
+		t.Fatal(len(jobs))
 	} else {
 		for _, job := range jobs {
 			if job.ID == "" {

config.json

@@ -34,7 +34,7 @@
       "Pickup": false,
       "Dropoff": false,
       "Pathed": {
-         "Enabled": true,
+         "Enabled": false,
          "DirectionsURIFormat": "https://maps.googleapis.com/maps/api/directions/json?origin=%s\u0026destination=%s\u0026mode=driving\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
          "PathedURIFormat": "https://maps.googleapis.com/maps/api/staticmap?size=250x250\u0026path=%s\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg\u0026size=250x250\u0026markers=%s|%s",
          "Zoom": {
@@ -69,14 +69,17 @@
          "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
       }
    },
-   "Once": true,
+   "Once": false,
    "Brokers": {
       "FastExact": {
-         "Mock": false,
+			"RadiusMiles": 100,
+         "Enabled": true,
+         "Mock": true,
          "Username": "birdman",
          "Password": "166647"
       },
       "NTG": {
+         "Enabled": false,
          "JobInfo": true,
          "Mock": true,
          "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",

config.main_test.json

@@ -72,11 +72,14 @@
    "Once": true,
    "Brokers": {
       "FastExact": {
+			"RadiusMiles": 100,
+         "Enabled": true,
          "Mock": true,
          "Username": "u",
          "Password": "p"
       },
       "NTG": {
+         "Enabled": true,
          "JobInfo": true,
          "Mock": true,
          "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",

config/config.go

@@ -65,6 +65,7 @@ type Config struct {
 	Once    bool
 	Brokers struct {
 		NTG struct {
+			Enabled              bool
 			JobInfo              bool
 			Mock                 bool
 			LoadPageURIFormat    string
@@ -73,9 +74,11 @@ type Config struct {
 			Password             string
 		}
 		FastExact struct {
-			Mock     bool
-			Username string
-			Password string
+			Enabled     bool
+			RadiusMiles int
+			Mock        bool
+			Username    string
+			Password    string
 		}
 	}
 

main.go

@@ -303,7 +303,7 @@ func once() error {
 		if ok, err := sendJob(jobs[i]); err != nil {
 			return err
 		} else if ok {
-			logtr.Debugf("sent job", jobs[i])
+			logtr.Debugf("sent job %+v", jobs[i])
 			if err := db.Set(jobs[i].UID(), []byte(`sent`)); err != nil {
 				return err
 			}
@@ -314,9 +314,16 @@ func once() error {
 
 func getJobs() ([]broker.Job, error) {
 	states := config.AllStates()
-	ntg := broker.NewNTGVision()
-	fe := broker.NewFastExact()
-	brokers := []broker.Broker{ntg, fe}
+	brokers := []broker.Broker{}
+	if config.Get().Brokers.NTG.Enabled {
+		logtr.Debugf("NTG enabled")
+		brokers = append(brokers, broker.NewNTGVision())
+	}
+	if config.Get().Brokers.FastExact.Enabled {
+		logtr.Debugf("FastExact enabled")
+		brokers = append(brokers, broker.NewFastExact())
+	}
+	logtr.Debugf("brokers=%+v", brokers)
 	jobs := []broker.Job{}
 	for _, broker := range brokers {
 		somejobs, err := broker.Search(states)
@@ -345,7 +352,7 @@ func updateDeadJobs(jobs []broker.Job) error {
 		wouldBe := strings.TrimPrefix(listEntry, "sent_job_")
 		found := false
 		for i := range jobs {
-			found = found || jobs[i].ID == wouldBe
+			found = found || jobs[i].UID() == wouldBe || jobs[i].ID == wouldBe
 		}
 		logtr.Debugf("found job %s to be still alive==%v", wouldBe, found)
 		if !found {
@@ -364,14 +371,14 @@ func updateDeadJobs(jobs []broker.Job) error {
 			}
 			*/
 			if err := message.NewMatrix().Remove(recorded.MatrixID); err != nil {
-				return err
+				logtr.Debugf("failed to remove matrix: %v: %v", recorded.MatrixID, err)
 			}
 			if err := db.Set(listEntry, nil); err != nil {
-				return err
+				logtr.Debugf("failed to remove db: %v: %v", listEntry, err)
 			}
 			for _, imageid := range recorded.MatrixImageIDs {
 				if err := message.NewMatrix().Remove(imageid); err != nil {
-					return err
+					logtr.Debugf("failed to remove matrix image: %v: %v", imageid, err)
 				}
 			}
 		}
@@ -400,7 +407,7 @@ func dropBanlistJobs(jobs []broker.Job) ([]broker.Job, error) {
 func sendJob(job broker.Job) (bool, error) {
 	sender := message.NewMatrix()
 	payload := job.FormatMultilineText()
-	logtr.Debugf("once: send job %s if nonzero: %s", job.String(), payload)
+	logtr.Debugf("once: send job %s if nonzero: %q", job.String(), payload)
 	if len(payload) == 0 {
 		return false, nil
 	}
@@ -420,7 +427,7 @@ func sendJob(job broker.Job) (bool, error) {
 			logtr.Errorf("failed to marshal recorded job: %v", err)
 			return
 		}
-		if err := db.Set("sent_job_"+job.ID, b); err != nil {
+		if err := db.Set("sent_job_"+job.UID(), b); err != nil {
 			logtr.Errorf("failed to set recorded job: %v", err)
 			return
 		}

todo.yaml

@@ -1,20 +1,21 @@
 todo:
-- !states emits current state 
-- TEST. Just like, refactor and test to shit.
-- try search ntg by autoinc?
+- fast exact is dumb or...?
+- modify old items once no longer available; drop stale jobs good candidate but requires new matrix interaction
+- more than NTG; blue one
+- !states emits current state
 - test each !command callbacks to matrix
 - recv-as for clients so pa receives mas commands as writes
 - continuation is garbo, but I can still do better client side to avoid get-set high level
 - no hard code jpeg or have it in multiple places
 - change matrix so I test my custom logic even if I dont fetch remote
 - warn/err/etc. on clobbering ids.matrix since clients can mess with one another
-- modify old items once no longer available; drop stale jobs good candidate but requires new matrix interaction
-- more than NTG
 - todo: filter out jobs like CA
   subtasks:
   - banlist criteria like vendors, brokers, metadata
 - set up copy for caleb, broc
 done:
+- try search ntg by autoinc?
+- TEST. Just like, refactor and test to shit.
 - mark jobs no longer avail by modifying in matrix;; save matrix ID over dummy payload
 - TEST its falling apart
 - help() log on truckstop for stuff like perma 403