better ntg errs

broker/job.go

@@ -1,6 +1,7 @@
 package broker
 
 import (
+	"encoding/base64"
 	"fmt"
 	"local/truckstop/config"
 	"local/truckstop/logtr"
@@ -26,6 +27,18 @@ type JobLocation struct {
 	State string
 }
 
+func (j Job) UID() string {
+	return fmt.Sprintf(
+		"%v-%s-%s-%s-%s-%v",
+		j.ID,
+		j.Pickup.State,
+		base64.StdEncoding.EncodeToString([]byte(j.Pickup.City)),
+		j.Dropoff.State,
+		base64.StdEncoding.EncodeToString([]byte(j.Dropoff.City)),
+		j.Pickup.Date.Unix(),
+	)
+}
+
 func (j *Job) Secrets() {
 	if j.secrets == nil {
 		return

broker/ntgvision.go

@@ -109,7 +109,7 @@ func (ntgJob *ntgVisionJob) JobInfo() (ntgVisionJobInfo, error) {
 
 func (ntg NTGVision) searchJob(id int64) (io.ReadCloser, error) {
 	time.Sleep(config.Get().Interval.JobInfo.Get())
-	request, err := http.NewRequest(http.MethodGet, fmt.Sprintf(config.Get().Brokers.NTG.LoadPageURIFormat, id), nil)
+	request, err := http.NewRequest(http.MethodGet, fmt.Sprintf(config.Get().Brokers.NTG.LoadPageAPIURIFormat, id), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -175,8 +175,15 @@ func (ntg NTGVision) SearchJob(id int64) (ntgVisionJobInfo, error) {
 		return ntgVisionJobInfo{}, err
 	}
 	defer rc.Close()
+	b, err := ioutil.ReadAll(rc)
+	if err != nil {
+		return ntgVisionJobInfo{}, fmt.Errorf("failed to readall search job result: %w", err)
+	}
 	var result ntgVisionJobInfo
-	err = json.NewDecoder(rc).Decode(&result)
+	err = json.Unmarshal(b, &result)
+	if err != nil {
+		err = fmt.Errorf("failed to parse job info: %w: %s", err, b)
+	}
 	return result, err
 }
 
@@ -221,12 +228,14 @@ func setNTGToken(token string) {
 
 func (ntg NTGVision) search(states []config.State) (io.ReadCloser, error) {
 	if getNTGToken() == "" {
+		logtr.Debugf("NTG token is empty, refreshing ntg auth")
 		if err := ntg.refreshAuth(); err != nil {
 			return nil, err
 		}
 	}
 	rc, err := ntg._search(states)
 	if err == ErrNoAuth {
+		logtr.Debugf("err no auth on search, refreshing ntg auth")
 		if err := ntg.refreshAuth(); err != nil {
 			return nil, err
 		}
@@ -288,7 +297,9 @@ func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
 	if resp.StatusCode != http.StatusOK {
 		b, _ := ioutil.ReadAll(resp.Body)
 		resp.Body.Close()
-		if resp.StatusCode > 400 && resp.StatusCode < 500 && resp.StatusCode != 404 && resp.StatusCode != 410 {
+		logtr.Warnf("ntg auth bad status: url=%s, status=%v, body=%s", request.URL.String(), resp.StatusCode, b)
+		if resp.StatusCode > 400 && resp.StatusCode < 404 {
+			logtr.Warnf("ntg auth bad status: err no auth")
 			return nil, ErrNoAuth
 		}
 		return nil, fmt.Errorf("bad status searching ntg: %d: %s", resp.StatusCode, b)
@@ -298,7 +309,7 @@ func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
 
 func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) {
 	body, err := json.Marshal(map[string]interface{}{
-		"OriginFromDate":        time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
+		"OriginFromDate":        time.Now().Add(time.Hour * -24).UTC().Format("2006-01-02T15:04:05.000Z"),
 		"OriginToDate":          time.Now().UTC().Add(time.Hour * 24 * 30).Format("2006-01-02T15:04:05.000Z"),
 		"DestinationFromDate":   nil,
 		"DestinationToDate":     nil,

config.json

@@ -75,6 +75,7 @@
          "JobInfo": true,
          "Mock": true,
          "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",
+         "LoadPageAPIURIFormat": "https://ntgvision.com/api/v1/load/LoadDetails?loadId==%d",
          "Username": "noeasyrunstrucking@gmail.com",
          "Password": "thumper1234"
       }

config/config.go

@@ -65,11 +65,12 @@ type Config struct {
 	Once    bool
 	Brokers struct {
 		NTG struct {
-			JobInfo           bool
-			Mock              bool
-			LoadPageURIFormat string
-			Username          string
-			Password          string
+			JobInfo              bool
+			Mock                 bool
+			LoadPageURIFormat    string
+			LoadPageAPIURIFormat string
+			Username             string
+			Password             string
 		}
 	}
 

logtr/log.go

@@ -82,7 +82,7 @@ func SetLevel(l Level) {
 }
 
 func logf(l Level, format string, args []interface{}) {
-	format = fmt.Sprintf("%v: %v: %s\n", time.Now().Format("15:04:05"), l.String(), strings.TrimSpace(format))
+	format = fmt.Sprintf("%v: %v: %s\n", time.Now().Format("01-02T15:04:05"), l.String(), strings.TrimSpace(format))
 	cLevel := level
 	cAnsoser := ansoser
 	if l >= cLevel {
@@ -90,7 +90,9 @@ func logf(l Level, format string, args []interface{}) {
 	}
 	fmt.Fprintf(logger, format, args...)
 	if l == SOS && cAnsoser != nil {
-		cAnsoser.Send(fmt.Sprintf(format, args...))
+		if err := cAnsoser.Send(fmt.Sprintf(format, args...)); err != nil {
+			Errorf("failed to SOS: %v", err)
+		}
 	}
 }
 

main.go

@@ -9,7 +9,6 @@ import (
 	"local/truckstop/config"
 	"local/truckstop/logtr"
 	"local/truckstop/message"
-	"log"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -292,12 +291,13 @@ func once() error {
 		jobs[i].Secrets()
 	}
 	logtr.Infof("once: sending jobs: %+v", jobs)
+	db := config.Get().DB()
 	for i := range jobs {
 		if ok, err := sendJob(jobs[i]); err != nil {
 			return err
 		} else if ok {
 			logtr.Debugf("sent job", jobs[i])
-			if err := config.Get().DB().Set(jobs[i].ID, []byte(`sent`)); err != nil {
+			if err := db.Set(jobs[i].UID(), []byte(`sent`)); err != nil {
 				return err
 			}
 		}
@@ -333,7 +333,6 @@ func updateDeadJobs(jobs []broker.Job) error {
 	if err != nil {
 		return err
 	}
-	log.Printf("db.List() => %+v", list)
 	for _, listEntry := range list {
 		wouldBe := strings.TrimPrefix(listEntry, "sent_job_")
 		found := false
@@ -351,9 +350,14 @@ func updateDeadJobs(jobs []broker.Job) error {
 			if err := json.Unmarshal(b, &recorded); err != nil {
 				return err
 			}
+			/* // TODO this beeps on fluffychat
 			if err := message.NewMatrix().Update(recorded.MatrixID, recorded.Job.FormatMultilineTextDead()); err != nil {
 				return err
 			}
+			*/
+			if err := message.NewMatrix().Remove(recorded.MatrixID); err != nil {
+				return err
+			}
 			if err := db.Set(listEntry, nil); err != nil {
 				return err
 			}
@@ -370,7 +374,7 @@ func updateDeadJobs(jobs []broker.Job) error {
 func dropStaleJobs(jobs []broker.Job) ([]broker.Job, error) {
 	db := config.Get().DB()
 	for i := len(jobs) - 1; i >= 0; i-- {
-		if _, err := db.Get(jobs[i].ID); err == storage.ErrNotFound {
+		if _, err := db.Get(jobs[i].UID()); err == storage.ErrNotFound {
 		} else if err != nil {
 			return nil, err
 		} else {

todo.yaml

@@ -1,10 +1,11 @@
 todo:
-- mark jobs no longer avail by modifying in matrix;; save matrix ID over dummy payload
-- recv-as for clients so pa receives mas commands as writes
+- !states emits current state 
+- TEST. Just like, refactor and test to shit.
 - try search ntg by autoinc?
+- test each !command callbacks to matrix
+- recv-as for clients so pa receives mas commands as writes
 - continuation is garbo, but I can still do better client side to avoid get-set high level
 - no hard code jpeg or have it in multiple places
-- test each !command callbacks to matrix
 - change matrix so I test my custom logic even if I dont fetch remote
 - warn/err/etc. on clobbering ids.matrix since clients can mess with one another
 - modify old items once no longer available; drop stale jobs good candidate but requires new matrix interaction
@@ -14,6 +15,7 @@ todo:
   - banlist criteria like vendors, brokers, metadata
 - set up copy for caleb, broc
 done:
+- mark jobs no longer avail by modifying in matrix;; save matrix ID over dummy payload
 - TEST its falling apart
 - help() log on truckstop for stuff like perma 403
 - figure out zoom on maps;; is there like an auto-zoom I can leverage?