introduce job info slowness

.gitignore

@@ -3,3 +3,4 @@ todo-server-yaml
 cmd/cmd
 cmd/cli
 cmd/pttodo/pttodo
+/truckstop

broker/job.go

@@ -10,6 +10,7 @@ import (
 
 type Job struct {
 	ID      string
+	URI     string
 	Pickup  JobLocation
 	Dropoff JobLocation
 	Weight  int
@@ -25,12 +26,13 @@ type JobLocation struct {
 
 func (j Job) String() string {
 	return fmt.Sprintf(
-		`%s => %s (%d miles), Weight:%d, Notes:%s`,
+		`%s => %s (%d miles), Weight:%d, Notes:%s Link:%s`,
 		j.Pickup.String(),
 		j.Dropoff.String(),
 		j.Miles,
 		j.Weight,
 		j.Meta,
+		j.URI,
 	)
 }
 
@@ -41,15 +43,10 @@ func (j JobLocation) String() string {
 func (j Job) FormatMultilineText() string {
 	foo := func(client string) string {
 		return fmt.Sprintf(
-			"--- %s: %s => %s ---\nPickup: %s\nDropoff: %s\nNotes: %d lbs, %d miles, %s",
+			"--- %s: %s => %s ---",
 			client,
 			j.Pickup.State,
 			j.Dropoff.State,
-			j.Pickup.String(),
-			j.Dropoff.String(),
-			j.Weight,
-			j.Miles,
-			j.Meta,
 		)
 	}
 	out := ""
@@ -58,10 +55,22 @@ func (j Job) FormatMultilineText() string {
 		log.Printf("job multiline: %+v contains %s then use %v", clients[k].States, j.Pickup.State, k)
 		if strings.Contains(fmt.Sprint(clients[k].States), j.Pickup.State) {
 			if len(out) > 0 {
-				out += "\n\n"
+				out += "\n"
 			}
 			out += foo(k)
 		}
 	}
+	if len(out) > 0 {
+		out = fmt.Sprintf(
+			"%s\nPickup: %s\nDropoff: %s\nNotes: %d lbs, %d miles, %s\n%s",
+			out,
+			j.Pickup.String(),
+			j.Dropoff.String(),
+			j.Weight,
+			j.Miles,
+			j.Meta,
+			j.URI,
+		)
+	}
 	return out
 }

broker/ntgvision.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"io/ioutil"
 	"local/truckstop/config"
+	"log"
 	"net/http"
 	"time"
 )
@@ -32,13 +33,79 @@ type ntgVisionJob struct {
 	Weight       int    `json:"weight"`
 	Equipment    string `json:"equip"`
 	Temp         string `json:"temp"`
+	jobinfo      ntgVisionJobInfo
 }
 
-func (ntgJob ntgVisionJob) Job() Job {
+type ntgVisionJobInfo struct {
+	StopsInfo []struct {
+		StopHours       string `json:"stopHours"`
+		AppointmentTime string `json:"appointmentTime"`
+		Instructions    string `json:"instructions"`
+		IsDropTrailer   bool   `json:"isDropTrailer"`
+	} `json:"stopinfos"`
+	PayUpTo   float32 `json:"payUpTo"`
+	LoadState string  `json:"loadStatus"`
+	CanBidNow bool    `json:"canBidNow"`
+}
+
+func (ntgJob *ntgVisionJob) JobInfo() (ntgVisionJobInfo, error) {
+	if !config.Get().Brokers.NTG.JobInfo {
+		return ntgJob.jobinfo, nil
+	}
+	if fmt.Sprint(ntgJob.jobinfo) != fmt.Sprint(ntgVisionJobInfo{}) {
+		return ntgJob.jobinfo, nil
+	}
+	ji, err := ntgJob.jobInfo()
+	if err == ErrNoAuth {
+		if err := NewNTGVision().refreshAuth(); err != nil {
+			return ntgVisionJobInfo{}, err
+		}
+		ji, err = ntgJob.jobInfo()
+	}
+	ntgJob.jobinfo = ji
+	return ji, err
+}
+
+func (ntgJob *ntgVisionJob) jobInfo() (ntgVisionJobInfo, error) {
+	time.Sleep(config.Get().Interval.JobInfo.Get())
+	request, err := http.NewRequest(http.MethodGet, fmt.Sprintf(`https://ntgvision.com/api/v1/load/LoadDetails?loadId=%v`, ntgJob.ID), nil)
+	if err != nil {
+		return ntgVisionJobInfo{}, err
+	}
+	setNTGHeaders(request)
+	request.Header.Set("Authorization", "Bearer "+config.Get().Brokers.NTG.Token)
+	resp, err := do(request)
+	if err != nil {
+		return ntgVisionJobInfo{}, err
+	}
+	defer resp.Body.Close()
+	b, _ := ioutil.ReadAll(resp.Body)
+	log.Printf("fetch ntg job info %+v: %d: %s", request, resp.StatusCode, b)
+	if resp.StatusCode > 400 && resp.StatusCode < 500 && resp.StatusCode != 404 && resp.StatusCode != 410 {
+		return ntgVisionJobInfo{}, ErrNoAuth
+	}
+	var result ntgVisionJobInfo
+	err = json.Unmarshal(b, &result)
+	return result, err
+}
+
+func (ntgJob *ntgVisionJob) Job(info ...bool) Job {
+	if len(info) == 0 || !info[0] {
+		return ntgJob.job(ntgVisionJobInfo{})
+	}
+	jobInfo, err := ntgJob.JobInfo()
+	if err != nil {
+		log.Printf("failed to get jobinfo: %v", err)
+	}
+	return ntgJob.job(jobInfo)
+}
+
+func (ntgJob *ntgVisionJob) job(jobInfo ntgVisionJobInfo) Job {
 	pickup, _ := time.ParseInLocation("01/02/06", ntgJob.PickupDate, time.Local)
 	dropoff, _ := time.ParseInLocation("01/02/06", ntgJob.DropoffDate, time.Local)
 	return Job{
-		ID: fmt.Sprintf("ntg-%d", ntgJob.ID),
+		ID:  fmt.Sprintf("ntg-%d", ntgJob.ID),
+		URI: fmt.Sprintf(config.Get().Brokers.NTG.LoadPageURIFormat, ntgJob.ID),
 		Pickup: JobLocation{
 			Date:  pickup,
 			City:  ntgJob.PickupCity,
@@ -51,7 +118,7 @@ func (ntgJob ntgVisionJob) Job() Job {
 		},
 		Miles:  ntgJob.Miles,
 		Weight: ntgJob.Weight,
-		Meta:   fmt.Sprintf("equipment:%s", ntgJob.Equipment),
+		Meta:   fmt.Sprintf("equipment:%s, secrets:%+v", ntgJob.Equipment, jobInfo),
 	}
 }
 
@@ -73,12 +140,19 @@ func (ntg NTGVision) Search(states []config.State) ([]Job, error) {
 	}
 	defer rc.Close()
 
+	b, err := ioutil.ReadAll(rc)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Printf("ntg search for %+v: %s", states, b)
+
 	var ntgjobs []ntgVisionJob
-	err = json.NewDecoder(rc).Decode(&ntgjobs)
+	err = json.Unmarshal(b, &ntgjobs)
 
 	jobs := make([]Job, len(ntgjobs))
 	for i := range jobs {
-		jobs[i] = ntgjobs[i].Job()
+		jobs[i] = ntgjobs[i].Job(true)
 	}
 	return jobs, err
 }

config.json

@@ -1,8 +1,9 @@
 {
    "Interval": {
-      "Input": "15s..20s",
+      "Input": "5s..10s",
       "OK": "6h0m0s..6h0m0s",
-      "Error": "6h0m0s..6h0m0s"
+      "Error": "6h0m0s..6h0m0s",
+      "JobInfo": "60s..90s"
    },
    "Images": {
       "ClientID": "d9ac7cabe813d10",
@@ -12,23 +13,26 @@
       "RefreshURI": "https://api.imgur.com/oauth2/token",
       "RefreshFormat": "refresh_token=%s\u0026client_id=%s\u0026client_secret=%s\u0026grant_type=refresh_token",
       "RefreshMethod": "POST",
-      "UploadURI": "https://api.imgur.com/3/image",
+      "UploadURI": "https://api.imgur.com/3/image?name=something.jpeg",
       "UploadMethod": "POST"
    },
    "Maps": {
+      "DirectionsURIFormat": "https://maps.googleapis.com/maps/api/directions/json?origin=%s\u0026destination=%s\u0026mode=driving\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
+      "PathedURIFormat": "https://maps.googleapis.com/maps/api/staticmap?size=250x250\u0026path=%s\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg\u0026size=250x250\u0026markers=%s|%s\u0026zoom=5",
       "URIFormat": "https://maps.googleapis.com/maps/api/staticmap?center=%s\u0026markers=label=A|%s\u0026zoom=5\u0026size=250x250\u0026scale=1\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
-      "Pickup": true,
+      "Pathed": false,
+      "Pickup": false,
       "Dropoff": false
    },
    "Clients": {
       "bel": {
          "States": [
-            "NC"
+            "IL"
          ],
          "IDs": {
-            "Matrix": "@bel:synapse.home.blapointe.com"
+            "Matrix": "@bel:m.bltrucks.top"
          },
-         "Available": 1612328400
+         "Available": 1512328400
       },
       "broc": {
          "States": [
@@ -36,25 +40,16 @@
             "NC"
          ],
          "IDs": {
-            "Matrix": "@belandbroc:matrix.org"
+            "Matrix": "@broc:m.bltrucks.top"
          },
          "Available": 5642452800
       },
-      "caleb": {
-         "States": [
-            "FL"
-         ],
-         "IDs": {
-            "Matrix": "@belandbroc:matrix.org"
-         },
-         "Available": -62135596800
-      },
       "pa": {
          "States": [
             "NC"
          ],
          "IDs": {
-            "Matrix": "@belandbroc:matrix.org"
+            "Matrix": "@ron:m.bltrucks.top"
          },
          "Available": -62135596800
       }
@@ -67,18 +62,21 @@
       "Matrix": {
          "ReceiveEnabled": true,
          "Mock": false,
-         "Homeserver": "https://synapse.home.blapointe.com",
-         "Username": "@bel:synapse.home.blapointe.com",
-         "Token": "MDAyOGxvY2F0aW9uIHN5bmFwc2UuaG9tZS5ibGFwb2ludGUuY29tCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzMmNpZCB1c2VyX2lkID0gQGJlbDpzeW5hcHNlLmhvbWUuYmxhcG9pbnRlLmNvbQowMDE2Y2lkIHR5cGUgPSBhY2Nlc3MKMDAyMWNpZCBub25jZSA9IFpXTTtCaTNBODdtSTpsZkAKMDAyZnNpZ25hdHVyZSBX37kStY2wt_ftMfEYPfk1ADMJ6ZVfZsro9ic3weZ25Ao",
+         "Continuation": "1510",
+         "Homeserver": "https://m.bltrucks.top",
+         "Username": "@bot.m.bltrucks.top",
+         "Token": "mvDWB96KXMF8XhOam8EC5XVdQvSEw0CDeClcSWocBcYkwZX3FPNWZ5uOnQk2EmT1cjpzfeuD7gDYPPjOuyZlI3bE9TE35UjNOlZgi0Tugm25s91iVsbIF6kMZsCIhVMSmEf6w3jxX6wQYOWvmDZ4mu6f5c8wr221EMDcOpEzQV09d1zuBSWgKLBgjqAkYHJZ5dTRIWpEDpPgujhOFZa2ld1HiAOxrJKlIrlfDBN0CUsTlGOGplujDAr4VtpFzNRS",
          "Device": "TGNIOGKATZ",
-         "Room": "!WbawbZxHqnxxfhvcSj:synapse.home.blapointe.com"
+         "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
       }
    },
-   "Once": false,
+   "Once": true,
    "Brokers": {
       "NTG": {
+         "JobInfo": false,
          "Mock": true,
-         "Token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIzMTAyOSIsInVuaXF1ZV9uYW1lIjoibm9lYXN5cnVuc3RydWNraW5nQGdtYWlsLmNvbSIsImp0aSI6IjFmYzhmNjk1LTQzNTYtNGYzZS05NWY1LWZkNWVjMDJlMDkxNyIsImlhdCI6IjEvMTAvMjAyMiAxMTo1OTozNiBQTSIsIm50Z3ZSb2xlIjoiQ2FycmllckFwcHJvdmVkIiwidXNlckNhcnJpZXJzIjoiMTUzNDIzIiwib3RyVXNlciI6IkZhbHNlIiwibmJmIjoxNjQxODU5MTc2LCJleHAiOjE2NDE5NDE5NzYsImlzcyI6Ik5URyBTZWN1cml0eSBUb2tlbiBTZXJ2aWNlIiwiYXVkIjoiTlRHIn0.kpHOBTtcQhbdloAw7xEjnkzfxf4ToMgidrLCMomZmnmKQHlD_7OQuI8nQiCTHc_ntuGtt8Ui92kwWWUiLwN_2tT2vC7Jy6m9IjwqgbAzsgTLi4jAbIwITD-awiDh4FUKDwGq3XpEjs-i7XM3rI7tTk7jg9QSDId8EF3Pt5fJq6QhztC6y7-JaSFQZLMtkSCAWmOQx_TgKgVoVbgMeiqhHbZ2hhoA7TtpEIIL5Gnfq46t3E18ExdrsO96ZCGQGcBw5x8J1ustq2cwdlFKeg4ULNWAAd1ay1hojRa7jCHs98AcoJ3Nts9-o7yEMuN2rrfpK_nm68nciwFtF-ke1KoiBg",
+         "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",
+         "Token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIzMTAyOSIsInVuaXF1ZV9uYW1lIjoibm9lYXN5cnVuc3RydWNraW5nQGdtYWlsLmNvbSIsImp0aSI6IjI3OTQ2YWRkLWFhYmQtNGFhYi1iODg2LTg5MWRkMWRlZjQwNCIsImlhdCI6IjEvMTQvMjAyMiA2OjAxOjIwIEFNIiwibnRndlJvbGUiOiJDYXJyaWVyQXBwcm92ZWQiLCJ1c2VyQ2FycmllcnMiOiIxNTM0MjMiLCJvdHJVc2VyIjoiRmFsc2UiLCJuYmYiOjE2NDIxNDAwODAsImV4cCI6MTY0MjIyMjg4MCwiaXNzIjoiTlRHIFNlY3VyaXR5IFRva2VuIFNlcnZpY2UiLCJhdWQiOiJOVEcifQ.ksvyfnoqvwYaG7HtBDX6iFUaVk9cPnJpf4Jga5YDz6CGEXk85Dk10oUiSx2SA25r3lYN9by5DZIcBbEMfk69kynC28n2Te3hOR03Q0t8p3scj9aTe99fXapVKgma2s7JG_AIdkElwg81VBgyPNo3Nvn2mPKdV3ueAOkyX2aAHK4VLMm_YcbbFxiv74mPtFgw2SwnRumtgpvlOQrW0b7SXhA0s78E3kiYAjCiSS5Y7jQE1-x3P-VOPpvfXx3c8E-nHNah210Ewp2cGFvnXEevIvB0LDGeT3_HxBocwRSVU_jCVFjWX6U96u91FQAyw3yjpgkeMhX_QU_n3Nt3uXXvew",
          "Username": "noeasyrunstrucking@gmail.com",
          "Password": "thumper123"
       }

config/config.go

@@ -11,9 +11,10 @@ import (
 
 type Config struct {
 	Interval struct {
-		Input Duration
-		OK    Duration
-		Error Duration
+		Input   Duration
+		OK      Duration
+		Error   Duration
+		JobInfo Duration
 	}
 	Images struct {
 		ClientID      string
@@ -27,9 +28,12 @@ type Config struct {
 		UploadMethod  string
 	}
 	Maps struct {
-		URIFormat string
-		Pickup    bool
-		Dropoff   bool
+		DirectionsURIFormat string
+		PathedURIFormat     string
+		URIFormat           string
+		Pathed              bool
+		Pickup              bool
+		Dropoff             bool
 	}
 	Clients map[string]Client
 	Storage []string
@@ -37,6 +41,7 @@ type Config struct {
 		Matrix struct {
 			ReceiveEnabled bool
 			Mock           bool
+			Continuation   string
 			Homeserver     string
 			Username       string
 			Token          string
@@ -47,10 +52,12 @@ type Config struct {
 	Once    bool
 	Brokers struct {
 		NTG struct {
-			Mock     bool
-			Token    string
-			Username string
-			Password string
+			JobInfo           bool
+			Mock              bool
+			LoadPageURIFormat string
+			Token             string
+			Username          string
+			Password          string
 		}
 	}
 

main.go

@@ -9,6 +9,8 @@ import (
 	"local/truckstop/config"
 	"local/truckstop/message"
 	"log"
+	"net/http"
+	"net/url"
 	"regexp"
 	"sort"
 	"strings"
@@ -22,8 +24,10 @@ func main() {
 	if err := config.Refresh(); err != nil {
 		panic(err)
 	}
-	if err := matrixrecv(); err != nil {
-		panic(err)
+	if config.Get().Message.Matrix.ReceiveEnabled {
+		if err := matrixrecv(); err != nil {
+			panic(err)
+		}
 	}
 	lock := &sync.Mutex{}
 	go func() {
@@ -68,7 +72,7 @@ func matrixrecv() error {
 			if !printed {
 				if _, err := db.Get(key); err == storage.ErrNotFound {
 					log.Printf("sending help")
-					help := fmt.Sprintf("commands:\n...`!help`...print this help\n...`!state nc NC nC Nc`...set states for self\n...`!available 2022-12-31`...set date self is available for work\n\nrun a command for someone else: `!state ga @caleb`")
+					help := fmt.Sprintf("commands:\n   `!help`   print this help\n   `!state nc NC nC Nc`   set states for self\n   `!available 2022-12-31`   set date self is available for work\n\nrun a command for someone else: `!state ga @caleb`")
 					if err := sender.Send(help); err != nil {
 						log.Printf("failed to send help: %v", err)
 					} else {
@@ -87,42 +91,61 @@ func matrixrecv() error {
 	}()
 	func() {
 		log.Printf("looking for states")
+		db := config.Get().DB()
 		states := map[string]map[config.State]struct{}{}
 		for _, msg := range messages {
+			key := fmt.Sprintf("states_%d", msg.Timestamp.Unix())
 			if !strings.HasPrefix(msg.Content, "!state") {
 				continue
 			}
 			if _, ok := states[msg.Sender]; ok {
 				continue
 			}
-			states[msg.Sender] = map[config.State]struct{}{}
-			for _, state := range parseOutStates([]byte(msg.Content)) {
-				states[msg.Sender][state] = struct{}{}
+			if _, err := db.Get(key); err == storage.ErrNotFound {
+				states[msg.Sender] = map[config.State]struct{}{}
+				for _, state := range parseOutStates([]byte(msg.Content)) {
+					states[msg.Sender][state] = struct{}{}
+				}
+			}
+			if err := db.Set(key, []byte{'k'}); err != nil {
+				log.Printf("failed to mark state gathered @%s: %v", key, err)
 			}
 		}
 		setNewStates(states)
 	}()
 	func() {
 		log.Printf("looking for pauses")
+		db := config.Get().DB()
 		pauses := map[string]time.Time{}
 		for _, msg := range messages {
+			key := fmt.Sprintf("pauses_%d", msg.Timestamp.Unix())
 			if !strings.HasPrefix(msg.Content, "!available ") {
 				continue
 			}
 			if _, ok := pauses[msg.Sender]; ok {
 				continue
 			}
-			t, err := time.ParseInLocation(
-				"2006-01-02",
-				strings.TrimSpace(strings.TrimPrefix(msg.Content, "!available ")),
-				time.Local,
-			)
-			if err == nil {
-				pauses[msg.Sender] = t
+			if _, err := db.Get(key); err == storage.ErrNotFound {
+				t, err := time.ParseInLocation(
+					"2006-01-02",
+					strings.TrimSpace(strings.TrimPrefix(msg.Content, "!available ")),
+					time.Local,
+				)
+				if err == nil {
+					pauses[msg.Sender] = t
+				}
+			}
+			if err := db.Set(key, []byte{'k'}); err != nil {
+				log.Printf("failed to mark state gathered @%s: %v", key, err)
 			}
 		}
 		setNewPauses(pauses)
 	}()
+	conf := *config.Get()
+	if conf.Message.Matrix.Continuation != sender.Continuation() {
+		conf.Message.Matrix.Continuation = sender.Continuation()
+		config.Set(conf)
+	}
 	return nil
 }
 
@@ -244,21 +267,25 @@ func once() error {
 	if err != nil {
 		return err
 	}
+	log.Printf("once: all jobs: %+v", alljobs)
 	newjobs, err := dropStaleJobs(alljobs)
 	if err != nil {
 		return err
 	}
+	log.Printf("once: new jobs: %+v", newjobs)
 	jobs, err := dropBanlistJobs(newjobs)
 	if err != nil {
 		return err
 	}
+	log.Printf("once: sending jobs: %+v", jobs)
 	for i := range jobs {
-		if err := sendJob(jobs[i]); err != nil {
-			return err
-		}
-		log.Println("sent job", jobs[i])
-		if err := config.Get().DB().Set(jobs[i].ID, []byte(`sent`)); err != nil {
+		if ok, err := sendJob(jobs[i]); err != nil {
 			return err
+		} else if ok {
+			log.Println("sent job", jobs[i])
+			if err := config.Get().DB().Set(jobs[i].ID, []byte(`sent`)); err != nil {
+				return err
+			}
 		}
 	}
 	return nil
@@ -300,33 +327,73 @@ func dropBanlistJobs(jobs []broker.Job) ([]broker.Job, error) {
 	return jobs, nil
 }
 
-func sendJob(job broker.Job) error {
+func sendJob(job broker.Job) (bool, error) {
 	sender := message.NewMatrix()
 	payload := job.FormatMultilineText()
+	log.Printf("once: send job %s if nonzero: %s", job.String(), payload)
 	if len(payload) == 0 {
-		return nil
+		return false, nil
 	}
 	if err := sender.Send(payload); err != nil {
-		return err
+		return false, err
 	}
 	maps := config.Get().Maps
+	pickup := fmt.Sprintf("%s,%s", url.QueryEscape(job.Pickup.City), job.Pickup.State)
+	dropoff := fmt.Sprintf("%s,%s", url.QueryEscape(job.Dropoff.City), job.Dropoff.State)
+	if maps.Pathed {
+		directionsURI := fmt.Sprintf(maps.DirectionsURIFormat, pickup, dropoff)
+		resp, err := http.Get(directionsURI)
+		if err != nil {
+			return true, err
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			return true, fmt.Errorf("bad status getting path: %d", resp.StatusCode)
+		}
+		var directionsResp struct {
+			Routes []struct {
+				Legs []struct {
+					Steps []struct {
+						StartLocation struct {
+							Lat float32 `json:"lat"`
+							Lng float32 `json:"lng"`
+						} `json:"start_location"`
+					} `json:"steps"`
+				} `json:"legs"`
+			} `json:"routes"`
+		}
+		if err := json.NewDecoder(resp.Body).Decode(&directionsResp); err != nil {
+			return true, err
+		}
+		if len(directionsResp.Routes) < 1 || len(directionsResp.Routes[0].Legs) < 1 || len(directionsResp.Routes[0].Legs[0].Steps) < 1 {
+		} else {
+			latLngPath := make([]string, 0)
+			for _, v := range directionsResp.Routes[0].Legs[0].Steps {
+				latLngPath = append(latLngPath, fmt.Sprintf("%.9f,%.9f", v.StartLocation.Lat, v.StartLocation.Lng))
+			}
+			pathQuery := strings.Join(latLngPath, "|")
+			uri := fmt.Sprintf(maps.PathedURIFormat, pathQuery, pickup, dropoff)
+			log.Printf("sending pathed image: %s", uri)
+			if err := sender.SendImage(uri); err != nil {
+				return true, err
+			}
+		}
+	}
 	if maps.Pickup {
-		pickup := fmt.Sprintf("%s,%s", job.Pickup.City, job.Pickup.State)
 		uri := fmt.Sprintf(maps.URIFormat, pickup, pickup)
 		log.Printf("sending pickup image: %s", uri)
 		if err := sender.SendImage(uri); err != nil {
-			return err
+			return true, err
 		}
 	}
 	if maps.Dropoff {
-		dropoff := fmt.Sprintf("%s,%s", job.Dropoff.City, job.Dropoff.State)
 		uri := fmt.Sprintf(maps.URIFormat, dropoff, dropoff)
 		log.Printf("sending dropoff image: %s", uri)
 		if err := sender.SendImage(uri); err != nil {
-			return err
+			return true, err
 		}
 	}
-	return nil
+	return true, nil
 }
 
 func sendNewStates(client string, states []config.State) error {

message/images.go

@@ -7,8 +7,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"local/truckstop/config"
+	"log"
 	"mime/multipart"
 	"net/http"
+	"net/url"
 	"time"
 )
 
@@ -30,9 +32,17 @@ func refreshToken() error {
 }
 
 func uploadImage(b []byte) (string, error) {
+	images := config.Get().Images
 	buff := bytes.NewBuffer(nil)
 	writer := multipart.NewWriter(buff)
-	part, err := writer.CreateFormFile("image", "name")
+	name := "name"
+	if u, err := url.Parse(images.UploadURI); err != nil {
+	} else if s, ok := u.Query()["name"]; !ok {
+	} else {
+		name = s[0]
+		log.Printf("found name in upload uri: %s", name)
+	}
+	part, err := writer.CreateFormFile("image", name)
 	if err != nil {
 		return "", err
 	}
@@ -43,7 +53,6 @@ func uploadImage(b []byte) (string, error) {
 	}
 	writer.Close()
 
-	images := config.Get().Images
 	request, err := http.NewRequest(
 		images.UploadMethod,
 		images.UploadURI,

message/matrix.go

@@ -1,6 +1,7 @@
 package message
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"local/truckstop/config"
@@ -14,21 +15,23 @@ import (
 )
 
 type Matrix struct {
-	mock       bool
-	homeserver string
-	username   string
-	token      string
-	room       string
+	mock         bool
+	homeserver   string
+	username     string
+	token        string
+	room         string
+	continuation string
 }
 
 func NewMatrix() Matrix {
 	conf := config.Get().Message.Matrix
 	return Matrix{
-		homeserver: conf.Homeserver,
-		username:   conf.Username,
-		token:      conf.Token,
-		room:       conf.Room,
-		mock:       conf.Mock,
+		homeserver:   conf.Homeserver,
+		username:     conf.Username,
+		token:        conf.Token,
+		room:         conf.Room,
+		mock:         conf.Mock,
+		continuation: conf.Continuation,
 	}
 }
 
@@ -36,7 +39,11 @@ func (m Matrix) getclient() (*gomatrix.Client, error) {
 	return gomatrix.NewClient(m.homeserver, m.username, m.token)
 }
 
-func (m Matrix) Receive() ([]Message, error) {
+func (m Matrix) Continuation() string {
+	return m.continuation
+}
+
+func (m *Matrix) Receive() ([]Message, error) {
 	if m.mock {
 		log.Printf("matrix.Receive()")
 		messages := make([]Message, 0)
@@ -64,11 +71,14 @@ func (m Matrix) Receive() ([]Message, error) {
 		return nil, err
 	}
 	messages := make([]Message, 0)
-	result, err := c.Messages(m.room, "", "", 'b', 50)
+	result, err := c.Messages(m.room, "999999999999999999", m.continuation, 'b', 50)
 	if err != nil {
 		return nil, err
 	}
+	log.Printf("%s => {Start:%s End:%v};; %v, (%d)", m.continuation, result.Start, result.End, err, len(result.Chunk))
+	m.continuation = result.End
 	for _, event := range result.Chunk {
+		//log.Printf("%+v", event)
 		if _, ok := matrixIDs[event.Sender]; !ok {
 			continue
 		}
@@ -81,6 +91,7 @@ func (m Matrix) Receive() ([]Message, error) {
 		}
 	}
 	clientChange := regexp.MustCompile("@[a-z]+$")
+	log.Printf("rewriting messages based on @abc")
 	for i := range messages {
 		if found := clientChange.FindString(messages[i].Content); found != "" {
 			messages[i].Content = strings.TrimSpace(strings.ReplaceAll(messages[i].Content, found, ""))
@@ -94,6 +105,14 @@ func (m Matrix) Receive() ([]Message, error) {
 		}
 		messages[i].Content = strings.TrimSpace(messages[i].Content)
 	}
+	log.Printf("rewriting messages based on ! CoMmAnD ...")
+	for i := range messages {
+		if strings.HasPrefix(messages[i].Content, "!") {
+			messages[i].Content = "!" + strings.TrimSpace(messages[i].Content[1:])
+			splits := strings.Split(messages[i].Content, " ")
+			messages[i].Content = strings.ToLower(splits[0]) + " " + strings.Join(splits, " ")
+		}
+	}
 	return messages, nil
 }
 
@@ -119,19 +138,25 @@ func (m Matrix) SendImage(uri string) error {
 	if err != nil {
 		return err
 	}
+	if response.StatusCode != http.StatusOK {
+		b, _ := ioutil.ReadAll(response.Body)
+		response.Body.Close()
+		return fmt.Errorf("failed to get %s: (%d) %s", uri, response.StatusCode, b)
+	}
 	b, err := ioutil.ReadAll(response.Body)
 	response.Body.Close()
 	if err != nil {
 		return err
 	}
-	publicURI, err := UploadImage(b)
-	if err != nil {
-		return err
-	}
 	c, err := m.getclient()
 	if err != nil {
 		return err
 	}
+	mediaUpload, err := c.UploadToContentRepo(bytes.NewReader(b), "image/jpeg", int64(len(b)))
+	if err != nil {
+		return err
+	}
+	publicURI := mediaUpload.ContentURI
 	resp, err := c.SendImage(m.room, "img", publicURI)
 	log.Printf("sent image %s => %s: %+v", uri, publicURI, resp)
 	return err

message/matrix_test.go

@@ -20,7 +20,7 @@ func TestMatrixSend(t *testing.T) {
 	if err := json.Unmarshal(b, &c); err != nil {
 		t.Fatal(err)
 	}
-	var sender Sender = Matrix{
+	var sender Sender = &Matrix{
 		homeserver: c.Message.Matrix.Homeserver,
 		username:   c.Message.Matrix.Username,
 		token:      c.Message.Matrix.Token,
@@ -43,7 +43,7 @@ func TestMatrixReceive(t *testing.T) {
 	if err := json.Unmarshal(b, &c); err != nil {
 		t.Fatal(err)
 	}
-	var sender Sender = Matrix{
+	var sender Sender = &Matrix{
 		homeserver: c.Message.Matrix.Homeserver,
 		username:   c.Message.Matrix.Username,
 		token:      c.Message.Matrix.Token,

testdata/routed_map.sh

@@ -0,0 +1,3 @@
+#! /bin/bash
+
+poly="${poly:-"$(curl 'https://maps.googleapis.com/maps/api/directions/json?origin=advance,nc&destination=winston-salem,nc&mode=driving&key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg' | jq -c .routes[0].legs[0].steps[] | jq -c .start_location | jq -r '(.lat|tostring) + "," + (.lng|tostring)' | tr '\n' '|' | sed 's/.$//')"}"; curl "https://maps.googleapis.com/maps/api/staticmap?size=250x250&path=$(urlencode "$poly")&format=jpeg&maptype=roadmap&key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg&size=600x400&sensor=false"  > /tmp/f.jpg ; open /tmp/f.jpg

todo.yaml

@@ -1,4 +1,16 @@
 todo:
+- write-as for clients so ma can write to pa by default
+- continuation is garbo, but I can still do better client side to avoid get-set high level
+- todo: details from ntg; stophours for pickup, appointmenttime/facitlyhours for dest
+  details: |
+    curl 'https://ntgvision.com/api/v1/load/LoadDetails?loadId=5088453' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' -H 'Accept: application/json, text/plain, */*' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'Authorization: Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMzgyMSIsInVuaXF1ZV9uYW1lIjoiYmlyZGNvbXBhbnlsb2dpc3RpY3NAZ21haWwuY29tIiwianRpIjoiYTFiMjkxM2YtN2RjZS00ZDQ0LWEzMDYtMTRjMTBlYjQ2MmE1IiwiaWF0IjoiMS8xMy8yMDIyIDI6MTE6NTUgUE0iLCJudGd2Um9sZSI6IkNhcnJpZXJBcHByb3ZlZCIsImxvY2tVc2VyIjoiRmFsc2UiLCJvdmVycmlkZUJsYWNrbGlzdCI6IkZhbHNlIiwic2hvd1JhdGVzIjoiRmFsc2UiLCJ1c2VyQ2FycmllcnMiOiIxMTQxOTMiLCJvdHJVc2VyIjoiRmFsc2UiLCJuYmYiOjE2NDIwODMxMTUsImV4cCI6MTY0MjE2NTkxNSwiaXNzIjoiTlRHIFNlY3VyaXR5IFRva2VuIFNlcnZpY2UiLCJhdWQiOiJOVEcifQ.eDE2Jfok79b7oD1deP_00h9g8zZ3AnY60McnJ1hfS-8YJ12L3Bl-CJeN0eBos0aj2FXHv5MrqOJ6MFCaatqjIddPJl2MvtSByGUCLBzih67O20mkCowCmZfGjSnKvUu7DHFxjzo_y4_a3cjjGPYNc2LSCVJxV9NYzSXIuXG3WvXeE3dv8ml23bJFFhMWZzhSAfWBOW4kR61sibS6BpRD8dpkKoqUfXInq_7o8jz9_PsEhPBdJydqbXwg8ifQHkaPNAojsIr2rnJ3Tf_iQpom-6oAEUAOd2D3fK1XuWtRrysnD8s41YDthmqKpni2WOk72L-sKzRxD4KX2t_gyEb3Fw' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Cookie: cookiesession1=678A3E1398901234BCDEFGHIJKLMA492; _uiq_id.711119701.3d83=516d3e744ebe2d9a.1641792415.0.1642083653..; NTGAuthToken=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMzgyMSIsInVuaXF1ZV9uYW1lIjoiYmlyZGNvbXBhbnlsb2dpc3RpY3NAZ21haWwuY29tIiwianRpIjoiYTFiMjkxM2YtN2RjZS00ZDQ0LWEzMDYtMTRjMTBlYjQ2MmE1IiwiaWF0IjoiMS8xMy8yMDIyIDI6MTE6NTUgUE0iLCJudGd2Um9sZSI6IkNhcnJpZXJBcHByb3ZlZCIsImxvY2tVc2VyIjoiRmFsc2UiLCJvdmVycmlkZUJsYWNrbGlzdCI6IkZhbHNlIiwic2hvd1JhdGVzIjoiRmFsc2UiLCJ1c2VyQ2FycmllcnMiOiIxMTQxOTMiLCJvdHJVc2VyIjoiRmFsc2UiLCJuYmYiOjE2NDIwODMxMTUsImV4cCI6MTY0MjE2NTkxNSwiaXNzIjoiTlRHIFNlY3VyaXR5IFRva2VuIFNlcnZpY2UiLCJhdWQiOiJOVEcifQ.eDE2Jfok79b7oD1deP_00h9g8zZ3AnY60McnJ1hfS-8YJ12L3Bl-CJeN0eBos0aj2FXHv5MrqOJ6MFCaatqjIddPJl2MvtSByGUCLBzih67O20mkCowCmZfGjSnKvUu7DHFxjzo_y4_a3cjjGPYNc2LSCVJxV9NYzSXIuXG3WvXeE3dv8ml23bJFFhMWZzhSAfWBOW4kR61sibS6BpRD8dpkKoqUfXInq_7o8jz9_PsEhPBdJydqbXwg8ifQHkaPNAojsIr2rnJ3Tf_iQpom-6oAEUAOd2D3fK1XuWtRrysnD8s41YDthmqKpni2WOk72L-sKzRxD4KX2t_gyEb3Fw' -H 'Sec-Fetch-Dest: empty' -H 'Sec-Fetch-Mode: cors' -H 'Sec-Fetch-Site: same-origin' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache'
+    {"headerSummary":"Washington, NC to Abilene, TX","loadId":5088453,"miles":1481,"weight":6000,"temp":null,"equipment":"Str Truck W/ Lift Gate","categoryName":"Lighting / Lighting Fixtures","categoryLabel":"Product Category","cargoInformation":["Store Fixtures - 6000 lbs"],"loadRequirements":["Straight Truck with lift gate required"],"stopInfos":[{"loadInfoIds":[5431607],"stopDateTime":"0001-01-01T00:00:00","isPickup":true,"location":"Washington, NC","stopDate":"Friday, 01/14/22","stopHours":"10:00 - 12:00","appointmentTime":"","appointmentType":"FCFS","instructions":"***LIFTGATE, PALLET JACK, DRIVER ASSIST***--MUST DELIVER ONLY ON APPT TIME-- CANNOT DELIVER EARLY OR LATE \r\n","isDropTrailer":false,"shipperName":"IDX NORTH CAROLINA","processedStopDate":""},{"loadInfoIds":[5431607],"stopDateTime":"0001-01-01T00:00:00","isPickup":false,"location":"Abilene, TX","stopDate":"Monday, 01/17/22","stopHours":"09:00 - 10:00","appointmentTime":"09:00","appointmentType":"APPT","instructions":"***LIFTGATE, PALLET JACK, DRIVER ASSIST***--MUST DELIVER ONLY ON APPT TIME-- CANNOT DELIVER EARLY OR LATE \r\n","isDropTrailer":false,"shipperName":"hibbett sports","processedStopDate":""}],"drayStopInfos":null,"rateInfo":null,"isDray":false,"equipmentGroupId":8,"totalCarrierRate":2600.00,"payUpTo":2700.00,"firstLoadInfoId":5431607,"loadStatus":"ACTIVE","hasBlockingAlert":false,"customerLoadBlocked":false,"canSeeRate":false,"canBookNow":false,"canBidNow":true,"buttonData":{"useBidDialog":false,"lastBidAmount":null,"remainingBids":1,"carrierPhoneNumber":null,"carrierPhoneExtension":null},"stopData":[{"addr":"IDX NORTH CAROLINA","city":"Washington","state":"NC","zip":"27889"},{"addr":"hibbett sports","city":"Abilene","state":"TX","zip":"79606"}]}
+
+- no hard code jpeg or have it in multiple places
+- todo: switch house to selfhosted for rate limit
+  tags: now
+- mark consumed;; save scroll id for matrix
+- TEST its falling apart
 - test each !command callbacks to matrix
 - change matrix so I test my custom logic even if I dont fetch remote
 - warn/err/etc. on clobbering ids.matrix since clients can mess with one another
@@ -9,6 +21,10 @@ todo:
   - banlist criteria like vendors, brokers, metadata
 - set up copy for caleb, broc
 done:
+- link to view more
+- map with to-from line
+- TO CONFLUENT.RS OR W/E
+- rm get-set stuff for matrix now that it has continuation
 - setup ma on element !!fluffychat
 - !help,
 - !states optional but explicit option