origin from date lower

.gitignore

@@ -3,3 +3,5 @@ todo-server-yaml
 cmd/cmd
 cmd/cli
 cmd/pttodo/pttodo
+/truckstop
+/exec-truckstop

broker/broker.go

@@ -1,7 +1,28 @@
 package broker
 
-import "local/truckstop/config"
+import (
+	"context"
+	"local/truckstop/config"
+	"net/http"
+	"strings"
+
+	"golang.org/x/time/rate"
+)
+
+// once per minute
+var authlimiter = rate.NewLimiter(rate.Limit(1.0/60.0), 1)
+
+// thrice per minute
+var limiter = rate.NewLimiter(rate.Limit(1.0/20.0), 1)
 
 type Broker interface {
 	Search([]config.State) ([]Job, error)
 }
+
+func do(r *http.Request) (*http.Response, error) {
+	limiter.Wait(context.Background())
+	if strings.Contains(strings.ToLower(r.URL.Path), "login") {
+		authlimiter.Wait(context.Background())
+	}
+	return http.DefaultClient.Do(r)
+}

broker/job.go

@@ -1,18 +1,24 @@
 package broker
 
 import (
+	"encoding/base64"
 	"fmt"
 	"local/truckstop/config"
+	"local/truckstop/logtr"
+	"strings"
 	"time"
 )
 
 type Job struct {
 	ID      string
+	URI     string
 	Pickup  JobLocation
 	Dropoff JobLocation
 	Weight  int
 	Miles   int
 	Meta    string
+	Pays    string
+	secrets func(j *Job) `json:"-"`
 }
 
 type JobLocation struct {
@@ -21,14 +27,34 @@ type JobLocation struct {
 	State string
 }
 
+func (j Job) UID() string {
+	return fmt.Sprintf(
+		"%v-%s-%s-%s-%s-%v",
+		j.ID,
+		j.Pickup.State,
+		base64.StdEncoding.EncodeToString([]byte(j.Pickup.City)),
+		j.Dropoff.State,
+		base64.StdEncoding.EncodeToString([]byte(j.Dropoff.City)),
+		j.Pickup.Date.Unix(),
+	)
+}
+
+func (j *Job) Secrets() {
+	if j.secrets == nil {
+		return
+	}
+	j.secrets(j)
+}
+
 func (j Job) String() string {
 	return fmt.Sprintf(
-		`%s => %s (%d miles), Weight:%d, Notes:%s`,
+		`%s => %s (%d miles), Weight:%d, Notes:%s Link:%s`,
 		j.Pickup.String(),
 		j.Dropoff.String(),
 		j.Miles,
 		j.Weight,
 		j.Meta,
+		j.URI,
 	)
 }
 
@@ -36,16 +62,50 @@ func (j JobLocation) String() string {
 	return fmt.Sprintf("%s, %s @ %s", j.City, j.State, j.Date.Format("Monday Jan 02"))
 }
 
-func (j Job) FormatMultilineText() string {
+func (j Job) FormatMultilineTextDead() string {
 	return fmt.Sprintf(
-		"--- %s: %s => %s ---\nPickup: %s\nDropoff: %s\nNotes: %d lbs, %d miles, %s",
+		"no longer available: %s,%s => %s,%s for $%v @%s",
-		config.Get().Name,
+		j.Pickup.City,
 		j.Pickup.State,
+		j.Dropoff.City,
 		j.Dropoff.State,
-		j.Pickup.String(),
+		j.Pays,
-		j.Dropoff.String(),
+		j.URI,
-		j.Weight,
-		j.Miles,
-		j.Meta,
 	)
+
+}
+
+func (j Job) FormatMultilineText() string {
+	foo := func(client string) string {
+		return fmt.Sprintf(
+			"--- %s: %s => %s ---",
+			client,
+			j.Pickup.State,
+			j.Dropoff.State,
+		)
+	}
+	out := ""
+	clients := config.Clients(j.Pickup.Date)
+	for k := range clients {
+		logtr.Debugf("job multiline: %+v contains %s then use %v", clients[k].States, j.Pickup.State, k)
+		if strings.Contains(fmt.Sprint(clients[k].States), j.Pickup.State) {
+			if len(out) > 0 {
+				out += "\n"
+			}
+			out += foo(k)
+		}
+	}
+	if len(out) > 0 {
+		out = fmt.Sprintf(
+			"%s\nPickup: %s\nDropoff: %s\nNotes: %d lbs, %d miles, %s\n%s",
+			out,
+			j.Pickup.String(),
+			j.Dropoff.String(),
+			j.Weight,
+			j.Miles,
+			j.Meta,
+			j.URI,
+		)
+	}
+	return out
 }

broker/ntgvision.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"io/ioutil"
 	"local/truckstop/config"
+	"local/truckstop/logtr"
 	"net/http"
 	"time"
 )
@@ -15,6 +16,7 @@ import (
 type NTGVision struct {
 	searcher interface {
 		search(states []config.State) (io.ReadCloser, error)
+		searchJob(id int64) (io.ReadCloser, error)
 	}
 }
 
@@ -32,13 +34,102 @@ type ntgVisionJob struct {
 	Weight       int    `json:"weight"`
 	Equipment    string `json:"equip"`
 	Temp         string `json:"temp"`
+	jobinfo      ntgVisionJobInfo
 }
 
-func (ntgJob ntgVisionJob) Job() Job {
+type ntgVisionJobInfo struct {
+	StopsInfo []struct {
+		StopHours       string `json:"stopHours"`
+		AppointmentTime string `json:"appointmentTime"`
+		Instructions    string `json:"instructions"`
+		IsDropTrailer   bool   `json:"isDropTrailer"`
+	} `json:"stopinfos"`
+	PayUpTo          float32 `json:"payUpTo"`
+	TotalCarrierRate float32 `json:"totalCarrierRate"`
+	LoadState        string  `json:"loadStatus"`
+	//CanBidNow bool    `json:"canBidNow"`
+}
+
+func (ji ntgVisionJobInfo) IsZero() bool {
+	return len(ji.StopsInfo) == 0 && ji.TotalCarrierRate == 0 && ji.PayUpTo == 0 && ji.LoadState == ""
+}
+
+func (ji ntgVisionJobInfo) String() string {
+	if ji.IsZero() {
+		return ""
+	}
+	out := ""
+	if ji.PayUpTo != 0 {
+		out = fmt.Sprintf("\nPAYS:%v\n%s", ji.PayUpTo, out)
+	}
+	if ji.TotalCarrierRate != 0 {
+		out = fmt.Sprintf("%s Total_Carrier_Rate:%v", out, ji.TotalCarrierRate)
+	}
+	if ji.LoadState != "" {
+		out = fmt.Sprintf("%s Auction:%s", out, ji.LoadState)
+	}
+	if len(ji.StopsInfo) != 2 {
+		return out
+	}
+	return fmt.Sprintf(
+		"%s Pickup:{Hours:%s Notes:%s, DropTrailer:%v} Dropoff:{Appointment:%s Notes:%s}",
+		out,
+		ji.StopsInfo[0].StopHours,
+		ji.StopsInfo[0].Instructions,
+		ji.StopsInfo[0].IsDropTrailer,
+		ji.StopsInfo[1].AppointmentTime,
+		ji.StopsInfo[1].Instructions,
+	)
+}
+
+func (ntgJob *ntgVisionJob) JobInfo() (ntgVisionJobInfo, error) {
+	if !config.Get().Brokers.NTG.JobInfo {
+		return ntgJob.jobinfo, nil
+	}
+	if !ntgJob.jobinfo.IsZero() {
+		return ntgJob.jobinfo, nil
+	}
+	db := config.Get().DB()
+	key := fmt.Sprintf("ntg_job_info_%v", ntgJob.ID)
+	if b, err := db.Get(key); err != nil {
+	} else if err := json.Unmarshal(b, &ntgJob.jobinfo); err == nil {
+		return ntgJob.jobinfo, nil
+	}
+	ntg := NewNTGVision()
+	ji, err := ntg.SearchJob(ntgJob.ID)
+	if err == nil {
+		ntgJob.jobinfo = ji
+		b, err := json.Marshal(ntgJob.jobinfo)
+		if err == nil {
+			db.Set(key, b)
+		}
+	}
+	return ji, err
+}
+
+func (ntg NTGVision) searchJob(id int64) (io.ReadCloser, error) {
+	time.Sleep(config.Get().Interval.JobInfo.Get())
+	request, err := http.NewRequest(http.MethodGet, fmt.Sprintf(config.Get().Brokers.NTG.LoadPageAPIURIFormat, id), nil)
+	if err != nil {
+		return nil, err
+	}
+	setNTGHeaders(request)
+	resp, err := do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	b, _ := ioutil.ReadAll(resp.Body)
+	logtr.Debugf("fetch ntg job info %+v: %d: %s", request, resp.StatusCode, b)
+	return io.NopCloser(bytes.NewReader(b)), nil
+}
+
+func (ntgJob *ntgVisionJob) Job() Job {
 	pickup, _ := time.ParseInLocation("01/02/06", ntgJob.PickupDate, time.Local)
 	dropoff, _ := time.ParseInLocation("01/02/06", ntgJob.DropoffDate, time.Local)
 	return Job{
-		ID: fmt.Sprintf("ntg-%d", ntgJob.ID),
+		ID:  fmt.Sprintf("ntg-%d", ntgJob.ID),
+		URI: fmt.Sprintf(config.Get().Brokers.NTG.LoadPageURIFormat, ntgJob.ID),
 		Pickup: JobLocation{
 			Date:  pickup,
 			City:  ntgJob.PickupCity,
@@ -52,12 +143,24 @@ func (ntgJob ntgVisionJob) Job() Job {
 		Miles:  ntgJob.Miles,
 		Weight: ntgJob.Weight,
 		Meta:   fmt.Sprintf("equipment:%s", ntgJob.Equipment),
+		secrets: func(j *Job) {
+			jobInfo, err := ntgJob.JobInfo()
+			if err != nil {
+				logtr.Errorf("failed to get jobinfo: %v", err)
+				return
+			}
+			j.Meta = jobInfo.String()
+			j.Pays = fmt.Sprint(jobInfo.PayUpTo)
+		},
 	}
 }
 
 func NewNTGVision() NTGVision {
 	ntgv := NTGVision{}
 	ntgv.searcher = ntgv
+	if config.Get().Brokers.NTG.Mock {
+		ntgv = ntgv.WithMock()
+	}
 	return ntgv
 }
 
@@ -66,6 +169,24 @@ func (ntg NTGVision) WithMock() NTGVision {
 	return ntg
 }
 
+func (ntg NTGVision) SearchJob(id int64) (ntgVisionJobInfo, error) {
+	rc, err := ntg.searcher.searchJob(id)
+	if err != nil {
+		return ntgVisionJobInfo{}, err
+	}
+	defer rc.Close()
+	b, err := ioutil.ReadAll(rc)
+	if err != nil {
+		return ntgVisionJobInfo{}, fmt.Errorf("failed to readall search job result: %w", err)
+	}
+	var result ntgVisionJobInfo
+	err = json.Unmarshal(b, &result)
+	if err != nil {
+		err = fmt.Errorf("failed to parse job info: %w: %s", err, b)
+	}
+	return result, err
+}
+
 func (ntg NTGVision) Search(states []config.State) ([]Job, error) {
 	rc, err := ntg.searcher.search(states)
 	if err != nil {
@@ -73,8 +194,15 @@ func (ntg NTGVision) Search(states []config.State) ([]Job, error) {
 	}
 	defer rc.Close()
 
+	b, err := ioutil.ReadAll(rc)
+	if err != nil {
+		return nil, err
+	}
+
+	logtr.Debugf("ntg search for %+v: %s", states, b)
+
 	var ntgjobs []ntgVisionJob
-	err = json.NewDecoder(rc).Decode(&ntgjobs)
+	err = json.Unmarshal(b, &ntgjobs)
 
 	jobs := make([]Job, len(ntgjobs))
 	for i := range jobs {
@@ -83,8 +211,23 @@ func (ntg NTGVision) Search(states []config.State) ([]Job, error) {
 	return jobs, err
 }
 
+func getNTGTokenKey() string {
+	return "brokers_ntg_token"
+}
+
+func getNTGToken() string {
+	db := config.Get().DB()
+	b, _ := db.Get(getNTGTokenKey())
+	return string(b)
+}
+
+func setNTGToken(token string) {
+	db := config.Get().DB()
+	db.Set(getNTGTokenKey(), []byte(token))
+}
+
 func (ntg NTGVision) search(states []config.State) (io.ReadCloser, error) {
-	if config.Get().Brokers.NTG.Token == "" {
+	if getNTGToken() == "" {
 		if err := ntg.refreshAuth(); err != nil {
 			return nil, err
 		}
@@ -100,7 +243,15 @@ func (ntg NTGVision) search(states []config.State) (io.ReadCloser, error) {
 }
 
 func (ntg NTGVision) refreshAuth() error {
-	time.Sleep(time.Minute * 2) // TODO
+	err := ntg._refreshAuth()
+	if err != nil {
+		logtr.SOSf("failed to refresh ntg auth: %v", err)
+	}
+	return err
+}
+
+func (ntg NTGVision) _refreshAuth() error {
+	logtr.Infof("refreshing ntg auth...")
 	b, _ := json.Marshal(map[string]string{
 		"username": config.Get().Brokers.NTG.Username,
 		"password": config.Get().Brokers.NTG.Password,
@@ -110,7 +261,7 @@ func (ntg NTGVision) refreshAuth() error {
 		return err
 	}
 	setNTGHeaders(request)
-	resp, err := http.DefaultClient.Do(request)
+	resp, err := do(request)
 	if err != nil {
 		return err
 	}
@@ -128,9 +279,7 @@ func (ntg NTGVision) refreshAuth() error {
 	if len(v.Token) == 0 {
 		return errors.New("failed to get token from login call")
 	}
-	conf := config.Get()
+	setNTGToken(v.Token)
-	conf.Brokers.NTG.Token = v.Token
-	config.Set(*conf)
 	return nil
 }
 
@@ -139,8 +288,7 @@ func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
 	if err != nil {
 		return nil, err
 	}
-	c := http.Client{Timeout: time.Minute}
+	resp, err := do(request)
-	resp, err := c.Do(request)
 	if err != nil {
 		return nil, err
 	}
@@ -157,7 +305,7 @@ func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
 
 func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) {
 	body, err := json.Marshal(map[string]interface{}{
-		"OriginFromDate":        time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
+		"OriginFromDate":        time.Now().Add(time.Hour * -24).UTC().Format("2006-01-02T15:04:05.000Z"),
 		"OriginToDate":          time.Now().UTC().Add(time.Hour * 24 * 30).Format("2006-01-02T15:04:05.000Z"),
 		"DestinationFromDate":   nil,
 		"DestinationToDate":     nil,
@@ -180,7 +328,7 @@ func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) {
 		return nil, err
 	}
 	setNTGHeaders(request)
-	request.Header.Set("Authorization", "Bearer "+config.Get().Brokers.NTG.Token)
+	request.Header.Set("Authorization", "Bearer "+getNTGToken())
 
 	return request, nil
 }
@@ -191,7 +339,7 @@ func setNTGHeaders(request *http.Request) {
 	request.Header.Set("Accept-Language", "en-US,en;q=0.5")
 	request.Header.Set("Accept-Encoding", "gzip, deflate, br")
 	request.Header.Set("Content-Type", "application/json;charset=utf-8")
-	//request.Header.Set("Authorization", "Bearer "+config.Get().Brokers.NTG.Token)
+	//request.Header.Set("Authorization", "Bearer "+getNTGToken())
 	request.Header.Set("Origin", "https://ntgvision.com")
 	request.Header.Set("DNT", "1")
 	request.Header.Set("Connection", "keep-alive")

broker/ntgvision_mock.go

@@ -20,3 +20,9 @@ func (ntgm NTGVisionMock) search(states []config.State) (io.ReadCloser, error) {
 	b, err := ioutil.ReadFile(path)
 	return io.NopCloser(bytes.NewReader(b)), err
 }
+
+func (ntgm NTGVisionMock) searchJob(id int64) (io.ReadCloser, error) {
+	path := path.Join(os.Getenv("GOPATH"), "src", "local", "truckstop", "broker", "testdata", "ntgvision_jobinfo_response.json")
+	b, err := ioutil.ReadFile(path)
+	return io.NopCloser(bytes.NewReader(b)), err
+}

broker/testdata/ntgvision_jobinfo_response.json

@@ -0,0 +1,84 @@
+{
+  "headerSummary": "A to B",
+  "loadId": 123,
+  "miles": 123,
+  "weight": 123,
+  "temp": null,
+  "equipment": "equipment",
+  "categoryName": "category",
+  "categoryLabel": "category generic",
+  "cargoInformation": [
+    "carto info"
+  ],
+  "loadRequirements": [
+    "load requirements"
+  ],
+  "stopInfos": [
+    {
+      "loadInfoIds": [
+        123
+      ],
+      "stopDateTime": "0001-01-01T00:00:00",
+      "isPickup": true,
+      "location": "A",
+      "stopDate": "Friday, 01/14/22",
+      "stopHours": "10:00 - 12:00",
+      "appointmentTime": "",
+      "appointmentType": "FCFS",
+      "instructions": "instruction",
+      "isDropTrailer": false,
+      "shipperName": "client",
+      "processedStopDate": ""
+    },
+    {
+      "loadInfoIds": [
+        123
+      ],
+      "stopDateTime": "0001-01-01T00:00:00",
+      "isPickup": false,
+      "location": "B",
+      "stopDate": "Monday, 01/17/22",
+      "stopHours": "09:00 - 10:00",
+      "appointmentTime": "09:00",
+      "appointmentType": "APPT",
+      "instructions": "instructions",
+      "isDropTrailer": false,
+      "shipperName": "client",
+      "processedStopDate": ""
+    }
+  ],
+  "drayStopInfos": null,
+  "rateInfo": null,
+  "isDray": false,
+  "equipmentGroupId": 8,
+  "totalCarrierRate": 2600,
+  "payUpTo": 2700,
+  "firstLoadInfoId": 123,
+  "loadStatus": "ACTIVE",
+  "hasBlockingAlert": false,
+  "customerLoadBlocked": false,
+  "canSeeRate": false,
+  "canBookNow": false,
+  "canBidNow": true,
+  "buttonData": {
+    "useBidDialog": false,
+    "lastBidAmount": null,
+    "remainingBids": 1,
+    "carrierPhoneNumber": null,
+    "carrierPhoneExtension": null
+  },
+  "stopData": [
+    {
+      "addr": "IDX NORTH CAROLINA",
+      "city": "Washington",
+      "state": "NC",
+      "zip": "27889"
+    },
+    {
+      "addr": "hibbett sports",
+      "city": "Abilene",
+      "state": "TX",
+      "zip": "79606"
+    }
+  ]
+}

broker/testdata/ntgvision_response.json

@@ -1,30 +1,13 @@
 [
-  {
-    "id": 4650337,
-    "sDate": "01/12/22",
-    "sCity": "Columbus",
-    "sState": "OH",
-    "sdh": null,
-    "cDate": "01/13/22",
-    "cCity": "Jamaica",
-    "cState": "NY",
-    "cdh": null,
-    "stopCnt": 2,
-    "miles": 578,
-    "weight": 5000,
-    "equip": "Str Truck W/ Lift Gate",
-    "temp": "",
-    "alertReasons": []
-  },
   {
     "id": 4650338,
     "sDate": "01/12/22",
-    "sCity": "Somewhere",
+    "sCity": "Advance",
-    "sState": "OH",
+    "sState": "NC",
     "sdh": null,
     "cDate": "01/13/22",
-    "cCity": "SomewhereElse",
+    "cCity": "Winston-Salem",
-    "cState": "GA",
+    "cState": "NC",
     "cdh": null,
     "stopCnt": 2,
     "miles": 378,

config.json

@@ -1,47 +1,83 @@
 {
-   "Name": "pa",
+   "Log": {
-   "Interval": {
+      "Path": "/tmp/truckstop.log",
-      "Email": "15m0s..15m0s",
+      "Level": "debug",
-      "OK": "6h0m0s..6h0m0s",
+      "SOSMatrix": {
-      "Error": "6h0m0s..6h0m0s"
+         "ReceiveEnabled": true,
+         "Mock": false,
+         "Homeserver": "https://m.bltrucks.top",
+         "Username": "@bot.m.bltrucks.top",
+         "Token": "mvDWB96KXMF8XhOam8EC5XVdQvSEw0CDeClcSWocBcYkwZX3FPNWZ5uOnQk2EmT1cjpzfeuD7gDYPPjOuyZlI3bE9TE35UjNOlZgi0Tugm25s91iVsbIF6kMZsCIhVMSmEf6w3jxX6wQYOWvmDZ4mu6f5c8wr221EMDcOpEzQV09d1zuBSWgKLBgjqAkYHJZ5dTRIWpEDpPgujhOFZa2ld1HiAOxrJKlIrlfDBN0CUsTlGOGplujDAr4VtpFzNRS",
+         "Device": "TGNIOGKATZ",
+         "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
+      }
+   },
+   "Interval": {
+      "Input": "5s..10s",
+      "OK": "6h0m0s..6h0m0s",
+      "Error": "6h0m0s..6h0m0s",
+      "JobInfo": "10s..20s"
+   },
+   "Images": {
+      "ClientID": "d9ac7cabe813d10",
+      "ClientSecret": "9d0b3d82800b30ca88f595d3bcd6985f627d7d82",
+      "RefreshToken": "171417741bf762b99b0b9f9137491b7a69874a77",
+      "AccessToken": "e63db98f92d2db7ac7f56914a2030c889b378e9b",
+      "RefreshURI": "https://api.imgur.com/oauth2/token",
+      "RefreshFormat": "refresh_token=%s\u0026client_id=%s\u0026client_secret=%s\u0026grant_type=refresh_token",
+      "RefreshMethod": "POST",
+      "UploadURI": "https://api.imgur.com/3/image?name=something.jpeg",
+      "UploadMethod": "POST"
+   },
+   "Maps": {
+      "URIFormat": "https://maps.googleapis.com/maps/api/staticmap?center=%s\u0026markers=label=A|%s\u0026zoom=5\u0026size=250x250\u0026scale=1\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
+      "Pickup": false,
+      "Dropoff": false,
+      "Pathed": {
+         "Enabled": true,
+         "DirectionsURIFormat": "https://maps.googleapis.com/maps/api/directions/json?origin=%s\u0026destination=%s\u0026mode=driving\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
+         "PathedURIFormat": "https://maps.googleapis.com/maps/api/staticmap?size=250x250\u0026path=%s\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg\u0026size=250x250\u0026markers=%s|%s",
+         "Zoom": {
+            "AcceptableLatLngDelta": 2,
+            "Override": 5
+         }
+      }
+   },
+   "Clients": {
+      "bel": {
+         "States": [
+            "NC"
+         ],
+         "IDs": {
+            "Matrix": "@bel:m.bltrucks.top"
+         },
+         "Available": 1512328400
+      }
    },
-   "States": [
-      "FL",
-      "GA",
-      "NC"
-   ],
    "Storage": [
-      "map"
+      "files",
+      "/tmp/play"
    ],
-   "Client": "breellocaldev@gmail.com",
    "Message": {
       "Matrix": {
          "ReceiveEnabled": true,
-         "Client": "@belandbroc:matrix.org",
+         "Mock": false,
-         "Mock": true,
+         "Homeserver": "https://m.bltrucks.top",
-         "Homeserver": "https://matrix-client.matrix.org",
+         "Username": "@bot.m.bltrucks.top",
-         "Username": "@breellocaldev:matrix.org",
+         "Token": "mvDWB96KXMF8XhOam8EC5XVdQvSEw0CDeClcSWocBcYkwZX3FPNWZ5uOnQk2EmT1cjpzfeuD7gDYPPjOuyZlI3bE9TE35UjNOlZgi0Tugm25s91iVsbIF6kMZsCIhVMSmEf6w3jxX6wQYOWvmDZ4mu6f5c8wr221EMDcOpEzQV09d1zuBSWgKLBgjqAkYHJZ5dTRIWpEDpPgujhOFZa2ld1HiAOxrJKlIrlfDBN0CUsTlGOGplujDAr4VtpFzNRS",
-         "Token": "syt_YnJlZWxsb2NhbGRldg_HTewKMMePdEcLvceAKEz_2fHsHa",
          "Device": "TGNIOGKATZ",
-         "Room": "!rMvyKroCAJMRqFwTwC:matrix.org"
+         "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
       }
    },
    "Once": true,
    "Brokers": {
       "NTG": {
+         "JobInfo": true,
          "Mock": true,
-         "Token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIzMTAyOSIsInVuaXF1ZV9uYW1lIjoibm9lYXN5cnVuc3RydWNraW5nQGdtYWlsLmNvbSIsImp0aSI6IjFmYzhmNjk1LTQzNTYtNGYzZS05NWY1LWZkNWVjMDJlMDkxNyIsImlhdCI6IjEvMTAvMjAyMiAxMTo1OTozNiBQTSIsIm50Z3ZSb2xlIjoiQ2FycmllckFwcHJvdmVkIiwidXNlckNhcnJpZXJzIjoiMTUzNDIzIiwib3RyVXNlciI6IkZhbHNlIiwibmJmIjoxNjQxODU5MTc2LCJleHAiOjE2NDE5NDE5NzYsImlzcyI6Ik5URyBTZWN1cml0eSBUb2tlbiBTZXJ2aWNlIiwiYXVkIjoiTlRHIn0.kpHOBTtcQhbdloAw7xEjnkzfxf4ToMgidrLCMomZmnmKQHlD_7OQuI8nQiCTHc_ntuGtt8Ui92kwWWUiLwN_2tT2vC7Jy6m9IjwqgbAzsgTLi4jAbIwITD-awiDh4FUKDwGq3XpEjs-i7XM3rI7tTk7jg9QSDId8EF3Pt5fJq6QhztC6y7-JaSFQZLMtkSCAWmOQx_TgKgVoVbgMeiqhHbZ2hhoA7TtpEIIL5Gnfq46t3E18ExdrsO96ZCGQGcBw5x8J1ustq2cwdlFKeg4ULNWAAd1ay1hojRa7jCHs98AcoJ3Nts9-o7yEMuN2rrfpK_nm68nciwFtF-ke1KoiBg",
+         "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",
+         "LoadPageAPIURIFormat": "https://ntgvision.com/api/v1/load/LoadDetails?loadId==%d",
          "Username": "noeasyrunstrucking@gmail.com",
-         "Password": "thumper123"
+         "Password": "thumper1234"
       }
-   },
+   }
-   "Emailer": {
-      "From": "breellocaldev@gmail.com",
-      "SMTP": "",
-      "POP3": "",
-      "IMAP": "imap.gmail.com:993",
-      "Password": "gojfkkfrkmtxzyro",
-      "Limit": 0
-   },
-   "EmailerEnabled": false
 }

config/config.go

@@ -3,50 +3,89 @@ package config
 import (
 	"encoding/json"
 	"io/ioutil"
-	"local/sandbox/contact/contact"
 	"local/storage"
+	"local/truckstop/logtr"
 	"os"
 	"sync"
+	"time"
 )
 
+type Matrix struct {
+	ReceiveEnabled bool
+	Mock           bool
+	Homeserver     string
+	Username       string
+	Token          string
+	Device         string
+	Room           string
+}
+
 type Config struct {
-	Name     string
+	Log struct {
-	Interval struct {
+		Path      string
-		Email Duration
+		Level     logtr.Level
-		OK    Duration
+		SOSMatrix Matrix
-		Error Duration
 	}
-	States  []State
+	Interval struct {
-	Storage []string
+		Input   Duration
-	Client  string
+		OK      Duration
-	Message struct {
+		Error   Duration
-		Matrix struct {
+		JobInfo Duration
-			ReceiveEnabled bool
+	}
-			Client         string
+	Images struct {
-			Mock           bool
+		ClientID      string
-			Homeserver     string
+		ClientSecret  string
-			Username       string
+		RefreshToken  string
-			Token          string
+		AccessToken   string
-			Device         string
+		RefreshURI    string
-			Room           string
+		RefreshFormat string
+		RefreshMethod string
+		UploadURI     string
+		UploadMethod  string
+	}
+	Maps struct {
+		URIFormat string
+		Pickup    bool
+		Dropoff   bool
+		Pathed    struct {
+			Enabled             bool
+			DirectionsURIFormat string
+			PathedURIFormat     string
+			Zoom                struct {
+				AcceptableLatLngDelta float32
+				Override              int
+			}
 		}
 	}
+	Clients map[string]Client
+	Storage []string
+	Message struct {
+		Matrix Matrix
+	}
 	Once    bool
 	Brokers struct {
 		NTG struct {
-			Mock     bool
+			JobInfo              bool
-			Token    string
+			Mock                 bool
-			Username string
+			LoadPageURIFormat    string
-			Password string
+			LoadPageAPIURIFormat string
+			Username             string
+			Password             string
 		}
 	}
-	Emailer        contact.Emailer
-	EmailerEnabled bool
 
 	lock sync.Mutex
 	db   storage.DB
 }
 
+type Client struct {
+	States []State
+	IDs    struct {
+		Matrix string
+	}
+	Available Time
+}
+
 var live Config
 
 func configPath() string {
@@ -57,7 +96,32 @@ func configPath() string {
 	return p
 }
 
-func Refresh() error {
+func Clients(t time.Time) map[string]Client {
+	clients := Get().Clients
+	result := map[string]Client{}
+	for k := range clients {
+		if clients[k].Available.Get().IsZero() || t.After(clients[k].Available.Get()) {
+			result[k] = clients[k]
+		}
+	}
+	return result
+}
+
+func AllStates() []State {
+	statem := map[State]struct{}{}
+	for _, v := range Clients(time.Now().Add(time.Hour * 24 * 365)) {
+		for _, state := range v.States {
+			statem[state] = struct{}{}
+		}
+	}
+	states := make([]State, 0, len(statem)+1)
+	for k := range statem {
+		states = append(states, k)
+	}
+	return states
+}
+
+func Refresh(soser func() logtr.SOSer) error {
 	b, err := ioutil.ReadFile(configPath())
 	if err != nil {
 		return err
@@ -66,10 +130,15 @@ func Refresh() error {
 	if err := json.Unmarshal(b, &c); err != nil {
 		return err
 	}
+	logtr.SetLogpath(c.Log.Path)
+	logtr.SetLevel(c.Log.Level)
 	if live.db != nil {
 		live.db.Close()
 	}
 	live = c
+	if soser != nil {
+		logtr.SetSOSer(soser())
+	}
 	return nil
 }
 
@@ -83,7 +152,7 @@ func Set(other Config) {
 		return
 	}
 	ioutil.WriteFile(configPath(), b, os.ModePerm)
-	Refresh()
+	Refresh(nil)
 }
 
 func (c *Config) DB() storage.DB {

config/duration.go

@@ -14,6 +14,9 @@ type Duration struct {
 }
 
 func (d Duration) Get() time.Duration {
+	if d.least == 0 {
+		d.least = time.Second
+	}
 	jitter := d.most - d.least
 	if jitter >= time.Second {
 		jitter = time.Second * time.Duration(rand.Int()%int(jitter.Seconds()))

config/time.go

@@ -0,0 +1,23 @@
+package config
+
+import (
+	"encoding/json"
+	"time"
+)
+
+type Time time.Time
+
+func (t Time) Get() time.Time {
+	return time.Time(t)
+}
+
+func (t Time) MarshalJSON() ([]byte, error) {
+	return json.Marshal(t.Get().Unix())
+}
+
+func (t *Time) UnmarshalJSON(b []byte) error {
+	var d int64
+	err := json.Unmarshal(b, &d)
+	*t = Time(time.Unix(d, 0))
+	return err
+}

config/time_test.go

@@ -0,0 +1,22 @@
+package config
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+func TestTimeMarshal(t *testing.T) {
+	now := time.Now()
+	nowt := Time(now)
+	var other Time
+	if b, err := json.Marshal(nowt); err != nil {
+		t.Fatal(err)
+	} else if len(b) < 5 {
+		t.Fatal(string(b))
+	} else if err := json.Unmarshal(b, &other); err != nil {
+		t.Fatal(err)
+	} else if now.Unix() != other.Get().Unix() {
+		t.Fatal(other.Get().Unix(), now.Unix())
+	}
+}

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/golang/protobuf v1.2.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/gomodule/redigo v1.8.5 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect
 	github.com/json-iterator/go v1.1.9 // indirect
 	github.com/klauspost/compress v1.9.5 // indirect

logtr/log.go

@@ -0,0 +1,138 @@
+package logtr
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+type Level int
+
+const (
+	VERBOSE = Level(7)
+	DEBUG   = Level(8)
+	INFO    = Level(9)
+	ERROR   = Level(10)
+	SOS     = Level(15)
+)
+
+type SOSer interface {
+	Send(string) error
+}
+
+func (l Level) MarshalJSON() ([]byte, error) {
+	return json.Marshal(l.String())
+}
+
+func (l *Level) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+	s = strings.TrimSpace(s)
+	s = strings.ToUpper(s)
+	if len(s) > 3 {
+		s = s[:3]
+	}
+	for i := 0; i < int(SOS)+5; i++ {
+		l2 := Level(i)
+		if l2.String() == s {
+			*l = l2
+			return nil
+		}
+	}
+	return errors.New("unknown log level: " + s)
+}
+
+var logger io.Writer = os.Stderr
+var loggerPath string = ""
+var lock = &sync.Mutex{}
+var level Level = INFO
+var ansoser SOSer = nil
+
+func SetLogpath(p string) {
+	lock.Lock()
+	defer lock.Unlock()
+	if p == loggerPath {
+		return
+	}
+	f, err := os.OpenFile(p, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		panic(err)
+	}
+	logger = f
+	loggerPath = p
+}
+
+func SetSOSer(another SOSer) {
+	lock.Lock()
+	defer lock.Unlock()
+	ansoser = another
+}
+
+func SetLevel(l Level) {
+	lock.Lock()
+	defer lock.Unlock()
+	level = l
+}
+
+func logf(l Level, format string, args []interface{}) {
+	format = fmt.Sprintf("%v: %v: %s\n", time.Now().Format("15:04:05"), l.String(), strings.TrimSpace(format))
+	cLevel := level
+	cAnsoser := ansoser
+	if l >= cLevel {
+		fmt.Fprintf(os.Stderr, format, args...)
+	}
+	fmt.Fprintf(logger, format, args...)
+	if l == SOS && cAnsoser != nil {
+		if err := cAnsoser.Send(fmt.Sprintf(format, args...)); err != nil {
+			Errorf("failed to SOS: %v", err)
+		}
+	}
+}
+
+func SOSf(format string, args ...interface{}) {
+	logf(SOS, format, args)
+}
+
+func Infof(format string, args ...interface{}) {
+	logf(INFO, format, args)
+}
+
+func Printf(format string, args ...interface{}) {
+	Infof(format, args...)
+}
+
+func Debugf(format string, args ...interface{}) {
+	logf(DEBUG, format, args)
+}
+
+func Verbosef(format string, args ...interface{}) {
+	logf(VERBOSE, format, args)
+}
+
+func Errorf(format string, args ...interface{}) {
+	logf(ERROR, format, args)
+}
+
+func (l Level) String() string {
+	switch l {
+	case ERROR:
+		return "ERR"
+	case INFO:
+		return "INF"
+	case DEBUG:
+		return "DEB"
+	case VERBOSE:
+		return "VER"
+	case SOS:
+		return "SOS"
+	default:
+		return "?"
+	}
+}

main.go

@@ -4,13 +4,15 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"local/storage"
 	"local/truckstop/broker"
 	"local/truckstop/config"
+	"local/truckstop/logtr"
 	"local/truckstop/message"
-	"log"
+	"net/http"
+	"net/url"
 	"regexp"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -19,24 +21,26 @@ import (
 var stateFinder = regexp.MustCompile(`[A-Za-z]+`)
 
 func main() {
+	if err := config.Refresh(message.NewSOSMatrix); err != nil {
+		panic(err)
+	}
+	if config.Get().Message.Matrix.ReceiveEnabled {
+		if err := matrixrecv(); err != nil {
+			logtr.SOSf("failed to recv matrix on boot: %v", err)
+			panic(err)
+		}
+	}
 	lock := &sync.Mutex{}
 	go func() {
 		for {
-			time.Sleep(config.Get().Interval.Email.Get())
+			time.Sleep(config.Get().Interval.Input.Get())
-			if err := config.Refresh(); err != nil {
+			if err := config.Refresh(message.NewSOSMatrix); err != nil {
-				log.Println(err)
+				logtr.Errorf("failed parsing config: %v", err)
 			} else {
-				if config.Get().EmailerEnabled {
-					lock.Lock()
-					if err := email(); err != nil {
-						log.Print(err)
-					}
-					lock.Unlock()
-				}
 				if config.Get().Message.Matrix.ReceiveEnabled {
 					lock.Lock()
 					if err := matrixrecv(); err != nil {
-						log.Print(err)
+						logtr.Errorf("failed receiving and parsing matrix: %v", err)
 					}
 					lock.Unlock()
 				}
@@ -44,74 +48,166 @@ func main() {
 		}
 	}()
 	if err := _main(); err != nil {
+		logtr.SOSf("failed _main: %v", err)
 		panic(err)
 	}
 	lock.Lock()
 }
 
 func matrixrecv() error {
-	log.Printf("checking matrix...")
+	logtr.Debugf("checking matrix...")
-	defer log.Printf("/checking matrix...")
+	defer logtr.Debugf("/checking matrix...")
 	sender := message.NewMatrix()
 	messages, err := sender.Receive()
 	if err != nil {
 		return err
 	}
-	states := map[config.State]struct{}{}
+	func() {
-	for _, msg := range messages {
+		logtr.Debugf("looking for help")
-		if len(states) > 0 {
+		printed := false
-			continue
+		for _, msg := range messages {
+			if !strings.HasPrefix(msg.Content, "!help") {
+				continue
+			}
+			key := fmt.Sprintf("help_%d", msg.Timestamp.Unix())
+			db := config.Get().DB()
+			if !printed {
+				if _, err := db.Get(key); err == storage.ErrNotFound {
+					logtr.Debugf("sending help")
+					help := fmt.Sprintf("commands:\n...`!help`   print this help\n...`!state nc NC nC Nc`   set states for self\n...`!available 2022-12-31`   set date self is available for work\n\nrun a command for someone else: `!state ga @caleb`")
+					if err := sender.Send(help); err != nil {
+						logtr.Errorf("failed to send help: %v", err)
+					} else {
+						printed = true
+						if err := db.Set(key, []byte{'k'}); err != nil {
+							logtr.Errorf("failed to mark help given @%s: %v", key, err)
+						}
+					}
+				}
+			} else {
+				if err := db.Set(key, []byte{'k'}); err != nil {
+					logtr.Errorf("failed to mark help given @%s: %v", key, err)
+				}
+			}
 		}
-		for _, state := range parseOutStates([]byte(msg)) {
+	}()
-			states[state] = struct{}{}
+	func() {
+		logtr.Debugf("looking for states")
+		db := config.Get().DB()
+		states := map[string]map[config.State]struct{}{}
+		for _, msg := range messages {
+			key := fmt.Sprintf("states_%d", msg.Timestamp.Unix())
+			if !strings.HasPrefix(msg.Content, "!state") {
+				continue
+			}
+			if _, ok := states[msg.Sender]; ok {
+				continue
+			}
+			if _, err := db.Get(key); err == storage.ErrNotFound {
+				states[msg.Sender] = map[config.State]struct{}{}
+				for _, state := range parseOutStates([]byte(msg.Content)) {
+					states[msg.Sender][state] = struct{}{}
+				}
+			}
+			if err := db.Set(key, []byte{'k'}); err != nil {
+				logtr.Errorf("failed to mark state gathered @%s: %v", key, err)
+			}
 		}
-	}
+		setNewStates(states)
-	if len(states) == 0 {
+	}()
-		return nil
+	func() {
-	}
+		logtr.Debugf("looking for pauses")
-	conf := *config.Get()
+		db := config.Get().DB()
-	conf.States = []config.State{}
+		pauses := map[string]time.Time{}
-	for k := range states {
+		for _, msg := range messages {
-		conf.States = append(conf.States, k)
+			key := fmt.Sprintf("pauses_%d", msg.Timestamp.Unix())
-	}
+			if !strings.HasPrefix(msg.Content, "!available ") {
-	log.Printf("%+v, %+v", states, conf)
+				continue
-	config.Set(conf)
+			}
+			if _, ok := pauses[msg.Sender]; ok {
+				continue
+			}
+			if _, err := db.Get(key); err == storage.ErrNotFound {
+				t, err := time.ParseInLocation(
+					"2006-01-02",
+					strings.TrimSpace(strings.TrimPrefix(msg.Content, "!available ")),
+					time.Local,
+				)
+				if err == nil {
+					pauses[msg.Sender] = t
+				}
+			}
+			if err := db.Set(key, []byte{'k'}); err != nil {
+				logtr.Errorf("failed to mark state gathered @%s: %v", key, err)
+			}
+		}
+		setNewPauses(pauses)
+	}()
+	message.SetMatrixContinuation(sender.Continuation())
 	return nil
 }
 
-func email() error {
+func setNewPauses(pauses map[string]time.Time) {
-	log.Printf("checking email...")
+	if len(pauses) == 0 {
-	ch, err := config.Get().Emailer.ReadIMAP()
+		return
-	if err != nil {
-		ch, err = config.Get().Emailer.ReadIMAP()
 	}
-	if err != nil {
+	logtr.Debugf("set new pauses: %+v", pauses)
-		return err
+	conf := *config.Get()
-	}
+	changed := map[string]time.Time{}
-	states := map[config.State]struct{}{}
+	for client, pause := range pauses {
-	for email := range ch {
+		clientconf := conf.Clients[client]
-		if len(states) > 0 {
+		if clientconf.Available.Get().Unix() == pause.Unix() {
 			continue
 		}
-		if !strings.Contains(email.Header.Get("From"), config.Get().Client) {
+		clientconf.Available = config.Time(pause)
-			continue
+		conf.Clients[client] = clientconf
-		}
+		changed[client] = pause
-		b, _ := ioutil.ReadAll(email.Body)
+	}
-		for _, state := range parseOutStates(b) {
+	if len(changed) == 0 {
-			states[state] = struct{}{}
+		return
+	}
+	logtr.Infof("updating config new pauses: %+v", conf)
+	config.Set(conf)
+	for client, pause := range changed {
+		if err := sendNewPause(client, pause); err != nil {
+			logtr.Errorf("failed to send new pause %s/%+v: %v", client, pause, err)
 		}
 	}
+}
+
+func setNewStates(states map[string]map[config.State]struct{}) {
 	if len(states) == 0 {
-		return nil
+		return
 	}
 	conf := *config.Get()
-	conf.States = []config.State{}
+	changed := map[string][]config.State{}
-	for k := range states {
+	for client, clientStates := range states {
-		conf.States = append(conf.States, k)
+		newstates := []config.State{}
+		for k := range clientStates {
+			newstates = append(newstates, k)
+		}
+		sort.Slice(newstates, func(i, j int) bool {
+			return newstates[i] < newstates[j]
+		})
+		clientconf := conf.Clients[client]
+		if fmt.Sprint(newstates) == fmt.Sprint(clientconf.States) {
+			message.NewMatrix().Send(fmt.Sprintf("%s: still searching for %+v", client, newstates))
+			continue
+		}
+		clientconf.States = newstates
+		conf.Clients[client] = clientconf
+		changed[client] = newstates
 	}
-	log.Printf("%+v, %+v", states, conf)
+	if len(changed) == 0 {
+		return
+	}
+	logtr.Infof("updating config new states: %+v", conf)
 	config.Set(conf)
-	return nil
+	for client, states := range changed {
+		if err := sendNewStates(client, states); err != nil {
+			logtr.Errorf("failed to send new states %s/%+v: %v", client, states, err)
+		}
+	}
 }
 
 func parseOutStates(b []byte) []config.State {
@@ -136,9 +232,10 @@ func _main() error {
 	for {
 		err := _mainOne()
 		if err != nil {
-			log.Println(err)
+			logtr.Errorf("failed _main: %v", err)
 		}
 		if config.Get().Once {
+			time.Sleep(3 * time.Second)
 			return err
 		}
 		if err != nil {
@@ -151,15 +248,17 @@ func _main() error {
 }
 
 func _mainOne() error {
-	log.Println("config.refreshing...")
+	logtr.Debugf("config.refreshing...")
-	if err := config.Refresh(); err != nil {
+	if err := config.Refresh(message.NewSOSMatrix); err != nil {
+		logtr.SOSf("bad config: %v", err)
 		return err
 	}
-	log.Println("once...")
+	logtr.Debugf("once...")
 	if err := once(); err != nil {
+		logtr.SOSf("failed once(): %v", err)
 		return err
 	}
-	log.Println("/_mainOne")
+	logtr.Debugf("/_mainOne")
 	return nil
 }
 
@@ -168,32 +267,47 @@ func once() error {
 	if err != nil {
 		return err
 	}
+	logtr.Debugf("once: update dead jobs: %+v", alljobs)
+	err = updateDeadJobs(alljobs)
+	if err != nil {
+		return err
+	}
+	logtr.Debugf("once: all jobs: %+v", alljobs)
 	newjobs, err := dropStaleJobs(alljobs)
 	if err != nil {
 		return err
 	}
+	logtr.Debugf("once: new jobs: %+v", newjobs)
 	jobs, err := dropBanlistJobs(newjobs)
 	if err != nil {
 		return err
 	}
+	logtr.Debugf("found jobs: %+v", jobs)
+	if len(jobs) == 0 {
+		return nil
+	}
+	logtr.Debugf("once: loading job secrets: %+v", jobs)
 	for i := range jobs {
-		if err := sendJob(jobs[i]); err != nil {
+		jobs[i].Secrets()
-			return err
+	}
-		}
+	logtr.Infof("once: sending jobs: %+v", jobs)
-		log.Println("sent job", jobs[i])
+	db := config.Get().DB()
-		if err := config.Get().DB().Set(jobs[i].ID, []byte(`sent`)); err != nil {
+	for i := range jobs {
+		if ok, err := sendJob(jobs[i]); err != nil {
 			return err
+		} else if ok {
+			logtr.Debugf("sent job", jobs[i])
+			if err := db.Set(jobs[i].UID(), []byte(`sent`)); err != nil {
+				return err
+			}
 		}
 	}
 	return nil
 }
 
 func getJobs() ([]broker.Job, error) {
-	states := config.Get().States
+	states := config.AllStates()
 	ntg := broker.NewNTGVision()
-	if config.Get().Brokers.NTG.Mock {
-		ntg = ntg.WithMock()
-	}
 	brokers := []broker.Broker{ntg}
 	jobs := []broker.Job{}
 	for _, broker := range brokers {
@@ -206,10 +320,56 @@ func getJobs() ([]broker.Job, error) {
 	return jobs, nil
 }
 
+type recordedJob struct {
+	Job            broker.Job
+	SentNS         int64
+	MatrixID       string
+	MatrixImageIDs []string
+}
+
+func updateDeadJobs(jobs []broker.Job) error {
+	db := config.Get().DB()
+	list, err := db.List([]string{}, "sent_job_", "sent_job_}}")
+	if err != nil {
+		return err
+	}
+	for _, listEntry := range list {
+		wouldBe := strings.TrimPrefix(listEntry, "sent_job_")
+		found := false
+		for i := range jobs {
+			found = found || jobs[i].ID == wouldBe
+		}
+		logtr.Debugf("found job %s to be still alive==%v", wouldBe, found)
+		if !found {
+			logtr.Debugf("updating dead job %+v", listEntry)
+			b, err := db.Get(listEntry)
+			if err != nil {
+				return err
+			}
+			var recorded recordedJob
+			if err := json.Unmarshal(b, &recorded); err != nil {
+				return err
+			}
+			if err := message.NewMatrix().Update(recorded.MatrixID, recorded.Job.FormatMultilineTextDead()); err != nil {
+				return err
+			}
+			if err := db.Set(listEntry, nil); err != nil {
+				return err
+			}
+			for _, imageid := range recorded.MatrixImageIDs {
+				if err := message.NewMatrix().Remove(imageid); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func dropStaleJobs(jobs []broker.Job) ([]broker.Job, error) {
 	db := config.Get().DB()
 	for i := len(jobs) - 1; i >= 0; i-- {
-		if _, err := db.Get(jobs[i].ID); err == storage.ErrNotFound {
+		if _, err := db.Get(jobs[i].UID()); err == storage.ErrNotFound {
 		} else if err != nil {
 			return nil, err
 		} else {
@@ -224,7 +384,125 @@ func dropBanlistJobs(jobs []broker.Job) ([]broker.Job, error) {
 	return jobs, nil
 }
 
-func sendJob(job broker.Job) error {
+func sendJob(job broker.Job) (bool, error) {
 	sender := message.NewMatrix()
-	return sender.Send(job.FormatMultilineText())
+	payload := job.FormatMultilineText()
+	logtr.Debugf("once: send job %s if nonzero: %s", job.String(), payload)
+	if len(payload) == 0 {
+		return false, nil
+	}
+	id, err := sender.SendTracked(payload)
+	if err != nil {
+		return false, err
+	}
+	recordedJob := recordedJob{
+		Job:      job,
+		SentNS:   time.Now().UnixNano(),
+		MatrixID: id,
+	}
+	defer func() {
+		db := config.Get().DB()
+		b, err := json.Marshal(recordedJob)
+		if err != nil {
+			logtr.Errorf("failed to marshal recorded job: %v", err)
+			return
+		}
+		if err := db.Set("sent_job_"+job.ID, b); err != nil {
+			logtr.Errorf("failed to set recorded job: %v", err)
+			return
+		}
+	}()
+	maps := config.Get().Maps
+	pickup := fmt.Sprintf("%s,%s", url.QueryEscape(job.Pickup.City), job.Pickup.State)
+	dropoff := fmt.Sprintf("%s,%s", url.QueryEscape(job.Dropoff.City), job.Dropoff.State)
+	if maps.Pathed.Enabled {
+		directionsURI := fmt.Sprintf(maps.Pathed.DirectionsURIFormat, pickup, dropoff)
+		resp, err := http.Get(directionsURI)
+		if err != nil {
+			return true, err
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			return true, fmt.Errorf("bad status getting path: %d", resp.StatusCode)
+		}
+		var directionsResp struct {
+			Routes []struct {
+				Legs []struct {
+					Steps []struct {
+						StartLocation struct {
+							Lat float32 `json:"lat"`
+							Lng float32 `json:"lng"`
+						} `json:"start_location"`
+					} `json:"steps"`
+				} `json:"legs"`
+			} `json:"routes"`
+		}
+		if err := json.NewDecoder(resp.Body).Decode(&directionsResp); err != nil {
+			return true, err
+		}
+		if len(directionsResp.Routes) < 1 || len(directionsResp.Routes[0].Legs) < 1 || len(directionsResp.Routes[0].Legs[0].Steps) < 1 {
+		} else {
+			latLngPath := make([]string, 0)
+			minLat := directionsResp.Routes[0].Legs[0].Steps[0].StartLocation.Lat
+			maxLat := directionsResp.Routes[0].Legs[0].Steps[0].StartLocation.Lat
+			minLng := directionsResp.Routes[0].Legs[0].Steps[0].StartLocation.Lng
+			maxLng := directionsResp.Routes[0].Legs[0].Steps[0].StartLocation.Lng
+			for _, v := range directionsResp.Routes[0].Legs[0].Steps {
+				if v.StartLocation.Lng < minLng {
+					minLng = v.StartLocation.Lng
+				}
+				if v.StartLocation.Lng > maxLng {
+					maxLng = v.StartLocation.Lng
+				}
+				if v.StartLocation.Lat < minLat {
+					minLat = v.StartLocation.Lat
+				}
+				if v.StartLocation.Lat > maxLat {
+					maxLat = v.StartLocation.Lat
+				}
+				latLngPath = append(latLngPath, fmt.Sprintf("%.9f,%.9f", v.StartLocation.Lat, v.StartLocation.Lng))
+			}
+			pathQuery := strings.Join(latLngPath, "|")
+			uri := fmt.Sprintf(maps.Pathed.PathedURIFormat, pathQuery, pickup, dropoff)
+			// if the bigger delta is <acceptable, override zoom
+			if maxLat-minLat <= maps.Pathed.Zoom.AcceptableLatLngDelta && maxLng-minLng <= maps.Pathed.Zoom.AcceptableLatLngDelta {
+				uri = fmt.Sprintf("%s&zoom=%d", uri, maps.Pathed.Zoom.Override)
+			}
+			logtr.Debugf("sending pathed image: %s", uri)
+			pathedid, err := sender.SendImageTracked(uri)
+			if err != nil {
+				return true, err
+			}
+			recordedJob.MatrixImageIDs = append(recordedJob.MatrixImageIDs, pathedid)
+		}
+	}
+	if maps.Pickup {
+		uri := fmt.Sprintf(maps.URIFormat, pickup, pickup)
+		logtr.Debugf("sending pickup image: %s", uri)
+		pickupid, err := sender.SendImageTracked(uri)
+		if err != nil {
+			return true, err
+		}
+		recordedJob.MatrixImageIDs = append(recordedJob.MatrixImageIDs, pickupid)
+	}
+	if maps.Dropoff {
+		uri := fmt.Sprintf(maps.URIFormat, dropoff, dropoff)
+		logtr.Debugf("sending dropoff image: %s", uri)
+		dropid, err := sender.SendImageTracked(uri)
+		if err != nil {
+			return true, err
+		}
+		recordedJob.MatrixImageIDs = append(recordedJob.MatrixImageIDs, dropid)
+	}
+	return true, nil
+}
+
+func sendNewStates(client string, states []config.State) error {
+	sender := message.NewMatrix()
+	return sender.Send(fmt.Sprintf("%s: now searching for loads from: %+v", client, states))
+}
+
+func sendNewPause(client string, pause time.Time) error {
+	sender := message.NewMatrix()
+	return sender.Send(fmt.Sprintf("%s: only searching for loads on and after %s", client, pause.Format("2006-01-02")))
 }

message/images.go

@@ -0,0 +1,90 @@
+package message
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"local/truckstop/config"
+	"local/truckstop/logtr"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+var ErrBadAuth = errors.New("auth has failed")
+
+func UploadImage(b []byte) (string, error) {
+	path, err := uploadImage(b)
+	if err == ErrBadAuth {
+		if err := refreshToken(); err != nil {
+			return "", err
+		}
+		path, err = uploadImage(b)
+	}
+	return path, err
+}
+
+func refreshToken() error {
+	return errors.New("not impl")
+}
+
+func uploadImage(b []byte) (string, error) {
+	images := config.Get().Images
+	buff := bytes.NewBuffer(nil)
+	writer := multipart.NewWriter(buff)
+	name := "name"
+	if u, err := url.Parse(images.UploadURI); err != nil {
+	} else if s, ok := u.Query()["name"]; !ok {
+	} else {
+		name = s[0]
+		logtr.Debugf("found name in upload uri: %s", name)
+	}
+	part, err := writer.CreateFormFile("image", name)
+	if err != nil {
+		return "", err
+	}
+	if n, err := part.Write(b); err != nil {
+		return "", err
+	} else if n < len(b) {
+		return "", errors.New("short write")
+	}
+	writer.Close()
+
+	request, err := http.NewRequest(
+		images.UploadMethod,
+		images.UploadURI,
+		buff,
+	)
+	if err != nil {
+		return "", err
+	}
+	request.Header.Set("Authorization", "Bearer "+images.AccessToken)
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+
+	c := &http.Client{Timeout: time.Minute}
+	response, err := c.Do(request)
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+	b, _ = ioutil.ReadAll(response.Body)
+	switch response.StatusCode {
+	case http.StatusOK:
+	case 401, 403:
+		return "", ErrBadAuth
+	default:
+		return "", fmt.Errorf("error uploading image: (%d) %s", response.StatusCode, b)
+	}
+	var result struct {
+		Data struct {
+			Link string `json:"link"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(b, &result); err != nil {
+		return "", err
+	}
+	return result.Data.Link, nil
+}

message/images_test.go

@@ -0,0 +1,27 @@
+package message
+
+import (
+	"io/ioutil"
+	"local/truckstop/config"
+	"os"
+	"testing"
+)
+
+func TestImageUpload(t *testing.T) {
+	if os.Getenv("INTEGRATION") == "" {
+		t.Skip("$INTEGRATION not set")
+	}
+	os.Setenv("CONFIG", "../config.json")
+	b, err := ioutil.ReadFile("./testdata/whatever.jpg")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := config.Refresh(nil); err != nil {
+		t.Fatal(err)
+	}
+	got, err := UploadImage(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(got)
+}

message/matrix.go

@@ -1,30 +1,46 @@
 package message
 
 import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
 	"local/truckstop/config"
-	"log"
+	"local/truckstop/logtr"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
 
 	"github.com/matrix-org/gomatrix"
 )
 
 type Matrix struct {
-	mock       bool
+	mock         bool
-	homeserver string
+	homeserver   string
-	username   string
+	username     string
-	token      string
+	token        string
-	room       string
+	room         string
-	client     string
+	continuation string
+}
+
+func NewSOSMatrix() logtr.SOSer {
+	conf := config.Get().Log.SOSMatrix
+	return newMatrix(conf, "0")
 }
 
 func NewMatrix() Matrix {
 	conf := config.Get().Message.Matrix
+	return newMatrix(conf, GetMatrixContinuation())
+}
+
+func newMatrix(conf config.Matrix, cont string) Matrix {
 	return Matrix{
-		homeserver: conf.Homeserver,
+		homeserver:   conf.Homeserver,
-		username:   conf.Username,
+		username:     conf.Username,
-		token:      conf.Token,
+		token:        conf.Token,
-		room:       conf.Room,
+		room:         conf.Room,
-		mock:       conf.Mock,
+		mock:         conf.Mock,
-		client:     conf.Client,
+		continuation: cont,
 	}
 }
 
@@ -32,44 +48,207 @@ func (m Matrix) getclient() (*gomatrix.Client, error) {
 	return gomatrix.NewClient(m.homeserver, m.username, m.token)
 }
 
-func (m Matrix) Receive() ([]string, error) {
+func (m Matrix) Continuation() string {
+	return m.continuation
+}
+
+func (m *Matrix) Receive() ([]Message, error) {
 	if m.mock {
-		log.Printf("matrix.Receive()")
+		logtr.Infof("matrix.Receive()")
-		return []string{"FL, GA, NC"}, nil
+		messages := make([]Message, 0)
+		for k := range config.Get().Clients {
+			messages = append(messages, Message{Timestamp: time.Now(), Sender: k, Content: "!state nc"})
+			if k == "bel" {
+				messages = append(messages, Message{Timestamp: time.Now(), Sender: k, Content: "!help"})
+			}
+			if k == "broc" {
+				messages = append(messages, Message{Timestamp: time.Now(), Sender: k, Content: "!available 2148-10-" + fmt.Sprint(time.Now().Unix()%28)})
+			}
+		}
+		return messages, nil
+	}
+	clients := config.Get().Clients
+	matrixIDs := map[string]struct{}{}
+	for k := range clients {
+		matrixIDs[clients[k].IDs.Matrix] = struct{}{}
+	}
+	if len(matrixIDs) == 0 {
+		return nil, nil
 	}
 	c, err := m.getclient()
 	if err != nil {
 		return nil, err
 	}
-	messages := make([]string, 0)
+	messages := make([]Message, 0)
-	result, err := c.Messages(m.room, "", "", 'b', 50)
+	result, err := c.Messages(m.room, "999999999999999999", m.Continuation(), 'b', 50)
+	if err != nil {
+		return nil, err
+	}
+	logtr.Debugf("%s => {Start:%s End:%v};; %v, (%d)", m.Continuation(), result.Start, result.End, err, len(result.Chunk))
+	m.continuation = result.End
 	for _, event := range result.Chunk {
-		if event.Sender != m.client {
+		logtr.Verbosef("matrix event: %+v", event)
+		if _, ok := matrixIDs[event.Sender]; !ok {
 			continue
 		}
 		switch event.Type {
 		case "m.room.message":
 			b, ok := event.Body()
 			if ok {
-				messages = append(messages, b)
+				messages = append(messages, Message{Timestamp: time.Unix(0, event.Timestamp*int64(time.Millisecond)), Sender: event.Sender, Content: strings.TrimSpace(b)})
 			}
 		}
 	}
-	if err != nil {
+	clientChange := regexp.MustCompile("@[a-z]+$")
-		return nil, err
+	logtr.Debugf("rewriting messages based on @abc")
+	for i := range messages {
+		if found := clientChange.FindString(messages[i].Content); found != "" {
+			messages[i].Content = strings.TrimSpace(strings.ReplaceAll(messages[i].Content, found, ""))
+			messages[i].Sender = found[1:]
+		} else {
+			for k, v := range config.Get().Clients {
+				if v.IDs.Matrix == messages[i].Sender {
+					messages[i].Sender = k
+				}
+			}
+		}
+		messages[i].Content = strings.TrimSpace(messages[i].Content)
+	}
+	logtr.Debugf("rewriting messages based on ! CoMmAnD ...")
+	for i := range messages {
+		if strings.HasPrefix(messages[i].Content, "!") {
+			messages[i].Content = "!" + strings.TrimSpace(messages[i].Content[1:])
+			splits := strings.Split(messages[i].Content, " ")
+			messages[i].Content = strings.ToLower(splits[0]) + " " + strings.Join(splits, " ")
+		}
 	}
 	return messages, nil
 }
 
-func (m Matrix) Send(text string) error {
+func (m Matrix) Remove(id string) error {
 	if m.mock {
-		log.Printf("matrix.Send(%s)", text)
+		logtr.Infof("matrix.Remove(%s)", id)
 		return nil
 	}
 	c, err := m.getclient()
 	if err != nil {
 		return err
 	}
-	_, err = c.SendText(m.room, text)
+	_, err = c.RedactEvent(m.room, id, &gomatrix.ReqRedact{Reason: "stale"})
 	return err
 }
+
+func (m Matrix) Update(id, text string) error {
+	if m.mock {
+		logtr.Infof("matrix.Update(%s)", text)
+		return nil
+	}
+	c, err := m.getclient()
+	if err != nil {
+		return err
+	}
+	type MRelatesTo struct {
+		EventID string `json:"event_id"`
+		RelType string `json:"rel_type"`
+	}
+	type NewContent struct {
+		Body    string `json:"body"`
+		MsgType string `json:"msgtype"`
+	}
+	type RelatesToRoomMessage struct {
+		Body        string     `json:"body"`
+		MsgType     string     `json:"msgtype"`
+		MRelatesTo  MRelatesTo `json:"m.relates_to"`
+		MNewContent NewContent `json:"m.new_content"`
+	}
+	_, err = c.SendMessageEvent(m.room, "m.room.message", RelatesToRoomMessage{
+		Body:    "",
+		MsgType: "m.text",
+		MNewContent: NewContent{
+			Body:    text,
+			MsgType: "m.text",
+		},
+		MRelatesTo: MRelatesTo{
+			EventID: id,
+			RelType: "m.replace",
+		},
+	})
+	return err
+}
+
+func (m Matrix) Send(text string) error {
+	_, err := m.SendTracked(text)
+	return err
+}
+
+func (m Matrix) SendTracked(text string) (string, error) {
+	if m.mock {
+		logtr.Infof("matrix.SendTracked(%s)", text)
+		return "", nil
+	}
+	c, err := m.getclient()
+	if err != nil {
+		return "", err
+	}
+	resp, err := c.SendText(m.room, text)
+	if err != nil {
+		return "", err
+	}
+	return resp.EventID, nil
+}
+
+func (m Matrix) SendImage(uri string) error {
+	_, err := m.SendImageTracked(uri)
+	return err
+}
+
+func (m Matrix) SendImageTracked(uri string) (string, error) {
+	if m.mock {
+		logtr.Infof("matrix.SendImage(%s)", uri)
+		return "", nil
+	}
+	response, err := http.Get(uri)
+	if err != nil {
+		return "", err
+	}
+	if response.StatusCode != http.StatusOK {
+		b, _ := ioutil.ReadAll(response.Body)
+		response.Body.Close()
+		return "", fmt.Errorf("failed to get %s: (%d) %s", uri, response.StatusCode, b)
+	}
+	b, err := ioutil.ReadAll(response.Body)
+	response.Body.Close()
+	if err != nil {
+		return "", err
+	}
+	c, err := m.getclient()
+	if err != nil {
+		return "", err
+	}
+	mediaUpload, err := c.UploadToContentRepo(bytes.NewReader(b), "image/jpeg", int64(len(b)))
+	if err != nil {
+		return "", err
+	}
+	publicURI := mediaUpload.ContentURI
+	resp, err := c.SendImage(m.room, "img", publicURI)
+	logtr.Debugf("sent image %s => %s: %+v", uri, publicURI, resp)
+	return resp.EventID, err
+}
+
+func SetMatrixContinuation(continuation string) {
+	db := config.Get().DB()
+	db.Set(getMatrixContinuationKey(), []byte(continuation))
+}
+
+func GetMatrixContinuation() string {
+	db := config.Get().DB()
+	b, _ := db.Get(getMatrixContinuationKey())
+	if b == nil {
+		return "0"
+	}
+	return string(b)
+}
+
+func getMatrixContinuationKey() string {
+	return "matrix_continuation"
+}

message/matrix_test.go

@@ -1,57 +1,65 @@
 package message
 
 import (
-	"encoding/json"
 	"io/ioutil"
 	"local/truckstop/config"
 	"os"
+	"path"
 	"testing"
+
+	"github.com/google/uuid"
 )
 
-func TestMatrixSend(t *testing.T) {
+func TestMatrixSendDel(t *testing.T) {
-	if len(os.Getenv("INTEGRATION")) == 0 {
+	sender := testMatrix(t)
-		t.Skip("$INTEGRATION not set")
+	if id, err := sender.SendTracked("hello world from unittest"); err != nil {
-	}
-	var c config.Config
-	b, err := ioutil.ReadFile("../config.json")
-	if err != nil {
 		t.Fatal(err)
-	}
+	} else if err := sender.Remove(id); err != nil {
-	if err := json.Unmarshal(b, &c); err != nil {
-		t.Fatal(err)
-	}
-	var sender Sender = Matrix{
-		homeserver: c.Message.Matrix.Homeserver,
-		username:   c.Message.Matrix.Username,
-		token:      c.Message.Matrix.Token,
-		room:       c.Message.Matrix.Room,
-	}
-	if err := sender.Send("hello world from unittest"); err != nil {
 		t.Fatal(err)
 	}
 }
 
-func TestMatrixReceive(t *testing.T) {
+func TestMatrixUpdate(t *testing.T) {
-	if len(os.Getenv("INTEGRATION")) == 0 {
+	// 19:32:34: VER: matrix event: {StateKey:<nil> Sender:@bot:m.bltrucks.top Type:m.room.message Timestamp:1642471594729 ID:$n0ln9TFZrko_cBNFXeh8YyICZ3fFm17Jhz3bmZcqig4 RoomID:!OYZqtInrBCn1cyz90D:m.bltrucks.top Redacts: Unsigned:map[transaction_id:go1642471594571852423] Content:map[body:hello world from unittest format: formatted_body: msgtype:m.text] PrevContent:map[]}
-		t.Skip("$INTEGRATION not set")
+	// func (m Matrix) Update(id, text string) error {
-	}
+	m := testMatrix(t)
-	var c config.Config
+	uid := uuid.New().String()[:3]
-	b, err := ioutil.ReadFile("../config.json")
+	id, err := m.SendTracked("hello, heheheh from test matrix update " + uid)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := json.Unmarshal(b, &c); err != nil {
+	err = m.Update(id, "heheheh i updated from test matrix update "+uid)
+	if err != nil {
 		t.Fatal(err)
 	}
-	var sender Sender = Matrix{
+	m.Receive()
-		homeserver: c.Message.Matrix.Homeserver,
+}
-		username:   c.Message.Matrix.Username,
+
-		token:      c.Message.Matrix.Token,
+func TestMatrixReceive(t *testing.T) {
-		room:       c.Message.Matrix.Room,
+	sender := testMatrix(t)
-	}
 	if msgs, err := sender.Receive(); err != nil {
 		t.Fatal(err)
 	} else {
 		t.Logf("%+v", msgs)
 	}
 }
+
+func testMatrix(t *testing.T) Matrix {
+	if len(os.Getenv("INTEGRATION")) == 0 {
+		t.Skip("$INTEGRATION not set")
+	}
+	d := t.TempDir()
+	f := path.Join(d, "config.test.json")
+	b, err := ioutil.ReadFile("../config.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(f, b, os.ModePerm); err != nil {
+		t.Fatal(err)
+	}
+	os.Setenv("CONFIG", f)
+	if err := config.Refresh(nil); err != nil {
+		t.Fatal(err)
+	}
+	return NewMatrix()
+}

message/message.go

@@ -1,6 +1,16 @@
 package message
 
+import "time"
+
 type Sender interface {
 	Send(string) error
-	Receive() ([]string, error)
+	SendTracked(string) (string, error)
+	Update(string, string) (string, error)
+	Receive() ([]Message, error)
+}
+
+type Message struct {
+	Sender    string
+	Content   string
+	Timestamp time.Time
 }

testdata/index.html

@@ -0,0 +1,5 @@
+<html>
+<body>
+  <img src='https://maps.googleapis.com/maps/api/staticmap?center=Advance,NC&zoom=13&size=400x400&maptype=roadmap&key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg'/>
+</body>
+</html>

testdata/routed_map.sh

@@ -0,0 +1,3 @@
+#! /bin/bash
+
+poly="${poly:-"$(curl 'https://maps.googleapis.com/maps/api/directions/json?origin=advance,nc&destination=winston-salem,nc&mode=driving&key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg' | jq -c .routes[0].legs[0].steps[] | jq -c .start_location | jq -r '(.lat|tostring) + "," + (.lng|tostring)' | tr '\n' '|' | sed 's/.$//')"}"; curl "https://maps.googleapis.com/maps/api/staticmap?size=250x250&path=$(urlencode "$poly")&format=jpeg&maptype=roadmap&key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg&size=600x400&sensor=false"  > /tmp/f.jpg ; open /tmp/f.jpg

todo.yaml

@@ -1,12 +1,67 @@
 todo:
+- !states emits current state 
+- TEST. Just like, refactor and test to shit.
+- try search ntg by autoinc?
+- test each !command callbacks to matrix
+- recv-as for clients so pa receives mas commands as writes
+- continuation is garbo, but I can still do better client side to avoid get-set high level
+- no hard code jpeg or have it in multiple places
+- change matrix so I test my custom logic even if I dont fetch remote
+- warn/err/etc. on clobbering ids.matrix since clients can mess with one another
+- modify old items once no longer available; drop stale jobs good candidate but requires new matrix interaction
+- more than NTG
 - todo: filter out jobs like CA
   subtasks:
   - banlist criteria like vendors, brokers, metadata
-- quiet hours
-- setup ma on element
-- accept states via element for one system
 - set up copy for caleb, broc
 done:
+- mark jobs no longer avail by modifying in matrix;; save matrix ID over dummy payload
+- TEST its falling apart
+- help() log on truckstop for stuff like perma 403
+- figure out zoom on maps;; is there like an auto-zoom I can leverage?
+- mock is nigh useless
+- mark consumed;; save scroll id for matrix
+- todo: switch house to selfhosted for rate limit
+  tags: now
+- todo: details from ntg; stophours for pickup, appointmenttime/facitlyhours for dest
+  details: |
+    curl 'https://ntgvision.com/api/v1/load/LoadDetails?loadId=5088453' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' -H 'Accept: application/json, text/plain, */*' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'Authorization: Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMzgyMSIsInVuaXF1ZV9uYW1lIjoiYmlyZGNvbXBhbnlsb2dpc3RpY3NAZ21haWwuY29tIiwianRpIjoiYTFiMjkxM2YtN2RjZS00ZDQ0LWEzMDYtMTRjMTBlYjQ2MmE1IiwiaWF0IjoiMS8xMy8yMDIyIDI6MTE6NTUgUE0iLCJudGd2Um9sZSI6IkNhcnJpZXJBcHByb3ZlZCIsImxvY2tVc2VyIjoiRmFsc2UiLCJvdmVycmlkZUJsYWNrbGlzdCI6IkZhbHNlIiwic2hvd1JhdGVzIjoiRmFsc2UiLCJ1c2VyQ2FycmllcnMiOiIxMTQxOTMiLCJvdHJVc2VyIjoiRmFsc2UiLCJuYmYiOjE2NDIwODMxMTUsImV4cCI6MTY0MjE2NTkxNSwiaXNzIjoiTlRHIFNlY3VyaXR5IFRva2VuIFNlcnZpY2UiLCJhdWQiOiJOVEcifQ.eDE2Jfok79b7oD1deP_00h9g8zZ3AnY60McnJ1hfS-8YJ12L3Bl-CJeN0eBos0aj2FXHv5MrqOJ6MFCaatqjIddPJl2MvtSByGUCLBzih67O20mkCowCmZfGjSnKvUu7DHFxjzo_y4_a3cjjGPYNc2LSCVJxV9NYzSXIuXG3WvXeE3dv8ml23bJFFhMWZzhSAfWBOW4kR61sibS6BpRD8dpkKoqUfXInq_7o8jz9_PsEhPBdJydqbXwg8ifQHkaPNAojsIr2rnJ3Tf_iQpom-6oAEUAOd2D3fK1XuWtRrysnD8s41YDthmqKpni2WOk72L-sKzRxD4KX2t_gyEb3Fw' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Cookie: cookiesession1=678A3E1398901234BCDEFGHIJKLMA492; _uiq_id.711119701.3d83=516d3e744ebe2d9a.1641792415.0.1642083653..; NTGAuthToken=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMzgyMSIsInVuaXF1ZV9uYW1lIjoiYmlyZGNvbXBhbnlsb2dpc3RpY3NAZ21haWwuY29tIiwianRpIjoiYTFiMjkxM2YtN2RjZS00ZDQ0LWEzMDYtMTRjMTBlYjQ2MmE1IiwiaWF0IjoiMS8xMy8yMDIyIDI6MTE6NTUgUE0iLCJudGd2Um9sZSI6IkNhcnJpZXJBcHByb3ZlZCIsImxvY2tVc2VyIjoiRmFsc2UiLCJvdmVycmlkZUJsYWNrbGlzdCI6IkZhbHNlIiwic2hvd1JhdGVzIjoiRmFsc2UiLCJ1c2VyQ2FycmllcnMiOiIxMTQxOTMiLCJvdHJVc2VyIjoiRmFsc2UiLCJuYmYiOjE2NDIwODMxMTUsImV4cCI6MTY0MjE2NTkxNSwiaXNzIjoiTlRHIFNlY3VyaXR5IFRva2VuIFNlcnZpY2UiLCJhdWQiOiJOVEcifQ.eDE2Jfok79b7oD1deP_00h9g8zZ3AnY60McnJ1hfS-8YJ12L3Bl-CJeN0eBos0aj2FXHv5MrqOJ6MFCaatqjIddPJl2MvtSByGUCLBzih67O20mkCowCmZfGjSnKvUu7DHFxjzo_y4_a3cjjGPYNc2LSCVJxV9NYzSXIuXG3WvXeE3dv8ml23bJFFhMWZzhSAfWBOW4kR61sibS6BpRD8dpkKoqUfXInq_7o8jz9_PsEhPBdJydqbXwg8ifQHkaPNAojsIr2rnJ3Tf_iQpom-6oAEUAOd2D3fK1XuWtRrysnD8s41YDthmqKpni2WOk72L-sKzRxD4KX2t_gyEb3Fw' -H 'Sec-Fetch-Dest: empty' -H 'Sec-Fetch-Mode: cors' -H 'Sec-Fetch-Site: same-origin' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache'
+    {"headerSummary":"Washington, NC to Abilene, TX","loadId":5088453,"miles":1481,"weight":6000,"temp":null,"equipment":"Str Truck W/ Lift Gate","categoryName":"Lighting / Lighting Fixtures","categoryLabel":"Product Category","cargoInformation":["Store Fixtures - 6000 lbs"],"loadRequirements":["Straight Truck with lift gate required"],"stopInfos":[{"loadInfoIds":[5431607],"stopDateTime":"0001-01-01T00:00:00","isPickup":true,"location":"Washington, NC","stopDate":"Friday, 01/14/22","stopHours":"10:00 - 12:00","appointmentTime":"","appointmentType":"FCFS","instructions":"***LIFTGATE, PALLET JACK, DRIVER ASSIST***--MUST DELIVER ONLY ON APPT TIME-- CANNOT DELIVER EARLY OR LATE \r\n","isDropTrailer":false,"shipperName":"IDX NORTH CAROLINA","processedStopDate":""},{"loadInfoIds":[5431607],"stopDateTime":"0001-01-01T00:00:00","isPickup":false,"location":"Abilene, TX","stopDate":"Monday, 01/17/22","stopHours":"09:00 - 10:00","appointmentTime":"09:00","appointmentType":"APPT","instructions":"***LIFTGATE, PALLET JACK, DRIVER ASSIST***--MUST DELIVER ONLY ON APPT TIME-- CANNOT DELIVER EARLY OR LATE \r\n","isDropTrailer":false,"shipperName":"hibbett sports","processedStopDate":""}],"drayStopInfos":null,"rateInfo":null,"isDray":false,"equipmentGroupId":8,"totalCarrierRate":2600.00,"payUpTo":2700.00,"firstLoadInfoId":5431607,"loadStatus":"ACTIVE","hasBlockingAlert":false,"customerLoadBlocked":false,"canSeeRate":false,"canBookNow":false,"canBidNow":true,"buttonData":{"useBidDialog":false,"lastBidAmount":null,"remainingBids":1,"carrierPhoneNumber":null,"carrierPhoneExtension":null},"stopData":[{"addr":"IDX NORTH CAROLINA","city":"Washington","state":"NC","zip":"27889"},{"addr":"hibbett sports","city":"Abilene","state":"TX","zip":"79606"}]}
+
+- tokens in db, not in config
+- cache on disk jobinfo
+- link to view more
+- map with to-from line
+- TO CONFLUENT.RS OR W/E
+- rm get-set stuff for matrix now that it has continuation
+- setup ma on element !!fluffychat
+- !help,
+- !states optional but explicit option
+- TEST; convert pauseuntil to search results only on and after target date
+- pause until => !busy until
+- todo: maps of to+from to get location within state via api
+  details: |
+    curl  'https://maps.googleapis.com/maps/api/staticmap?center=Advance,NC&markers=label=A|Advance,NC&zoom=5&size=250x250&scale=2&format=jpeg&maptype=roadmap&key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' -H 'Accept: image/avif,image/webp,*/*' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'DNT: 1' -H 'Alt-Used: maps.googleapis.com' -H 'Connection: keep-alive' -H 'Sec-Fetch-Dest: image' -H 'Sec-Fetch-Mode: no-cors' -H 'Sec-Fetch-Site: cross-site' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'TE: trailers'  > whatever.jpg; open whatever.jpg~/Go/src/local/truckstop
+  subtasks:
+  - DONE; yandex; key 9baa3e42-c6e5-4eb5-a891-05ffcede6a25 yandex maps
+  - google; key AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg
+- todo: upload g map to imgur
+  details: |
+    w oath d9ac7cabe813d10 9d0b3d82800b30ca88f595d3bcd6985f627d7d82
+    Authorization: Client-ID d9ac7cabe813d10
+    Authorization: Bearer YOUR_ACCESS_TOKEN
+- quiet hours
+- "@caleb commands: args"
+- accept after date
+- accept pause commands
+- accept states via element for one system
+- write to matrix on config change like states
+- rate LIMIT
+- rate limit brokers
+- many users -> 1 ntg query
+- multi client
+- rm email
+- send matrix msg on config change
 - setup pa on element
 - configurable email interval
 - jitter on intervals, including dedicated err span