try another cookie format, ntg submits ntgauthtoken as cookie

.gitignore

@@ -4,3 +4,4 @@ cmd/cmd
 cmd/cli
 cmd/pttodo/pttodo
 /truckstop
+/exec-truckstop

broker/broker.go

@@ -2,9 +2,13 @@ package broker
 
 import (
 	"context"
+	"encoding/json"
+	"local/storage"
 	"local/truckstop/config"
+	"local/truckstop/logtr"
 	"net/http"
 	"strings"
+	"time"
 
 	"golang.org/x/time/rate"
 )
@@ -20,9 +24,77 @@ type Broker interface {
 }
 
 func do(r *http.Request) (*http.Response, error) {
+	return _do(config.Get().DB(), r)
+}
+
+func _do(db storage.DB, r *http.Request) (*http.Response, error) {
 	limiter.Wait(context.Background())
 	if strings.Contains(strings.ToLower(r.URL.Path), "login") {
 		authlimiter.Wait(context.Background())
 	}
-	return http.DefaultClient.Do(r)
+	client := &http.Client{
+		Timeout: time.Hour,
+	}
+
+	cookieJarKey := "cookies_" + r.URL.Host
+	cookies, err := getCookies(db, cookieJarKey)
+	if err != nil {
+		logtr.Errorf("failed to get cookies: %v", err)
+	} else {
+		logtr.Verbosef("got cookies for %s: %+v", cookieJarKey, cookies)
+		for _, cookie := range cookies {
+			r.Header.Add("Cookie", cookie)
+		}
+	}
+	logtr.Verbosef("_do: %+v", r)
+	resp, err := client.Do(r)
+	if err != nil {
+		return nil, err
+	}
+	if err := setCookies(db, cookieJarKey, resp); err != nil {
+		logtr.Errorf("failed to set cookies: %v", err)
+	}
+	return resp, err
+}
+
+func getCookies(db storage.DB, host string) ([]string, error) {
+	b, err := db.Get(host)
+	if err != nil {
+		return nil, err
+	}
+	var result []string
+	err = json.Unmarshal(b, &result)
+	return result, err
+}
+
+func setCookies(db storage.DB, host string, resp *http.Response) error {
+	nameValues := [][2]string{}
+	old, _ := getCookies(db, host)
+	for _, value := range old {
+		name := strings.Split(value, "=")[0]
+		nameValues = append(nameValues, [2]string{name, value})
+	}
+	for _, value := range resp.Header["Set-Cookie"] {
+		name := strings.Split(value, "=")[0]
+		found := false
+		for i := range nameValues {
+			if nameValues[i][0] == name {
+				found = true
+				nameValues[i][1] = value
+			}
+		}
+		if !found {
+			nameValues = append(nameValues, [2]string{name, value})
+		}
+	}
+	result := []string{}
+	for i := range nameValues {
+		result = append(result, nameValues[i][1])
+	}
+	logtr.Verbosef("setting cookies for %s: %+v", host, result)
+	b, err := json.Marshal(result)
+	if err != nil {
+		return err
+	}
+	return db.Set(host, b)
 }

broker/broker_test.go

@@ -0,0 +1,69 @@
+package broker
+
+import (
+	"fmt"
+	"local/storage"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+func TestBrokerInterface(t *testing.T) {
+	var b Broker
+	b = NTGVision{}
+	_ = b
+}
+
+func TestDoCookies(t *testing.T) {
+	limiter = rate.NewLimiter(rate.Limit(20.0), 1)
+	calls := 0
+	db := storage.NewMap()
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls += 1
+		if calls != 1 {
+			if !strings.Contains(fmt.Sprint(r.Header["Cookie"]), "name=value"+strconv.Itoa(calls-1)) {
+				w.WriteHeader(http.StatusBadRequest)
+				t.Error("cookie not set as latest")
+			}
+			if !strings.Contains(fmt.Sprint(r.Header["Cookie"]), "Expires") {
+				w.WriteHeader(http.StatusBadRequest)
+				t.Error("cookie not expiration: ", r.Header["Cookie"])
+			}
+		}
+		http.SetCookie(w, &http.Cookie{
+			Name:    "name",
+			Value:   "value" + strconv.Itoa(calls),
+			Expires: time.Now().Add(time.Hour),
+		})
+	}))
+	defer s.Close()
+	req, _ := http.NewRequest(http.MethodGet, s.URL, nil)
+
+	if cookies, err := getCookies(db, "cookies_"+req.URL.Host); err == nil {
+		t.Fatal(err)
+	} else if len(cookies) != 0 {
+		t.Fatal(cookies)
+	}
+
+	for i := 0; i < 3; i++ {
+		resp, err := _do(db, req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if resp.StatusCode != http.StatusOK {
+			t.Fatal(resp.StatusCode)
+		}
+		resp.Body.Close()
+
+		if cookies, err := getCookies(db, "cookies_"+req.URL.Host); err != nil {
+			t.Fatal(err)
+		} else if len(cookies) == 0 {
+			t.Fatal(cookies)
+		}
+	}
+}

broker/job.go

@@ -1,6 +1,7 @@
 package broker
 
 import (
+	"encoding/base64"
 	"fmt"
 	"local/truckstop/config"
 	"local/truckstop/logtr"
@@ -16,7 +17,8 @@ type Job struct {
 	Weight  int
 	Miles   int
 	Meta    string
-	secrets func() interface{} `json:"-"`
+	Pays    string
+	secrets func(j *Job) `json:"-"`
 }
 
 type JobLocation struct {
@@ -25,16 +27,23 @@ type JobLocation struct {
 	State string
 }
 
+func (j Job) UID() string {
+	return fmt.Sprintf(
+		"%v-%s-%s-%s-%s-%v",
+		j.ID,
+		j.Pickup.State,
+		base64.StdEncoding.EncodeToString([]byte(j.Pickup.City)),
+		j.Dropoff.State,
+		base64.StdEncoding.EncodeToString([]byte(j.Dropoff.City)),
+		j.Pickup.Date.Unix(),
+	)
+}
+
 func (j *Job) Secrets() {
 	if j.secrets == nil {
 		return
 	}
-	v := j.secrets()
-	j.secrets = nil
-	if v == nil {
-		return
-	}
-	j.Meta = fmt.Sprintf("%s %+v", j.Meta, v)
+	j.secrets(j)
 }
 
 func (j Job) String() string {
@@ -53,6 +62,19 @@ func (j JobLocation) String() string {
 	return fmt.Sprintf("%s, %s @ %s", j.City, j.State, j.Date.Format("Monday Jan 02"))
 }
 
+func (j Job) FormatMultilineTextDead() string {
+	return fmt.Sprintf(
+		"no longer available: %s,%s => %s,%s for $%v @%s",
+		j.Pickup.City,
+		j.Pickup.State,
+		j.Dropoff.City,
+		j.Dropoff.State,
+		j.Pays,
+		j.URI,
+	)
+
+}
+
 func (j Job) FormatMultilineText() string {
 	foo := func(client string) string {
 		return fmt.Sprintf(

broker/ntgvision.go

@@ -109,7 +109,7 @@ func (ntgJob *ntgVisionJob) JobInfo() (ntgVisionJobInfo, error) {
 
 func (ntg NTGVision) searchJob(id int64) (io.ReadCloser, error) {
 	time.Sleep(config.Get().Interval.JobInfo.Get())
-	request, err := http.NewRequest(http.MethodGet, fmt.Sprintf(config.Get().Brokers.NTG.LoadPageURIFormat, id), nil)
+	request, err := http.NewRequest(http.MethodGet, fmt.Sprintf(config.Get().Brokers.NTG.LoadPageAPIURIFormat, id), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -143,13 +143,14 @@ func (ntgJob *ntgVisionJob) Job() Job {
 		Miles:  ntgJob.Miles,
 		Weight: ntgJob.Weight,
 		Meta:   fmt.Sprintf("equipment:%s", ntgJob.Equipment),
-		secrets: func() interface{} {
+		secrets: func(j *Job) {
 			jobInfo, err := ntgJob.JobInfo()
 			if err != nil {
 				logtr.Errorf("failed to get jobinfo: %v", err)
-				return nil
+				return
 			}
-			return jobInfo.String()
+			j.Meta = jobInfo.String()
+			j.Pays = fmt.Sprint(jobInfo.PayUpTo)
 		},
 	}
 }
@@ -174,8 +175,15 @@ func (ntg NTGVision) SearchJob(id int64) (ntgVisionJobInfo, error) {
 		return ntgVisionJobInfo{}, err
 	}
 	defer rc.Close()
+	b, err := ioutil.ReadAll(rc)
+	if err != nil {
+		return ntgVisionJobInfo{}, fmt.Errorf("failed to readall search job result: %w", err)
+	}
 	var result ntgVisionJobInfo
-	err = json.NewDecoder(rc).Decode(&result)
+	err = json.Unmarshal(b, &result)
+	if err != nil {
+		err = fmt.Errorf("failed to parse job info: %w: %s", err, b)
+	}
 	return result, err
 }
 
@@ -220,12 +228,14 @@ func setNTGToken(token string) {
 
 func (ntg NTGVision) search(states []config.State) (io.ReadCloser, error) {
 	if getNTGToken() == "" {
+		logtr.Debugf("NTG token is empty, refreshing ntg auth")
 		if err := ntg.refreshAuth(); err != nil {
 			return nil, err
 		}
 	}
 	rc, err := ntg._search(states)
 	if err == ErrNoAuth {
+		logtr.Debugf("err no auth on search, refreshing ntg auth")
 		if err := ntg.refreshAuth(); err != nil {
 			return nil, err
 		}
@@ -287,7 +297,11 @@ func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
 	if resp.StatusCode != http.StatusOK {
 		b, _ := ioutil.ReadAll(resp.Body)
 		resp.Body.Close()
-		if resp.StatusCode > 400 && resp.StatusCode < 500 && resp.StatusCode != 404 && resp.StatusCode != 410 {
+		request2, _ := ntg.newRequest(states)
+		requestb, _ := ioutil.ReadAll(request2.Body)
+		logtr.Debugf("ntg auth bad status: url=%s, status=%v, body=%s, headers=%+v, request=%+v, requestb=%s", request.URL.String(), resp.StatusCode, b, resp.Header, request2, requestb)
+		if resp.StatusCode > 400 && resp.StatusCode < 404 {
+			logtr.Debugf("ntg auth bad status: err no auth")
 			return nil, ErrNoAuth
 		}
 		return nil, fmt.Errorf("bad status searching ntg: %d: %s", resp.StatusCode, b)
@@ -297,7 +311,7 @@ func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
 
 func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) {
 	body, err := json.Marshal(map[string]interface{}{
-		"OriginFromDate":        time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
+		"OriginFromDate":        time.Now().Add(time.Hour * -24).UTC().Format("2006-01-02T15:04:05.000Z"),
 		"OriginToDate":          time.Now().UTC().Add(time.Hour * 24 * 30).Format("2006-01-02T15:04:05.000Z"),
 		"DestinationFromDate":   nil,
 		"DestinationToDate":     nil,
@@ -321,6 +335,7 @@ func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) {
 	}
 	setNTGHeaders(request)
 	request.Header.Set("Authorization", "Bearer "+getNTGToken())
+	request.Header.Set("Cookie", "NTGAuthToken="+getNTGToken())
 
 	return request, nil
 }

broker/testdata/ntgvision_response.json

@@ -1,21 +1,4 @@
 [
-  {
-    "id": 4650337,
-    "sDate": "01/12/22",
-    "sCity": "Advance",
-    "sState": "NC",
-    "sdh": null,
-    "cDate": "01/13/22",
-    "cCity": "Sacramento",
-    "cState": "CA",
-    "cdh": null,
-    "stopCnt": 2,
-    "miles": 578,
-    "weight": 5000,
-    "equip": "Str Truck W/ Lift Gate",
-    "temp": "",
-    "alertReasons": []
-  },
   {
     "id": 4650338,
     "sDate": "01/12/22",

config.json

@@ -1,7 +1,7 @@
 {
    "Log": {
       "Path": "/tmp/truckstop.log",
-      "Level": "error",
+      "Level": "debug",
       "SOSMatrix": {
          "ReceiveEnabled": true,
          "Mock": false,
@@ -34,7 +34,7 @@
       "Pickup": false,
       "Dropoff": false,
       "Pathed": {
-         "Enabled": false,
+         "Enabled": true,
          "DirectionsURIFormat": "https://maps.googleapis.com/maps/api/directions/json?origin=%s\u0026destination=%s\u0026mode=driving\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
          "PathedURIFormat": "https://maps.googleapis.com/maps/api/staticmap?size=250x250\u0026path=%s\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg\u0026size=250x250\u0026markers=%s|%s",
          "Zoom": {
@@ -69,14 +69,15 @@
          "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
       }
    },
-   "Once": false,
+   "Once": true,
    "Brokers": {
       "NTG": {
          "JobInfo": true,
-         "Mock": false,
+         "Mock": true,
          "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",
+         "LoadPageAPIURIFormat": "https://ntgvision.com/api/v1/load/LoadDetails?loadId==%d",
          "Username": "noeasyrunstrucking@gmail.com",
-         "Password": "thumper1234"
+         "Password": "thumper12345"
       }
    }
 }

config.main_test.json

@@ -0,0 +1,83 @@
+{
+   "Log": {
+      "Path": "/tmp/truckstop.log",
+      "Level": "sos",
+      "SOSMatrix": {
+         "ReceiveEnabled": true,
+         "Mock": true,
+         "Homeserver": "https://m.bltrucks.top",
+         "Username": "@bot.m.bltrucks.top",
+         "Token": "mvDWB96KXMF8XhOam8EC5XVdQvSEw0CDeClcSWocBcYkwZX3FPNWZ5uOnQk2EmT1cjpzfeuD7gDYPPjOuyZlI3bE9TE35UjNOlZgi0Tugm25s91iVsbIF6kMZsCIhVMSmEf6w3jxX6wQYOWvmDZ4mu6f5c8wr221EMDcOpEzQV09d1zuBSWgKLBgjqAkYHJZ5dTRIWpEDpPgujhOFZa2ld1HiAOxrJKlIrlfDBN0CUsTlGOGplujDAr4VtpFzNRS",
+         "Device": "TGNIOGKATZ",
+         "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
+      }
+   },
+   "Interval": {
+      "Input": "5s..10s",
+      "OK": "6h0m0s..6h0m0s",
+      "Error": "6h0m0s..6h0m0s",
+      "JobInfo": "10s..20s"
+   },
+   "Images": {
+      "ClientID": "d9ac7cabe813d10",
+      "ClientSecret": "9d0b3d82800b30ca88f595d3bcd6985f627d7d82",
+      "RefreshToken": "171417741bf762b99b0b9f9137491b7a69874a77",
+      "AccessToken": "e63db98f92d2db7ac7f56914a2030c889b378e9b",
+      "RefreshURI": "https://api.imgur.com/oauth2/token",
+      "RefreshFormat": "refresh_token=%s\u0026client_id=%s\u0026client_secret=%s\u0026grant_type=refresh_token",
+      "RefreshMethod": "POST",
+      "UploadURI": "https://api.imgur.com/3/image?name=something.jpeg",
+      "UploadMethod": "POST"
+   },
+   "Maps": {
+      "URIFormat": "https://maps.googleapis.com/maps/api/staticmap?center=%s\u0026markers=label=A|%s\u0026zoom=5\u0026size=250x250\u0026scale=1\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
+      "Pickup": false,
+      "Dropoff": false,
+      "Pathed": {
+         "Enabled": true,
+         "DirectionsURIFormat": "https://maps.googleapis.com/maps/api/directions/json?origin=%s\u0026destination=%s\u0026mode=driving\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
+         "PathedURIFormat": "https://maps.googleapis.com/maps/api/staticmap?size=250x250\u0026path=%s\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg\u0026size=250x250\u0026markers=%s|%s",
+         "Zoom": {
+            "AcceptableLatLngDelta": 2,
+            "Override": 5
+         }
+      }
+   },
+   "Clients": {
+      "bel": {
+         "States": [
+            "NC"
+         ],
+         "IDs": {
+            "Matrix": "@bel:m.bltrucks.top"
+         },
+         "Available": 1512328400
+      }
+   },
+   "Storage": [
+      "files",
+      "/tmp/play"
+   ],
+   "Message": {
+      "Matrix": {
+         "ReceiveEnabled": true,
+         "Mock": true,
+         "Homeserver": "https://m.bltrucks.top",
+         "Username": "@bot.m.bltrucks.top",
+         "Token": "mvDWB96KXMF8XhOam8EC5XVdQvSEw0CDeClcSWocBcYkwZX3FPNWZ5uOnQk2EmT1cjpzfeuD7gDYPPjOuyZlI3bE9TE35UjNOlZgi0Tugm25s91iVsbIF6kMZsCIhVMSmEf6w3jxX6wQYOWvmDZ4mu6f5c8wr221EMDcOpEzQV09d1zuBSWgKLBgjqAkYHJZ5dTRIWpEDpPgujhOFZa2ld1HiAOxrJKlIrlfDBN0CUsTlGOGplujDAr4VtpFzNRS",
+         "Device": "TGNIOGKATZ",
+         "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
+      }
+   },
+   "Once": true,
+   "Brokers": {
+      "NTG": {
+         "JobInfo": true,
+         "Mock": true,
+         "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",
+         "LoadPageAPIURIFormat": "https://ntgvision.com/api/v1/load/LoadDetails?loadId==%d",
+         "Username": "noeasyrunstrucking@gmail.com",
+         "Password": "thumper1234"
+      }
+   }
+}

config/config.go

@@ -65,11 +65,17 @@ type Config struct {
 	Once    bool
 	Brokers struct {
 		NTG struct {
-			JobInfo           bool
-			Mock              bool
-			LoadPageURIFormat string
-			Username          string
-			Password          string
+			JobInfo              bool
+			Mock                 bool
+			LoadPageURIFormat    string
+			LoadPageAPIURIFormat string
+			Username             string
+			Password             string
+		}
+		FastExact struct {
+			Mock     bool
+			Username string
+			Password string
 		}
 	}
 

config/state.go

@@ -7,56 +7,56 @@ import (
 
 type State string
 
-var States = map[string]State{
-	"99654": "AL",
-	"72401": "AR",
-	"85364": "AZ",
-	"90011": "CA",
-	"80013": "CO",
-	"06902": "CT",
-	"19720": "DE",
-	"33024": "FL",
-	"30043": "GA",
-	"96706": "HI",
-	"50613": "IA",
-	"83646": "ID",
-	"60629": "IL",
-	"47906": "IN",
-	"66062": "KS",
-	"40475": "KY",
-	"70726": "LA",
-	"02301": "MA",
-	"20906": "MD",
-	"04401": "ME",
-	"48197": "MI",
-	"55106": "MN",
-	"63376": "MO",
-	"39503": "MS",
-	"59901": "MT",
-	"27006": "NC",
-	"58103": "ND",
-	"68516": "NE",
-	"03103": "NH",
-	"08701": "NJ",
-	"87121": "NM",
-	"89108": "NV",
-	"11368": "NY",
-	"45011": "OH",
-	"73099": "OK",
-	"97006": "OR",
-	"19120": "PA",
-	"02860": "RI",
-	"29483": "SC",
-	"57106": "SD",
-	"37013": "TN",
-	"77449": "TX",
-	"84058": "UT",
-	"22193": "VA",
-	"05401": "VT",
-	"99301": "WA",
-	"53215": "WI",
-	"26554": "WV",
-	"82001": "WY",
+var States = map[State]string{
+	"AL": "99654",
+	"AR": "72401",
+	"AZ": "85364",
+	"CA": "90011",
+	"CO": "80013",
+	"CT": "06902",
+	"DE": "19720",
+	"FL": "33024",
+	"GA": "30043",
+	"HI": "96706",
+	"IA": "50613",
+	"ID": "83646",
+	"IL": "60629",
+	"IN": "47906",
+	"KS": "66062",
+	"KY": "40475",
+	"LA": "70726",
+	"MA": "02301",
+	"MD": "20906",
+	"ME": "04401",
+	"MI": "48197",
+	"MN": "55106",
+	"MO": "63376",
+	"MS": "39503",
+	"MT": "59901",
+	"NC": "27006",
+	"ND": "58103",
+	"NE": "68516",
+	"NH": "03103",
+	"NJ": "08701",
+	"NM": "87121",
+	"NV": "89108",
+	"NY": "11368",
+	"OH": "45011",
+	"OK": "73099",
+	"OR": "97006",
+	"PA": "19120",
+	"RI": "02860",
+	"SC": "29483",
+	"SD": "57106",
+	"TN": "37013",
+	"TX": "77449",
+	"UT": "84058",
+	"VA": "22193",
+	"VT": "05401",
+	"WA": "99301",
+	"WI": "53215",
+	"WV": "26554",
+	"WY": "82001",
 }
 
 func (state *State) UnmarshalJSON(b []byte) error {
@@ -65,8 +65,8 @@ func (state *State) UnmarshalJSON(b []byte) error {
 		return err
 	}
 	for k := range States {
-		if string(States[k]) == s {
-			*state = States[k]
+		if string(k) == s {
+			*state = k
 			return nil
 		}
 	}

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/golang/protobuf v1.2.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/gomodule/redigo v1.8.5 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect
 	github.com/json-iterator/go v1.1.9 // indirect
 	github.com/klauspost/compress v1.9.5 // indirect

logtr/log.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"strings"
 	"sync"
@@ -83,16 +82,18 @@ func SetLevel(l Level) {
 }
 
 func logf(l Level, format string, args []interface{}) {
-	format = fmt.Sprintf("%v: %v: %s\n", time.Now().Format("15:04:05"), l.String(), strings.TrimSpace(format))
+	format = fmt.Sprintf("%v: %v: %s\n", time.Now().Format("01-02T15:04:05"), l.String(), strings.TrimSpace(format))
+	logContent := strings.ReplaceAll(fmt.Sprintf(format, args...), "\n", "") + "\n"
 	cLevel := level
 	cAnsoser := ansoser
 	if l >= cLevel {
-		fmt.Fprintf(os.Stderr, format, args...)
+		fmt.Fprint(os.Stderr, logContent)
 	}
-	fmt.Fprintf(logger, format, args...)
-	log.Printf("l==sos=%v, ansoser!=nil=%v", l == SOS, ansoser != nil)
+	fmt.Fprint(logger, logContent)
 	if l == SOS && cAnsoser != nil {
-		cAnsoser.Send(fmt.Sprintf(format, args...))
+		if err := cAnsoser.Send(logContent); err != nil {
+			Errorf("failed to SOS: %v", err)
+		}
 	}
 }
 

main.go

@@ -21,12 +21,19 @@ import (
 var stateFinder = regexp.MustCompile(`[A-Za-z]+`)
 
 func main() {
-	if err := config.Refresh(message.NewSOSMatrix); err != nil {
+	if err := _main(); err != nil {
 		panic(err)
 	}
+}
+
+func _main() error {
+	if err := config.Refresh(message.NewSOSMatrix); err != nil {
+		return err
+	}
 	if config.Get().Message.Matrix.ReceiveEnabled {
 		if err := matrixrecv(); err != nil {
-			panic(err)
+			logtr.SOSf("failed to recv matrix on boot: %v", err)
+			return err
 		}
 	}
 	lock := &sync.Mutex{}
@@ -46,10 +53,12 @@ func main() {
 			}
 		}
 	}()
-	if err := _main(); err != nil {
-		panic(err)
+	if err := __main(); err != nil {
+		logtr.SOSf("failed __main: %v", err)
+		return err
 	}
 	lock.Lock()
+	return nil
 }
 
 func matrixrecv() error {
@@ -226,14 +235,14 @@ func parseOutStates(b []byte) []config.State {
 	return states
 }
 
-func _main() error {
+func __main() error {
 	for {
-		err := _mainOne()
+		err := __mainOne()
 		if err != nil {
 			logtr.Errorf("failed _main: %v", err)
 		}
 		if config.Get().Once {
-			time.Sleep(time.Second)
+			time.Sleep(3 * time.Second)
 			return err
 		}
 		if err != nil {
@@ -245,7 +254,7 @@ func _main() error {
 	return nil
 }
 
-func _mainOne() error {
+func __mainOne() error {
 	logtr.Debugf("config.refreshing...")
 	if err := config.Refresh(message.NewSOSMatrix); err != nil {
 		logtr.SOSf("bad config: %v", err)
@@ -256,7 +265,7 @@ func _mainOne() error {
 		logtr.SOSf("failed once(): %v", err)
 		return err
 	}
-	logtr.Debugf("/_mainOne")
+	logtr.Debugf("/__mainOne")
 	return nil
 }
 
@@ -265,6 +274,11 @@ func once() error {
 	if err != nil {
 		return err
 	}
+	logtr.Debugf("once: update dead jobs: %+v", alljobs)
+	err = updateDeadJobs(alljobs)
+	if err != nil {
+		return err
+	}
 	logtr.Debugf("once: all jobs: %+v", alljobs)
 	newjobs, err := dropStaleJobs(alljobs)
 	if err != nil {
@@ -275,17 +289,22 @@ func once() error {
 	if err != nil {
 		return err
 	}
+	logtr.Debugf("found jobs: %+v", jobs)
+	if len(jobs) == 0 {
+		return nil
+	}
 	logtr.Debugf("once: loading job secrets: %+v", jobs)
 	for i := range jobs {
 		jobs[i].Secrets()
 	}
 	logtr.Infof("once: sending jobs: %+v", jobs)
+	db := config.Get().DB()
 	for i := range jobs {
 		if ok, err := sendJob(jobs[i]); err != nil {
 			return err
 		} else if ok {
 			logtr.Debugf("sent job", jobs[i])
-			if err := config.Get().DB().Set(jobs[i].ID, []byte(`sent`)); err != nil {
+			if err := db.Set(jobs[i].UID(), []byte(`sent`)); err != nil {
 				return err
 			}
 		}
@@ -308,10 +327,61 @@ func getJobs() ([]broker.Job, error) {
 	return jobs, nil
 }
 
+type recordedJob struct {
+	Job            broker.Job
+	SentNS         int64
+	MatrixID       string
+	MatrixImageIDs []string
+}
+
+func updateDeadJobs(jobs []broker.Job) error {
+	db := config.Get().DB()
+	list, err := db.List([]string{}, "sent_job_", "sent_job_}}")
+	if err != nil {
+		return err
+	}
+	for _, listEntry := range list {
+		wouldBe := strings.TrimPrefix(listEntry, "sent_job_")
+		found := false
+		for i := range jobs {
+			found = found || jobs[i].ID == wouldBe
+		}
+		logtr.Debugf("found job %s to be still alive==%v", wouldBe, found)
+		if !found {
+			logtr.Debugf("updating dead job %+v", listEntry)
+			b, err := db.Get(listEntry)
+			if err != nil {
+				return err
+			}
+			var recorded recordedJob
+			if err := json.Unmarshal(b, &recorded); err != nil {
+				return err
+			}
+			/* // TODO this beeps on fluffychat
+			if err := message.NewMatrix().Update(recorded.MatrixID, recorded.Job.FormatMultilineTextDead()); err != nil {
+				return err
+			}
+			*/
+			if err := message.NewMatrix().Remove(recorded.MatrixID); err != nil {
+				return err
+			}
+			if err := db.Set(listEntry, nil); err != nil {
+				return err
+			}
+			for _, imageid := range recorded.MatrixImageIDs {
+				if err := message.NewMatrix().Remove(imageid); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func dropStaleJobs(jobs []broker.Job) ([]broker.Job, error) {
 	db := config.Get().DB()
 	for i := len(jobs) - 1; i >= 0; i-- {
-		if _, err := db.Get(jobs[i].ID); err == storage.ErrNotFound {
+		if _, err := db.Get(jobs[i].UID()); err == storage.ErrNotFound {
 		} else if err != nil {
 			return nil, err
 		} else {
@@ -333,9 +403,27 @@ func sendJob(job broker.Job) (bool, error) {
 	if len(payload) == 0 {
 		return false, nil
 	}
-	if err := sender.Send(payload); err != nil {
+	id, err := sender.SendTracked(payload)
+	if err != nil {
 		return false, err
 	}
+	recordedJob := recordedJob{
+		Job:      job,
+		SentNS:   time.Now().UnixNano(),
+		MatrixID: id,
+	}
+	defer func() {
+		db := config.Get().DB()
+		b, err := json.Marshal(recordedJob)
+		if err != nil {
+			logtr.Errorf("failed to marshal recorded job: %v", err)
+			return
+		}
+		if err := db.Set("sent_job_"+job.ID, b); err != nil {
+			logtr.Errorf("failed to set recorded job: %v", err)
+			return
+		}
+	}()
 	maps := config.Get().Maps
 	pickup := fmt.Sprintf("%s,%s", url.QueryEscape(job.Pickup.City), job.Pickup.State)
 	dropoff := fmt.Sprintf("%s,%s", url.QueryEscape(job.Dropoff.City), job.Dropoff.State)
@@ -393,24 +481,30 @@ func sendJob(job broker.Job) (bool, error) {
 				uri = fmt.Sprintf("%s&zoom=%d", uri, maps.Pathed.Zoom.Override)
 			}
 			logtr.Debugf("sending pathed image: %s", uri)
-			if err := sender.SendImage(uri); err != nil {
+			pathedid, err := sender.SendImageTracked(uri)
+			if err != nil {
 				return true, err
 			}
+			recordedJob.MatrixImageIDs = append(recordedJob.MatrixImageIDs, pathedid)
 		}
 	}
 	if maps.Pickup {
 		uri := fmt.Sprintf(maps.URIFormat, pickup, pickup)
 		logtr.Debugf("sending pickup image: %s", uri)
-		if err := sender.SendImage(uri); err != nil {
+		pickupid, err := sender.SendImageTracked(uri)
+		if err != nil {
 			return true, err
 		}
+		recordedJob.MatrixImageIDs = append(recordedJob.MatrixImageIDs, pickupid)
 	}
 	if maps.Dropoff {
 		uri := fmt.Sprintf(maps.URIFormat, dropoff, dropoff)
 		logtr.Debugf("sending dropoff image: %s", uri)
-		if err := sender.SendImage(uri); err != nil {
+		dropid, err := sender.SendImageTracked(uri)
+		if err != nil {
 			return true, err
 		}
+		recordedJob.MatrixImageIDs = append(recordedJob.MatrixImageIDs, dropid)
 	}
 	return true, nil
 }

main_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"local/truckstop/config"
+	"os"
 	"testing"
 )
 
@@ -33,3 +34,10 @@ Content-Type: text/html; charset="UTF-8"
 		t.Fatal(want, got)
 	}
 }
+
+func TestMain(t *testing.T) {
+	os.Setenv("CONFIG", "./config.main_test.json")
+	if err := _main(); err != nil {
+		t.Fatal(err)
+	}
+}

message/matrix.go

@@ -87,6 +87,7 @@ func (m *Matrix) Receive() ([]Message, error) {
 	logtr.Debugf("%s => {Start:%s End:%v};; %v, (%d)", m.Continuation(), result.Start, result.End, err, len(result.Chunk))
 	m.continuation = result.End
 	for _, event := range result.Chunk {
+		logtr.Verbosef("matrix event: %+v", event)
 		if _, ok := matrixIDs[event.Sender]; !ok {
 			continue
 		}
@@ -124,50 +125,114 @@ func (m *Matrix) Receive() ([]Message, error) {
 	return messages, nil
 }
 
-func (m Matrix) Send(text string) error {
+func (m Matrix) Remove(id string) error {
 	if m.mock {
-		logtr.Infof("matrix.Send(%s)", text)
+		logtr.Infof("matrix.Remove(%s)", id)
 		return nil
 	}
 	c, err := m.getclient()
 	if err != nil {
 		return err
 	}
-	_, err = c.SendText(m.room, text)
+	_, err = c.RedactEvent(m.room, id, &gomatrix.ReqRedact{Reason: "stale"})
 	return err
 }
 
+func (m Matrix) Update(id, text string) error {
+	if m.mock {
+		logtr.Infof("matrix.Update(%s)", text)
+		return nil
+	}
+	c, err := m.getclient()
+	if err != nil {
+		return err
+	}
+	type MRelatesTo struct {
+		EventID string `json:"event_id"`
+		RelType string `json:"rel_type"`
+	}
+	type NewContent struct {
+		Body    string `json:"body"`
+		MsgType string `json:"msgtype"`
+	}
+	type RelatesToRoomMessage struct {
+		Body        string     `json:"body"`
+		MsgType     string     `json:"msgtype"`
+		MRelatesTo  MRelatesTo `json:"m.relates_to"`
+		MNewContent NewContent `json:"m.new_content"`
+	}
+	_, err = c.SendMessageEvent(m.room, "m.room.message", RelatesToRoomMessage{
+		Body:    "",
+		MsgType: "m.text",
+		MNewContent: NewContent{
+			Body:    text,
+			MsgType: "m.text",
+		},
+		MRelatesTo: MRelatesTo{
+			EventID: id,
+			RelType: "m.replace",
+		},
+	})
+	return err
+}
+
+func (m Matrix) Send(text string) error {
+	_, err := m.SendTracked(text)
+	return err
+}
+
+func (m Matrix) SendTracked(text string) (string, error) {
+	if m.mock {
+		logtr.Infof("matrix.SendTracked(%s)", text)
+		return "", nil
+	}
+	c, err := m.getclient()
+	if err != nil {
+		return "", err
+	}
+	resp, err := c.SendText(m.room, text)
+	if err != nil {
+		return "", err
+	}
+	return resp.EventID, nil
+}
+
 func (m Matrix) SendImage(uri string) error {
+	_, err := m.SendImageTracked(uri)
+	return err
+}
+
+func (m Matrix) SendImageTracked(uri string) (string, error) {
 	if m.mock {
 		logtr.Infof("matrix.SendImage(%s)", uri)
-		return nil
+		return "", nil
 	}
 	response, err := http.Get(uri)
 	if err != nil {
-		return err
+		return "", err
 	}
 	if response.StatusCode != http.StatusOK {
 		b, _ := ioutil.ReadAll(response.Body)
 		response.Body.Close()
-		return fmt.Errorf("failed to get %s: (%d) %s", uri, response.StatusCode, b)
+		return "", fmt.Errorf("failed to get %s: (%d) %s", uri, response.StatusCode, b)
 	}
 	b, err := ioutil.ReadAll(response.Body)
 	response.Body.Close()
 	if err != nil {
-		return err
+		return "", err
 	}
 	c, err := m.getclient()
 	if err != nil {
-		return err
+		return "", err
 	}
 	mediaUpload, err := c.UploadToContentRepo(bytes.NewReader(b), "image/jpeg", int64(len(b)))
 	if err != nil {
-		return err
+		return "", err
 	}
 	publicURI := mediaUpload.ContentURI
 	resp, err := c.SendImage(m.room, "img", publicURI)
 	logtr.Debugf("sent image %s => %s: %+v", uri, publicURI, resp)
-	return err
+	return resp.EventID, err
 }
 
 func SetMatrixContinuation(continuation string) {

message/matrix_test.go

@@ -1,57 +1,65 @@
 package message
 
 import (
-	"encoding/json"
 	"io/ioutil"
 	"local/truckstop/config"
 	"os"
+	"path"
 	"testing"
+
+	"github.com/google/uuid"
 )
 
-func TestMatrixSend(t *testing.T) {
-	if len(os.Getenv("INTEGRATION")) == 0 {
-		t.Skip("$INTEGRATION not set")
-	}
-	var c config.Config
-	b, err := ioutil.ReadFile("../config.json")
-	if err != nil {
+func TestMatrixSendDel(t *testing.T) {
+	sender := testMatrix(t)
+	if id, err := sender.SendTracked("hello world from unittest"); err != nil {
 		t.Fatal(err)
-	}
-	if err := json.Unmarshal(b, &c); err != nil {
-		t.Fatal(err)
-	}
-	var sender Sender = &Matrix{
-		homeserver: c.Message.Matrix.Homeserver,
-		username:   c.Message.Matrix.Username,
-		token:      c.Message.Matrix.Token,
-		room:       c.Message.Matrix.Room,
-	}
-	if err := sender.Send("hello world from unittest"); err != nil {
+	} else if err := sender.Remove(id); err != nil {
 		t.Fatal(err)
 	}
 }
 
-func TestMatrixReceive(t *testing.T) {
-	if len(os.Getenv("INTEGRATION")) == 0 {
-		t.Skip("$INTEGRATION not set")
-	}
-	var c config.Config
-	b, err := ioutil.ReadFile("../config.json")
+func TestMatrixUpdate(t *testing.T) {
+	// 19:32:34: VER: matrix event: {StateKey:<nil> Sender:@bot:m.bltrucks.top Type:m.room.message Timestamp:1642471594729 ID:$n0ln9TFZrko_cBNFXeh8YyICZ3fFm17Jhz3bmZcqig4 RoomID:!OYZqtInrBCn1cyz90D:m.bltrucks.top Redacts: Unsigned:map[transaction_id:go1642471594571852423] Content:map[body:hello world from unittest format: formatted_body: msgtype:m.text] PrevContent:map[]}
+	// func (m Matrix) Update(id, text string) error {
+	m := testMatrix(t)
+	uid := uuid.New().String()[:3]
+	id, err := m.SendTracked("hello, heheheh from test matrix update " + uid)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := json.Unmarshal(b, &c); err != nil {
+	err = m.Update(id, "heheheh i updated from test matrix update "+uid)
+	if err != nil {
 		t.Fatal(err)
 	}
-	var sender Sender = &Matrix{
-		homeserver: c.Message.Matrix.Homeserver,
-		username:   c.Message.Matrix.Username,
-		token:      c.Message.Matrix.Token,
-		room:       c.Message.Matrix.Room,
-	}
+	m.Receive()
+}
+
+func TestMatrixReceive(t *testing.T) {
+	sender := testMatrix(t)
 	if msgs, err := sender.Receive(); err != nil {
 		t.Fatal(err)
 	} else {
 		t.Logf("%+v", msgs)
 	}
 }
+
+func testMatrix(t *testing.T) Matrix {
+	if len(os.Getenv("INTEGRATION")) == 0 {
+		t.Skip("$INTEGRATION not set")
+	}
+	d := t.TempDir()
+	f := path.Join(d, "config.test.json")
+	b, err := ioutil.ReadFile("../config.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(f, b, os.ModePerm); err != nil {
+		t.Fatal(err)
+	}
+	os.Setenv("CONFIG", f)
+	if err := config.Refresh(nil); err != nil {
+		t.Fatal(err)
+	}
+	return NewMatrix()
+}

message/message.go

@@ -4,6 +4,8 @@ import "time"
 
 type Sender interface {
 	Send(string) error
+	SendTracked(string) (string, error)
+	Update(string, string) (string, error)
 	Receive() ([]Message, error)
 }
 

todo.yaml

@@ -1,11 +1,11 @@
 todo:
-- help() log on truckstop for stuff like perma 403
-- TEST its falling apart
-- mark jobs no longer avail by modifying in matrix
-- write-as for clients so ma can write to pa by default
+- !states emits current state 
+- TEST. Just like, refactor and test to shit.
+- try search ntg by autoinc?
+- test each !command callbacks to matrix
+- recv-as for clients so pa receives mas commands as writes
 - continuation is garbo, but I can still do better client side to avoid get-set high level
 - no hard code jpeg or have it in multiple places
-- test each !command callbacks to matrix
 - change matrix so I test my custom logic even if I dont fetch remote
 - warn/err/etc. on clobbering ids.matrix since clients can mess with one another
 - modify old items once no longer available; drop stale jobs good candidate but requires new matrix interaction
@@ -15,6 +15,9 @@ todo:
   - banlist criteria like vendors, brokers, metadata
 - set up copy for caleb, broc
 done:
+- mark jobs no longer avail by modifying in matrix;; save matrix ID over dummy payload
+- TEST its falling apart
+- help() log on truckstop for stuff like perma 403
 - figure out zoom on maps;; is there like an auto-zoom I can leverage?
 - mock is nigh useless
 - mark consumed;; save scroll id for matrix