mark state, pause msgs as processed

broker/broker.go

@@ -1,7 +1,28 @@
 package broker
 
-import "local/truckstop/config"
+import (
+	"context"
+	"local/truckstop/config"
+	"net/http"
+	"strings"
+
+	"golang.org/x/time/rate"
+)
+
+// once per minute
+var authlimiter = rate.NewLimiter(rate.Limit(1.0/60.0), 1)
+
+// thrice per minute
+var limiter = rate.NewLimiter(rate.Limit(1.0/20.0), 1)
 
 type Broker interface {
 	Search([]config.State) ([]Job, error)
 }
+
+func do(r *http.Request) (*http.Response, error) {
+	limiter.Wait(context.Background())
+	if strings.Contains(strings.ToLower(r.URL.Path), "login") {
+		authlimiter.Wait(context.Background())
+	}
+	return http.DefaultClient.Do(r)
+}

broker/job.go

@@ -2,6 +2,9 @@ package broker
 
 import (
 	"fmt"
+	"local/truckstop/config"
+	"log"
+	"strings"
 	"time"
 )
 
@@ -36,14 +39,29 @@ func (j JobLocation) String() string {
 }
 
 func (j Job) FormatMultilineText() string {
-	return fmt.Sprintf(
+	foo := func(client string) string {
-		"--- %s => %s ---\nPickup: %s\nDropoff: %s\nNotes: %d lbs, %d miles, %s",
+		return fmt.Sprintf(
-		j.Pickup.State,
+			"--- %s: %s => %s ---\nPickup: %s\nDropoff: %s\nNotes: %d lbs, %d miles, %s",
-		j.Dropoff.State,
+			client,
-		j.Pickup.String(),
+			j.Pickup.State,
-		j.Dropoff.String(),
+			j.Dropoff.State,
-		j.Weight,
+			j.Pickup.String(),
-		j.Miles,
+			j.Dropoff.String(),
-		j.Meta,
+			j.Weight,
-	)
+			j.Miles,
+			j.Meta,
+		)
+	}
+	out := ""
+	clients := config.Clients(j.Pickup.Date)
+	for k := range clients {
+		log.Printf("job multiline: %+v contains %s then use %v", clients[k].States, j.Pickup.State, k)
+		if strings.Contains(fmt.Sprint(clients[k].States), j.Pickup.State) {
+			if len(out) > 0 {
+				out += "\n\n"
+			}
+			out += foo(k)
+		}
+	}
+	return out
 }

broker/ntgvision.go

@@ -109,7 +109,7 @@ func (ntg NTGVision) refreshAuth() error {
 		return err
 	}
 	setNTGHeaders(request)
-	resp, err := http.DefaultClient.Do(request)
+	resp, err := do(request)
 	if err != nil {
 		return err
 	}
@@ -138,15 +138,14 @@ func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
 	if err != nil {
 		return nil, err
 	}
-	c := http.Client{Timeout: time.Minute}
+	resp, err := do(request)
-	resp, err := c.Do(request)
 	if err != nil {
 		return nil, err
 	}
 	if resp.StatusCode != http.StatusOK {
 		b, _ := ioutil.ReadAll(resp.Body)
 		resp.Body.Close()
-		if resp.StatusCode > 400 && resp.StatusCode < 404 {
+		if resp.StatusCode > 400 && resp.StatusCode < 500 && resp.StatusCode != 404 && resp.StatusCode != 410 {
 			return nil, ErrNoAuth
 		}
 		return nil, fmt.Errorf("bad status searching ntg: %d: %s", resp.StatusCode, b)

broker/testdata/ntgvision_response.json

@@ -19,11 +19,11 @@
   {
     "id": 4650338,
     "sDate": "01/12/22",
-    "sCity": "Somewhere",
+    "sCity": "Advance",
-    "sState": "OH",
+    "sState": "NC",
     "sdh": null,
     "cDate": "01/13/22",
-    "cCity": "SomewhereElse",
+    "cCity": "Atlanta",
     "cState": "GA",
     "cdh": null,
     "stopCnt": 2,

config.json

@@ -1,26 +1,80 @@
 {
    "Interval": {
-      "OK": "6h0m0s",
+      "Input": "15s..20s",
-      "Error": "6h"
+      "OK": "6h0m0s..6h0m0s",
+      "Error": "6h0m0s..6h0m0s"
    },
-   "States": [
+   "Images": {
-      "GA"
+      "ClientID": "d9ac7cabe813d10",
-   ],
+      "ClientSecret": "9d0b3d82800b30ca88f595d3bcd6985f627d7d82",
-   "Storage": [
+      "RefreshToken": "171417741bf762b99b0b9f9137491b7a69874a77",
-      "map"
+      "AccessToken": "e63db98f92d2db7ac7f56914a2030c889b378e9b",
-   ],
+      "RefreshURI": "https://api.imgur.com/oauth2/token",
-   "Client": "breellocaldev@gmail.com",
+      "RefreshFormat": "refresh_token=%s\u0026client_id=%s\u0026client_secret=%s\u0026grant_type=refresh_token",
-   "Message": {
+      "RefreshMethod": "POST",
-      "Matrix": {
+      "UploadURI": "https://api.imgur.com/3/image",
-         "Mock": true,
+      "UploadMethod": "POST"
-         "Homeserver": "https://matrix-client.matrix.org",
+   },
-         "Username": "@breellocaldev:matrix.org",
+   "Maps": {
-         "Token": "syt_YnJlZWxsb2NhbGRldg_HTewKMMePdEcLvceAKEz_2fHsHa",
+      "URIFormat": "https://maps.googleapis.com/maps/api/staticmap?center=%s\u0026markers=label=A|%s\u0026zoom=5\u0026size=250x250\u0026scale=1\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
-         "Device": "TGNIOGKATZ",
+      "Pickup": true,
-         "Room": "!vVwjXhWXMxZtOwexKa:matrix.org"
+      "Dropoff": false
+   },
+   "Clients": {
+      "bel": {
+         "States": [
+            "NC"
+         ],
+         "IDs": {
+            "Matrix": "@bel:synapse.home.blapointe.com"
+         },
+         "Available": 1612328400
+      },
+      "broc": {
+         "States": [
+            "FL",
+            "NC"
+         ],
+         "IDs": {
+            "Matrix": "@belandbroc:matrix.org"
+         },
+         "Available": 5642452800
+      },
+      "caleb": {
+         "States": [
+            "FL"
+         ],
+         "IDs": {
+            "Matrix": "@belandbroc:matrix.org"
+         },
+         "Available": -62135596800
+      },
+      "pa": {
+         "States": [
+            "NC"
+         ],
+         "IDs": {
+            "Matrix": "@belandbroc:matrix.org"
+         },
+         "Available": -62135596800
       }
    },
-   "Once": true,
+   "Storage": [
+      "files",
+      "/tmp/play"
+   ],
+   "Message": {
+      "Matrix": {
+         "ReceiveEnabled": true,
+         "Mock": false,
+         "Homeserver": "https://synapse.home.blapointe.com",
+         "Username": "@bel:synapse.home.blapointe.com",
+         "Token": "MDAyOGxvY2F0aW9uIHN5bmFwc2UuaG9tZS5ibGFwb2ludGUuY29tCjAwMTNpZGVudGlmaWVyIGtleQowMDEwY2lkIGdlbiA9IDEKMDAzMmNpZCB1c2VyX2lkID0gQGJlbDpzeW5hcHNlLmhvbWUuYmxhcG9pbnRlLmNvbQowMDE2Y2lkIHR5cGUgPSBhY2Nlc3MKMDAyMWNpZCBub25jZSA9IFpXTTtCaTNBODdtSTpsZkAKMDAyZnNpZ25hdHVyZSBX37kStY2wt_ftMfEYPfk1ADMJ6ZVfZsro9ic3weZ25Ao",
+         "Device": "TGNIOGKATZ",
+         "Room": "!WbawbZxHqnxxfhvcSj:synapse.home.blapointe.com"
+      }
+   },
+   "Once": false,
    "Brokers": {
       "NTG": {
          "Mock": true,
@@ -28,14 +82,5 @@
          "Username": "noeasyrunstrucking@gmail.com",
          "Password": "thumper123"
       }
-   },
+   }
-   "Emailer": {
-      "From": "breellocaldev@gmail.com",
-      "SMTP": "",
-      "POP3": "",
-      "IMAP": "imap.gmail.com:993",
-      "Password": "gojfkkfrkmtxzyro",
-      "Limit": 0
-   },
-   "EmailerEnabled": false
 }

config/config.go

@@ -3,28 +3,45 @@ package config
 import (
 	"encoding/json"
 	"io/ioutil"
-	"local/sandbox/contact/contact"
 	"local/storage"
 	"os"
 	"sync"
+	"time"
 )
 
 type Config struct {
 	Interval struct {
+		Input Duration
 		OK    Duration
 		Error Duration
 	}
-	States  []State
+	Images struct {
+		ClientID      string
+		ClientSecret  string
+		RefreshToken  string
+		AccessToken   string
+		RefreshURI    string
+		RefreshFormat string
+		RefreshMethod string
+		UploadURI     string
+		UploadMethod  string
+	}
+	Maps struct {
+		URIFormat string
+		Pickup    bool
+		Dropoff   bool
+	}
+	Clients map[string]Client
 	Storage []string
-	Client  string
 	Message struct {
 		Matrix struct {
-			Mock       bool
+			ReceiveEnabled bool
-			Homeserver string
+			Mock           bool
-			Username   string
+			Homeserver     string
-			Token      string
+			Username       string
-			Device     string
+			Token          string
-			Room       string
+			Device         string
+			Room           string
 		}
 	}
 	Once    bool
@@ -36,13 +53,19 @@ type Config struct {
 			Password string
 		}
 	}
-	Emailer        contact.Emailer
-	EmailerEnabled bool
 
 	lock sync.Mutex
 	db   storage.DB
 }
 
+type Client struct {
+	States []State
+	IDs    struct {
+		Matrix string
+	}
+	Available Time
+}
+
 var live Config
 
 func configPath() string {
@@ -53,6 +76,31 @@ func configPath() string {
 	return p
 }
 
+func Clients(t time.Time) map[string]Client {
+	clients := Get().Clients
+	result := map[string]Client{}
+	for k := range clients {
+		if clients[k].Available.Get().IsZero() || t.After(clients[k].Available.Get()) {
+			result[k] = clients[k]
+		}
+	}
+	return result
+}
+
+func AllStates() []State {
+	statem := map[State]struct{}{}
+	for _, v := range Clients(time.Now().Add(time.Hour * 24 * 365)) {
+		for _, state := range v.States {
+			statem[state] = struct{}{}
+		}
+	}
+	states := make([]State, 0, len(statem)+1)
+	for k := range statem {
+		states = append(states, k)
+	}
+	return states
+}
+
 func Refresh() error {
 	b, err := ioutil.ReadFile(configPath())
 	if err != nil {

config/duration.go

@@ -14,6 +14,9 @@ type Duration struct {
 }
 
 func (d Duration) Get() time.Duration {
+	if d.least == 0 {
+		d.least = time.Second
+	}
 	jitter := d.most - d.least
 	if jitter >= time.Second {
 		jitter = time.Second * time.Duration(rand.Int()%int(jitter.Seconds()))

config/time.go

@@ -0,0 +1,23 @@
+package config
+
+import (
+	"encoding/json"
+	"time"
+)
+
+type Time time.Time
+
+func (t Time) Get() time.Time {
+	return time.Time(t)
+}
+
+func (t Time) MarshalJSON() ([]byte, error) {
+	return json.Marshal(t.Get().Unix())
+}
+
+func (t *Time) UnmarshalJSON(b []byte) error {
+	var d int64
+	err := json.Unmarshal(b, &d)
+	*t = Time(time.Unix(d, 0))
+	return err
+}

config/time_test.go

@@ -0,0 +1,22 @@
+package config
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+func TestTimeMarshal(t *testing.T) {
+	now := time.Now()
+	nowt := Time(now)
+	var other Time
+	if b, err := json.Marshal(nowt); err != nil {
+		t.Fatal(err)
+	} else if len(b) < 5 {
+		t.Fatal(string(b))
+	} else if err := json.Unmarshal(b, &other); err != nil {
+		t.Fatal(err)
+	} else if now.Unix() != other.Get().Unix() {
+		t.Fatal(other.Get().Unix(), now.Unix())
+	}
+}

go.mod

@@ -9,6 +9,7 @@ replace local/logb => ../logb
 replace local/sandbox/contact/contact => ../sandbox/contact/contact
 
 require (
+	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	local/sandbox/contact/contact v0.0.0-00010101000000-000000000000
 	local/storage v0.0.0-00010101000000-000000000000
 )
@@ -33,7 +34,6 @@ require (
 	github.com/json-iterator/go v1.1.9 // indirect
 	github.com/klauspost/compress v1.9.5 // indirect
 	github.com/klauspost/cpuid v1.2.3 // indirect
-	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 // indirect
 	github.com/minio/md5-simd v1.1.0 // indirect
 	github.com/minio/minio-go/v6 v6.0.57 // indirect
 	github.com/minio/sha256-simd v0.1.1 // indirect
@@ -46,22 +46,25 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.0.2 // indirect
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	go.mongodb.org/mongo-driver v1.7.2 // indirect
-	golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 // indirect
+	golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect
-	golang.org/x/net v0.0.0-20190522155817-f3200d17e092 // indirect
+	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
 	golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
-	golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2 // indirect
+	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
+	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
 	google.golang.org/api v0.0.0-20181120235003-faade3cbb06a // indirect
 	google.golang.org/appengine v1.3.0 // indirect
 	gopkg.in/ini.v1 v1.42.0 // indirect
-	gopkg.in/yaml.v2 v2.2.8 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	local/logb v0.0.0-00010101000000-000000000000 // indirect
 )

go.sum

@@ -184,13 +184,15 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/t3rm1n4l/go-mega v0.0.0-20190205172012-55a226cf41da/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0=
-github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
@@ -208,15 +210,18 @@ golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
+golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba h1:YDkOrzGLLYybtuP6ZgebnO4OWYEYVMFSniazXsxrFN8=
 golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -234,8 +239,13 @@ golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2 h1:T5DasATyLQfmbTpfEXx/IOL9vfjzW6up+ZDkmHvIf2s=
 golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -268,7 +278,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -4,13 +4,14 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"local/storage"
 	"local/truckstop/broker"
 	"local/truckstop/config"
 	"local/truckstop/message"
 	"log"
+	"net/url"
 	"regexp"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -19,18 +20,27 @@ import (
 var stateFinder = regexp.MustCompile(`[A-Za-z]+`)
 
 func main() {
+	if err := config.Refresh(); err != nil {
+		panic(err)
+	}
+	if err := matrixrecv(); err != nil {
+		panic(err)
+	}
 	lock := &sync.Mutex{}
 	go func() {
-		c := time.NewTicker(time.Minute)
+		for {
-		for range c.C {
+			time.Sleep(config.Get().Interval.Input.Get())
-			if !config.Get().EmailerEnabled {
+			if err := config.Refresh(); err != nil {
-				continue
+				log.Println(err)
+			} else {
+				if config.Get().Message.Matrix.ReceiveEnabled {
+					lock.Lock()
+					if err := matrixrecv(); err != nil {
+						log.Print(err)
+					}
+					lock.Unlock()
+				}
 			}
-			lock.Lock()
-			if err := email(); err != nil {
-				log.Print(err)
-			}
-			lock.Unlock()
 		}
 	}()
 	if err := _main(); err != nil {
@@ -39,39 +49,159 @@ func main() {
 	lock.Lock()
 }
 
-func email() error {
+func matrixrecv() error {
-	log.Printf("checking email...")
+	log.Printf("checking matrix...")
-	ch, err := config.Get().Emailer.ReadIMAP()
+	defer log.Printf("/checking matrix...")
-	if err != nil {
+	sender := message.NewMatrix()
-		ch, err = config.Get().Emailer.ReadIMAP()
+	messages, err := sender.Receive()
-	}
 	if err != nil {
 		return err
 	}
-	states := map[config.State]struct{}{}
+	func() {
-	for email := range ch {
+		log.Printf("looking for help")
-		if len(states) > 0 {
+		printed := false
+		for _, msg := range messages {
+			if !strings.HasPrefix(msg.Content, "!help") {
+				continue
+			}
+			key := fmt.Sprintf("help_%d", msg.Timestamp.Unix())
+			db := config.Get().DB()
+			if !printed {
+				if _, err := db.Get(key); err == storage.ErrNotFound {
+					log.Printf("sending help")
+					help := fmt.Sprintf("commands:\n...`!help`...print this help\n...`!state nc NC nC Nc`...set states for self\n...`!available 2022-12-31`...set date self is available for work\n\nrun a command for someone else: `!state ga @caleb`")
+					if err := sender.Send(help); err != nil {
+						log.Printf("failed to send help: %v", err)
+					} else {
+						printed = true
+						if err := db.Set(key, []byte{'k'}); err != nil {
+							log.Printf("failed to mark help given @%s: %v", key, err)
+						}
+					}
+				}
+			} else {
+				if err := db.Set(key, []byte{'k'}); err != nil {
+					log.Printf("failed to mark help given @%s: %v", key, err)
+				}
+			}
+		}
+	}()
+	func() {
+		log.Printf("looking for states")
+		db := config.Get().DB()
+		states := map[string]map[config.State]struct{}{}
+		for _, msg := range messages {
+			key := fmt.Sprintf("states_%d", msg.Timestamp.Unix())
+			if !strings.HasPrefix(msg.Content, "!state") {
+				continue
+			}
+			if _, ok := states[msg.Sender]; ok {
+				continue
+			}
+			if _, err := db.Get(key); err == storage.ErrNotFound {
+				states[msg.Sender] = map[config.State]struct{}{}
+				for _, state := range parseOutStates([]byte(msg.Content)) {
+					states[msg.Sender][state] = struct{}{}
+				}
+			}
+			if err := db.Set(key, []byte{'k'}); err != nil {
+				log.Printf("failed to mark state gathered @%s: %v", key, err)
+			}
+		}
+		setNewStates(states)
+	}()
+	func() {
+		log.Printf("looking for pauses")
+		db := config.Get().DB()
+		pauses := map[string]time.Time{}
+		for _, msg := range messages {
+			key := fmt.Sprintf("pauses_%d", msg.Timestamp.Unix())
+			if !strings.HasPrefix(msg.Content, "!available ") {
+				continue
+			}
+			if _, ok := pauses[msg.Sender]; ok {
+				continue
+			}
+			if _, err := db.Get(key); err == storage.ErrNotFound {
+				t, err := time.ParseInLocation(
+					"2006-01-02",
+					strings.TrimSpace(strings.TrimPrefix(msg.Content, "!available ")),
+					time.Local,
+				)
+				if err == nil {
+					pauses[msg.Sender] = t
+				}
+			}
+			if err := db.Set(key, []byte{'k'}); err != nil {
+				log.Printf("failed to mark state gathered @%s: %v", key, err)
+			}
+		}
+		setNewPauses(pauses)
+	}()
+	return nil
+}
+
+func setNewPauses(pauses map[string]time.Time) {
+	if len(pauses) == 0 {
+		return
+	}
+	log.Printf("set new pauses: %+v", pauses)
+	conf := *config.Get()
+	changed := map[string]time.Time{}
+	for client, pause := range pauses {
+		clientconf := conf.Clients[client]
+		if clientconf.Available.Get().Unix() == pause.Unix() {
 			continue
 		}
-		if !strings.Contains(email.Header.Get("From"), config.Get().Client) {
+		clientconf.Available = config.Time(pause)
-			continue
+		conf.Clients[client] = clientconf
-		}
+		changed[client] = pause
-		b, _ := ioutil.ReadAll(email.Body)
+	}
-		for _, state := range parseOutStates(b) {
+	if len(changed) == 0 {
-			states[state] = struct{}{}
+		return
+	}
+	log.Printf("updating config new pauses: %+v", conf)
+	config.Set(conf)
+	for client, pause := range changed {
+		if err := sendNewPause(client, pause); err != nil {
+			log.Printf("failed to send new pause %s/%+v: %v", client, pause, err)
 		}
 	}
+}
+
+func setNewStates(states map[string]map[config.State]struct{}) {
 	if len(states) == 0 {
-		return nil
+		return
 	}
 	conf := *config.Get()
-	conf.States = []config.State{}
+	changed := map[string][]config.State{}
-	for k := range states {
+	for client, clientStates := range states {
-		conf.States = append(conf.States, k)
+		newstates := []config.State{}
+		for k := range clientStates {
+			newstates = append(newstates, k)
+		}
+		sort.Slice(newstates, func(i, j int) bool {
+			return newstates[i] < newstates[j]
+		})
+		clientconf := conf.Clients[client]
+		if fmt.Sprint(newstates) == fmt.Sprint(clientconf.States) {
+			message.NewMatrix().Send(fmt.Sprintf("%s: still searching for %+v", client, newstates))
+			continue
+		}
+		clientconf.States = newstates
+		conf.Clients[client] = clientconf
+		changed[client] = newstates
 	}
-	log.Printf("%+v, %+v", states, conf)
+	if len(changed) == 0 {
+		return
+	}
+	log.Printf("updating config new states: %+v", conf)
 	config.Set(conf)
-	return nil
+	for client, states := range changed {
+		if err := sendNewStates(client, states); err != nil {
+			log.Printf("failed to send new states %s/%+v: %v", client, states, err)
+		}
+	}
 }
 
 func parseOutStates(b []byte) []config.State {
@@ -99,6 +229,7 @@ func _main() error {
 			log.Println(err)
 		}
 		if config.Get().Once {
+			time.Sleep(time.Second)
 			return err
 		}
 		if err != nil {
@@ -111,12 +242,15 @@ func _main() error {
 }
 
 func _mainOne() error {
+	log.Println("config.refreshing...")
 	if err := config.Refresh(); err != nil {
 		return err
 	}
+	log.Println("once...")
 	if err := once(); err != nil {
 		return err
 	}
+	log.Println("/_mainOne")
 	return nil
 }
 
@@ -137,6 +271,7 @@ func once() error {
 		if err := sendJob(jobs[i]); err != nil {
 			return err
 		}
+		log.Println("sent job", jobs[i])
 		if err := config.Get().DB().Set(jobs[i].ID, []byte(`sent`)); err != nil {
 			return err
 		}
@@ -145,7 +280,7 @@ func once() error {
 }
 
 func getJobs() ([]broker.Job, error) {
-	states := config.Get().States
+	states := config.AllStates()
 	ntg := broker.NewNTGVision()
 	if config.Get().Brokers.NTG.Mock {
 		ntg = ntg.WithMock()
@@ -182,5 +317,39 @@ func dropBanlistJobs(jobs []broker.Job) ([]broker.Job, error) {
 
 func sendJob(job broker.Job) error {
 	sender := message.NewMatrix()
-	return sender.Send(job.FormatMultilineText())
+	payload := job.FormatMultilineText()
+	if len(payload) == 0 {
+		return nil
+	}
+	if err := sender.Send(payload); err != nil {
+		return err
+	}
+	maps := config.Get().Maps
+	if maps.Pickup {
+		pickup := fmt.Sprintf("%s,%s", url.QueryEscape(job.Pickup.City), job.Pickup.State)
+		uri := fmt.Sprintf(maps.URIFormat, pickup, pickup)
+		log.Printf("sending pickup image: %s", uri)
+		if err := sender.SendImage(uri); err != nil {
+			return err
+		}
+	}
+	if maps.Dropoff {
+		dropoff := fmt.Sprintf("%s,%s", url.QueryEscape(job.Dropoff.City), job.Dropoff.State)
+		uri := fmt.Sprintf(maps.URIFormat, dropoff, dropoff)
+		log.Printf("sending dropoff image: %s", uri)
+		if err := sender.SendImage(uri); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func sendNewStates(client string, states []config.State) error {
+	sender := message.NewMatrix()
+	return sender.Send(fmt.Sprintf("%s: now searching for loads from: %+v", client, states))
+}
+
+func sendNewPause(client string, pause time.Time) error {
+	sender := message.NewMatrix()
+	return sender.Send(fmt.Sprintf("%s: only searching for loads on and after %s", client, pause.Format("2006-01-02")))
 }

message/images.go

@@ -0,0 +1,81 @@
+package message
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"local/truckstop/config"
+	"mime/multipart"
+	"net/http"
+	"time"
+)
+
+var ErrBadAuth = errors.New("auth has failed")
+
+func UploadImage(b []byte) (string, error) {
+	path, err := uploadImage(b)
+	if err == ErrBadAuth {
+		if err := refreshToken(); err != nil {
+			return "", err
+		}
+		path, err = uploadImage(b)
+	}
+	return path, err
+}
+
+func refreshToken() error {
+	return errors.New("not impl")
+}
+
+func uploadImage(b []byte) (string, error) {
+	buff := bytes.NewBuffer(nil)
+	writer := multipart.NewWriter(buff)
+	part, err := writer.CreateFormFile("image", "name")
+	if err != nil {
+		return "", err
+	}
+	if n, err := part.Write(b); err != nil {
+		return "", err
+	} else if n < len(b) {
+		return "", errors.New("short write")
+	}
+	writer.Close()
+
+	images := config.Get().Images
+	request, err := http.NewRequest(
+		images.UploadMethod,
+		images.UploadURI,
+		buff,
+	)
+	if err != nil {
+		return "", err
+	}
+	request.Header.Set("Authorization", "Bearer "+images.AccessToken)
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+
+	c := &http.Client{Timeout: time.Minute}
+	response, err := c.Do(request)
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+	b, _ = ioutil.ReadAll(response.Body)
+	switch response.StatusCode {
+	case http.StatusOK:
+	case 401, 403:
+		return "", ErrBadAuth
+	default:
+		return "", fmt.Errorf("error uploading image: (%d) %s", response.StatusCode, b)
+	}
+	var result struct {
+		Data struct {
+			Link string `json:"link"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(b, &result); err != nil {
+		return "", err
+	}
+	return result.Data.Link, nil
+}

message/images_test.go

@@ -0,0 +1,27 @@
+package message
+
+import (
+	"io/ioutil"
+	"local/truckstop/config"
+	"os"
+	"testing"
+)
+
+func TestImageUpload(t *testing.T) {
+	if os.Getenv("INTEGRATION") == "" {
+		t.Skip("$INTEGRATION not set")
+	}
+	os.Setenv("CONFIG", "../config.json")
+	b, err := ioutil.ReadFile("./testdata/whatever.jpg")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := config.Refresh(); err != nil {
+		t.Fatal(err)
+	}
+	got, err := UploadImage(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(got)
+}

message/matrix.go

@@ -1,8 +1,14 @@
 package message
 
 import (
+	"fmt"
+	"io/ioutil"
 	"local/truckstop/config"
 	"log"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
 
 	"github.com/matrix-org/gomatrix"
 )
@@ -26,19 +32,112 @@ func NewMatrix() Matrix {
 	}
 }
 
-func (m Matrix) client() (*gomatrix.Client, error) {
+func (m Matrix) getclient() (*gomatrix.Client, error) {
 	return gomatrix.NewClient(m.homeserver, m.username, m.token)
 }
 
+func (m Matrix) Receive() ([]Message, error) {
+	if m.mock {
+		log.Printf("matrix.Receive()")
+		messages := make([]Message, 0)
+		for k := range config.Get().Clients {
+			messages = append(messages, Message{Timestamp: time.Now(), Sender: k, Content: "!state OH"})
+			if k == "bel" {
+				messages = append(messages, Message{Timestamp: time.Now(), Sender: k, Content: "!help"})
+			}
+			if k == "broc" {
+				messages = append(messages, Message{Timestamp: time.Now(), Sender: k, Content: "!available 2148-10-" + fmt.Sprint(time.Now().Unix()%28)})
+			}
+		}
+		return messages, nil
+	}
+	clients := config.Get().Clients
+	matrixIDs := map[string]struct{}{}
+	for k := range clients {
+		matrixIDs[clients[k].IDs.Matrix] = struct{}{}
+	}
+	if len(matrixIDs) == 0 {
+		return nil, nil
+	}
+	c, err := m.getclient()
+	if err != nil {
+		return nil, err
+	}
+	messages := make([]Message, 0)
+	result, err := c.Messages(m.room, "", "", 'b', 50)
+	if err != nil {
+		return nil, err
+	}
+	for _, event := range result.Chunk {
+		if _, ok := matrixIDs[event.Sender]; !ok {
+			continue
+		}
+		switch event.Type {
+		case "m.room.message":
+			b, ok := event.Body()
+			if ok {
+				messages = append(messages, Message{Timestamp: time.Unix(0, event.Timestamp*int64(time.Millisecond)), Sender: event.Sender, Content: strings.TrimSpace(b)})
+			}
+		}
+	}
+	clientChange := regexp.MustCompile("@[a-z]+$")
+	for i := range messages {
+		if found := clientChange.FindString(messages[i].Content); found != "" {
+			messages[i].Content = strings.TrimSpace(strings.ReplaceAll(messages[i].Content, found, ""))
+			messages[i].Sender = found[1:]
+		} else {
+			for k, v := range config.Get().Clients {
+				if v.IDs.Matrix == messages[i].Sender {
+					messages[i].Sender = k
+				}
+			}
+		}
+		messages[i].Content = strings.TrimSpace(messages[i].Content)
+	}
+	return messages, nil
+}
+
 func (m Matrix) Send(text string) error {
 	if m.mock {
 		log.Printf("matrix.Send(%s)", text)
 		return nil
 	}
-	c, err := m.client()
+	c, err := m.getclient()
 	if err != nil {
 		return err
 	}
 	_, err = c.SendText(m.room, text)
 	return err
 }
+
+func (m Matrix) SendImage(uri string) error {
+	if m.mock {
+		log.Printf("matrix.SendImage(%s)", uri)
+		return nil
+	}
+	response, err := http.Get(uri)
+	if err != nil {
+		return err
+	}
+	if response.StatusCode != http.StatusOK {
+		b, _ := ioutil.ReadAll(response.Body)
+		response.Body.Close()
+		return fmt.Errorf("failed to get %s: (%d) %s", uri, response.StatusCode, b)
+	}
+	b, err := ioutil.ReadAll(response.Body)
+	response.Body.Close()
+	if err != nil {
+		return err
+	}
+	publicURI, err := UploadImage(b)
+	if err != nil {
+		return err
+	}
+	c, err := m.getclient()
+	if err != nil {
+		return err
+	}
+	resp, err := c.SendImage(m.room, "img", publicURI)
+	log.Printf("sent image %s => %s: %+v", uri, publicURI, resp)
+	return err
+}

message/matrix_test.go

@@ -30,3 +30,28 @@ func TestMatrixSend(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestMatrixReceive(t *testing.T) {
+	if len(os.Getenv("INTEGRATION")) == 0 {
+		t.Skip("$INTEGRATION not set")
+	}
+	var c config.Config
+	b, err := ioutil.ReadFile("../config.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := json.Unmarshal(b, &c); err != nil {
+		t.Fatal(err)
+	}
+	var sender Sender = Matrix{
+		homeserver: c.Message.Matrix.Homeserver,
+		username:   c.Message.Matrix.Username,
+		token:      c.Message.Matrix.Token,
+		room:       c.Message.Matrix.Room,
+	}
+	if msgs, err := sender.Receive(); err != nil {
+		t.Fatal(err)
+	} else {
+		t.Logf("%+v", msgs)
+	}
+}

message/message.go

@@ -1,5 +1,14 @@
 package message
 
+import "time"
+
 type Sender interface {
 	Send(string) error
+	Receive() ([]Message, error)
+}
+
+type Message struct {
+	Sender    string
+	Content   string
+	Timestamp time.Time
 }

testdata/index.html

@@ -0,0 +1,5 @@
+<html>
+<body>
+  <img src='https://maps.googleapis.com/maps/api/staticmap?center=Advance,NC&zoom=13&size=400x400&maptype=roadmap&key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg'/>
+</body>
+</html>

todo.yaml

@@ -1,10 +1,45 @@
 todo:
+- test each !command callbacks to matrix
+- change matrix so I test my custom logic even if I dont fetch remote
+- warn/err/etc. on clobbering ids.matrix since clients can mess with one another
+- modify old items once no longer available; drop stale jobs good candidate but requires new matrix interaction
+- more than NTG
 - todo: filter out jobs like CA
   subtasks:
   - banlist criteria like vendors, brokers, metadata
-- quiet hours
+- set up copy for caleb, broc
-- setup pa on element
 done:
+- setup ma on element !!fluffychat
+- !help,
+- !states optional but explicit option
+- TEST; convert pauseuntil to search results only on and after target date
+- pause until => !busy until
+- todo: maps of to+from to get location within state via api
+  details: |
+    curl  'https://maps.googleapis.com/maps/api/staticmap?center=Advance,NC&markers=label=A|Advance,NC&zoom=5&size=250x250&scale=2&format=jpeg&maptype=roadmap&key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' -H 'Accept: image/avif,image/webp,*/*' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'DNT: 1' -H 'Alt-Used: maps.googleapis.com' -H 'Connection: keep-alive' -H 'Sec-Fetch-Dest: image' -H 'Sec-Fetch-Mode: no-cors' -H 'Sec-Fetch-Site: cross-site' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'TE: trailers'  > whatever.jpg; open whatever.jpg~/Go/src/local/truckstop
+  subtasks:
+  - DONE; yandex; key 9baa3e42-c6e5-4eb5-a891-05ffcede6a25 yandex maps
+  - google; key AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg
+- todo: upload g map to imgur
+  details: |
+    w oath d9ac7cabe813d10 9d0b3d82800b30ca88f595d3bcd6985f627d7d82
+    Authorization: Client-ID d9ac7cabe813d10
+    Authorization: Bearer YOUR_ACCESS_TOKEN
+- quiet hours
+- "@caleb commands: args"
+- accept after date
+- accept pause commands
+- accept states via element for one system
+- write to matrix on config change like states
+- rate LIMIT
+- rate limit brokers
+- many users -> 1 ntg query
+- multi client
+- rm email
+- send matrix msg on config change
+- setup pa on element
+- configurable email interval
+- jitter on intervals, including dedicated err span
 - email doesnt get all matches
 - send jobs
 - read jobs