good enough

.gitignore

@@ -4,3 +4,4 @@ cmd/cmd
 cmd/cli
 cmd/pttodo/pttodo
 /truckstop
+/exec-truckstop

broker/broker.go

@@ -2,9 +2,13 @@ package broker
 
 import (
 	"context"
-	"local/truckstop/config"
+	"encoding/json"
+	"gogs.inhome.blapointe.com/local/storage"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/logtr"
 	"net/http"
 	"strings"
+	"time"
 
 	"golang.org/x/time/rate"
 )
@@ -16,13 +20,98 @@ var authlimiter = rate.NewLimiter(rate.Limit(1.0/60.0), 1)
 var limiter = rate.NewLimiter(rate.Limit(1.0/20.0), 1)
 
 type Broker interface {
-	Search([]config.State) ([]Job, error)
+	SearchStates([]config.State) ([]Job, error)
+	SearchZips([]string) ([]Job, error)
 }
 
 func do(r *http.Request) (*http.Response, error) {
+	return _do(config.Get().DB(), r)
+}
+
+func _do(db storage.DB, r *http.Request) (*http.Response, error) {
 	limiter.Wait(context.Background())
 	if strings.Contains(strings.ToLower(r.URL.Path), "login") {
 		authlimiter.Wait(context.Background())
 	}
-	return http.DefaultClient.Do(r)
+	client := &http.Client{
+		Timeout: time.Hour,
+	}
+
+	cookieJarKey := "cookies_" + r.URL.Host
+	cookies, err := getCookies(db, cookieJarKey)
+	if err != nil {
+		logtr.Errorf("failed to get cookies: %v", err)
+	} else {
+		logtr.Verbosef("got cookies for %s: %+v", cookieJarKey, cookies)
+		cookieV := strings.Join(r.Header["Cookie"], "; ")
+		for _, cookie := range cookies {
+			if len(cookieV) > 0 {
+				cookieV += "; "
+			}
+			cookieV += cookie
+		}
+		r.Header.Set("Cookie", cookieV)
+	}
+	logtr.Verbosef("_do: %+v", r)
+	resp, err := client.Do(r)
+	if err != nil {
+		return nil, err
+	}
+	if err := setCookies(db, cookieJarKey, resp); err != nil {
+		logtr.Errorf("failed to set cookies: %v", err)
+	}
+	return resp, err
+}
+
+func getCookies(db storage.DB, host string) ([]string, error) {
+	b, err := db.Get(host)
+	if err != nil {
+		return nil, err
+	}
+	var result []string
+	err = json.Unmarshal(b, &result)
+	return result, err
+}
+
+func setCookies(db storage.DB, host string, resp *http.Response) error {
+	nameValues := [][2]string{}
+	old, _ := getCookies(db, host)
+	for _, value := range old {
+		if len(value) == 0 {
+			continue
+		}
+		name := strings.Split(value, "=")[0]
+		value := strings.Split(value, ";")[0]
+		nameValues = append(nameValues, [2]string{name, value})
+	}
+	for _, cookie := range resp.Header["Set-Cookie"] {
+		if len(cookie) == 0 {
+			continue
+		}
+		name := strings.Split(cookie, "=")[0]
+		value := strings.Split(cookie, ";")[0]
+		found := false
+		for i := range nameValues {
+			if nameValues[i][0] == name {
+				found = true
+				nameValues[i][1] = value
+			}
+		}
+		if !found {
+			nameValues = append(nameValues, [2]string{name, value})
+		}
+	}
+	result := []string{}
+	for i := range nameValues {
+		if len(nameValues[i][1]) == 0 {
+			continue
+		}
+		result = append(result, nameValues[i][1])
+	}
+	logtr.Verbosef("setting cookies for %s: %+v", host, result)
+	b, err := json.Marshal(result)
+	if err != nil {
+		return err
+	}
+	return db.Set(host, b)
 }

broker/broker_test.go

@@ -0,0 +1,73 @@
+package broker
+
+import (
+	"fmt"
+	"gogs.inhome.blapointe.com/local/storage"
+	"gogs.inhome.blapointe.com/local/truckstop/logtr"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+func TestBrokerInterface(t *testing.T) {
+	logtr.SetLevel(logtr.SOS)
+	var b Broker
+	b = FastExact{}
+	b = NTGVision{}
+	_ = b
+}
+
+func TestDoCookies(t *testing.T) {
+	logtr.SetLevel(logtr.SOS)
+	limiter = rate.NewLimiter(rate.Limit(20.0), 1)
+	calls := 0
+	db := storage.NewMap()
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls += 1
+		if calls != 1 {
+			if !strings.Contains(fmt.Sprint(r.Header["Cookie"]), "name=value"+strconv.Itoa(calls-1)) {
+				w.WriteHeader(http.StatusBadRequest)
+				t.Error("cookie not set as latest: " + fmt.Sprint(r.Header["Cookie"]))
+			}
+			if strings.Contains(fmt.Sprint(r.Header["Cookie"]), "Expires") {
+				w.WriteHeader(http.StatusBadRequest)
+				t.Error("cookie not expiration: ", r.Header["Cookie"])
+			}
+		}
+		http.SetCookie(w, &http.Cookie{
+			Name:    "name",
+			Value:   "value" + strconv.Itoa(calls),
+			Expires: time.Now().Add(time.Hour),
+		})
+	}))
+	defer s.Close()
+	req, _ := http.NewRequest(http.MethodGet, s.URL, nil)
+
+	if cookies, err := getCookies(db, "cookies_"+req.URL.Host); err == nil {
+		t.Fatal(err)
+	} else if len(cookies) != 0 {
+		t.Fatal(cookies)
+	}
+
+	for i := 0; i < 3; i++ {
+		resp, err := _do(db, req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if resp.StatusCode != http.StatusOK {
+			t.Fatal(resp.StatusCode)
+		}
+		resp.Body.Close()
+
+		if cookies, err := getCookies(db, "cookies_"+req.URL.Host); err != nil {
+			t.Fatal(err)
+		} else if len(cookies) == 0 {
+			t.Fatal(cookies)
+		}
+	}
+}

broker/fastexact.go

@@ -0,0 +1,318 @@
+package broker
+
+import (
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"gogs.inhome.blapointe.com/local/storage"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/logtr"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type FastExact struct {
+	doer interface {
+		doRequest(*http.Request) (*http.Response, error)
+	}
+}
+
+func NewFastExact() FastExact {
+	fe := FastExact{}
+	fe.doer = fe
+	if config.Get().Brokers.FastExact.Mock {
+		fe = fe.WithMock()
+	}
+	return fe
+}
+
+func (fe FastExact) WithMock() FastExact {
+	fe.doer = mockFastExactDoer{}
+	return fe
+}
+
+func (fe FastExact) SearchZips(zips []string) ([]Job, error) {
+	jobs, err := fe.searchZips(zips)
+	if err == ErrNoAuth {
+		if err := fe.login(); err != nil {
+			return nil, err
+		}
+		jobs, err = fe.searchZips(zips)
+	}
+	return jobs, err
+}
+
+func (fe FastExact) SearchStates(states []config.State) ([]Job, error) {
+	jobs, err := fe.searchStates(states)
+	if err == ErrNoAuth {
+		if err := fe.login(); err != nil {
+			return nil, err
+		}
+		jobs, err = fe.searchStates(states)
+	}
+	return jobs, err
+}
+
+func (fe FastExact) login() error {
+	conf := config.Get()
+	return fe._login(conf.Brokers.FastExact.Username, conf.Brokers.FastExact.Password, conf.DB())
+}
+
+func (fe FastExact) _login(username, password string, db storage.DB) error {
+	req, err := http.NewRequest(
+		http.MethodPost,
+		`https://www.fastexact.com/secure/index.php?page=userLogin`,
+		strings.NewReader(fmt.Sprintf(
+			`user_name=%s&user_password=%s&buttonSubmit=Login`,
+			username,
+			password,
+		)),
+	)
+	if err != nil {
+		return err
+	}
+	fe.setHeaders(req)
+
+	db.Set("cookies_"+req.URL.Host, nil)
+	resp, err := fe.doer.doRequest(req)
+	if err != nil {
+		return err
+	}
+	b, _ := ioutil.ReadAll(resp.Body)
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("bad status logging into fast exact: %d: %s", resp.StatusCode, b)
+	}
+	return nil
+}
+
+func (fe FastExact) setHeaders(req *http.Request) {
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.5")
+	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+}
+
+func (fe FastExact) searchZips(zips []string) ([]Job, error) {
+	var jobs []Job
+	for _, zip := range zips {
+		subjobs, err := fe.searchOneZip(zip)
+		if err != nil {
+			return nil, err
+		}
+		jobs = append(jobs, subjobs...)
+	}
+	return fe.dedupeJobs(jobs), nil
+}
+
+func (fe FastExact) searchStates(states []config.State) ([]Job, error) {
+	var jobs []Job
+	for _, state := range states {
+		subjobs, err := fe.searchOneState(state)
+		if err != nil {
+			return nil, err
+		}
+		jobs = append(jobs, subjobs...)
+	}
+	return fe.dedupeJobs(jobs), nil
+}
+
+func (fe FastExact) dedupeJobs(jobs []Job) []Job {
+	sort.Slice(jobs, func(i, j int) bool {
+		return jobs[i].UID() < jobs[j].UID()
+	})
+	dedupeJobs := map[string]Job{}
+	for _, job := range jobs {
+		dedupeJobs[strings.ReplaceAll(job.UID(), job.ID, "")] = job
+	}
+	result := []Job{}
+	for _, job := range dedupeJobs {
+		result = append(result, job)
+	}
+	return result
+}
+
+func (fe FastExact) searchOneZip(zip string) ([]Job, error) {
+	req, err := fe.newRequest(zip)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := fe.doer.doRequest(req)
+	logtr.Verbosef("req: %+v => resp: %+v", req, resp)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	return fe.parse(resp)
+}
+
+func (fe FastExact) searchOneState(state config.State) ([]Job, error) {
+	req, err := fe.newRequestWithState(state)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := fe.doer.doRequest(req)
+	logtr.Verbosef("req: %+v => resp: %+v", req, resp)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	return fe.parse(resp)
+}
+
+func (fe FastExact) newRequestWithState(state config.State) (*http.Request, error) {
+	zip, ok := config.States[state]
+	if !ok {
+		return nil, fmt.Errorf("no configured zip for %s", state)
+	}
+	return fe.newRequest(zip)
+}
+
+func (fe FastExact) newRequest(zip string) (*http.Request, error) {
+	req, err := http.NewRequest(
+		http.MethodGet,
+		"https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance="+strconv.Itoa(config.Get().Brokers.RadiusMiles)+"&st_loc_zip=8",
+		nil,
+	)
+	if err != nil {
+		return nil, err
+	}
+	fe.setHeaders(req)
+	return req, nil
+}
+
+func (fe FastExact) doRequest(req *http.Request) (*http.Response, error) {
+	resp, err := do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		b, _ := ioutil.ReadAll(resp.Body)
+		logtr.Errorf("fastExact bad status: %d: %s", resp.StatusCode, b)
+		if resp.StatusCode > 400 && resp.StatusCode < 404 {
+			return nil, ErrNoAuth
+		}
+		return nil, fmt.Errorf("bad status from FastExact: %d: %s", resp.StatusCode, b)
+	}
+	return resp, nil
+}
+
+func (fe FastExact) parse(resp *http.Response) ([]Job, error) {
+	b, _ := ioutil.ReadAll(resp.Body)
+	if !bytes.HasPrefix(bytes.TrimSpace(b), []byte("<")) {
+		gzip, err := gzip.NewReader(bytes.NewReader(b))
+		if err != nil {
+			return nil, err
+		}
+		b, _ = ioutil.ReadAll(gzip)
+	}
+	if bytes.Contains(b, []byte(`<script>window.location.href='index.php'</script>`)) {
+		return nil, ErrNoAuth
+	}
+	logtr.Verbosef("fe.parse %s", b)
+	resp.Body = io.NopCloser(bytes.NewReader(b))
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	result := make([]Job, 0)
+	doc.Find("#list table tr").Each(func(i int, s *goquery.Selection) {
+		columns := []string{}
+		s.Find("td").Each(func(i int, s *goquery.Selection) {
+			if s.Nodes[0].LastChild != nil && len(s.Nodes[0].LastChild.Attr) > 0 {
+				attrs := s.Nodes[0].LastChild.Attr
+				columns = append(columns, attrs[len(attrs)-1].Val)
+			} else {
+				columns = append(columns, s.Text())
+			}
+		})
+		if len(columns) < 9 {
+			return
+		}
+		job := Job{
+			ID:  columns[0],
+			URI: columns[8],
+		}
+		job.Pickup.Date, _ = time.ParseInLocation("02-Jan-2006  15:04:05", columns[7], time.Local)
+		job.Pickup.City = strings.Title(strings.ToLower(strings.Split(columns[1], ",")[0]))
+		if strings.Contains(columns[1], ",") {
+			job.Pickup.State = strings.Title(strings.Split(strings.Split(columns[1], ",")[1], " ")[1])
+		}
+
+		job.Dropoff.Date = job.Pickup.Date
+		job.Dropoff.City = strings.Title(strings.ToLower(strings.Split(columns[2], ",")[0]))
+		if strings.Contains(columns[2], ",") {
+			job.Dropoff.State = strings.Title(strings.Split(strings.Split(columns[2], ",")[1], " ")[1])
+		}
+
+		job.Miles, _ = strconv.Atoi(columns[3])
+		if strings.Contains(columns[4], "/") {
+			weight, _ := strconv.ParseFloat(strings.TrimSpace(strings.Split(columns[4], "/")[1]), 32)
+			job.Weight = int(weight)
+		}
+
+		job.Meta = fmt.Sprintf(`dimensions:%s`, strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(columns[5], " ", ""), "\n", "")))
+
+		logtr.Verbosef("fe.parse %+v => %+v", columns, job)
+
+		result = append(result, job)
+	})
+	return result, nil
+}
+
+type mockFastExactDoer struct{}
+
+func (mock mockFastExactDoer) doRequest(req *http.Request) (*http.Response, error) {
+	if req.URL.Path != "/secure/index.php" {
+		return nil, errors.New("bad path")
+	}
+	switch req.URL.Query().Get("page") {
+	case "userLogin":
+		if b, _ := ioutil.ReadAll(req.Body); !bytes.Equal(b, []byte(`user_name=u&user_password=p&buttonSubmit=Login`)) {
+			return nil, errors.New("bad req body")
+		}
+		return &http.Response{
+			Status:     http.StatusText(http.StatusOK),
+			StatusCode: http.StatusOK,
+			Header:     http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromLogin; path=/"}},
+			Body:       io.NopCloser(bytes.NewReader([]byte{})),
+		}, nil
+	case "ajaxListJobs":
+		if req.URL.Query().Get("action") != "ajax" {
+			return nil, errors.New("bad query: action should be ajax")
+		}
+		if req.URL.Query().Get("records_per_page") != "50" {
+			return nil, errors.New("bad query: records_per_page should be 50")
+		}
+		if req.URL.Query().Get("distance") != strconv.Itoa(config.Get().Brokers.RadiusMiles) {
+			return nil, errors.New("bad query: distance should be as configured")
+		}
+		if req.URL.Query().Get("zipcode") == "" {
+			return nil, errors.New("bad query: zip code empty")
+		}
+		b, err := ioutil.ReadFile("./testdata/fastexact_search.xml")
+		if err != nil {
+			b, err = ioutil.ReadFile("./broker/testdata/fastexact_search.xml")
+		}
+		if err != nil {
+			return nil, err
+		}
+		return &http.Response{
+			Status:     http.StatusText(http.StatusOK),
+			StatusCode: http.StatusOK,
+			Header:     http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromSearch; path=/"}},
+			Body:       io.NopCloser(bytes.NewReader(b)),
+		}, nil
+	}
+	return nil, errors.New("bad query")
+}

broker/fastexact_test.go

@@ -0,0 +1,97 @@
+package broker
+
+import (
+	"gogs.inhome.blapointe.com/local/storage"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/logtr"
+	"os"
+	"path"
+	"testing"
+
+	"golang.org/x/time/rate"
+)
+
+func TestFastExactReal(t *testing.T) {
+	if os.Getenv("INTEGRATION_FAST_EXACT") == "" {
+		t.Skip("$INTEGRATION_FAST_EXACT not set")
+	}
+	os.Setenv("CONFIG", "../config.json")
+	if err := config.Refresh(nil); err != nil {
+		t.Fatal(err)
+	}
+	conf := config.Get()
+	conf.Storage = []string{"files", path.Join(t.TempDir(), "storage")}
+	os.Setenv("CONFIG", path.Join(t.TempDir(), "config.json"))
+	config.Set(*conf)
+	if err := config.Refresh(nil); err != nil {
+		t.Fatal(err)
+	}
+	states := []config.State{
+		config.State("NC"),
+	}
+
+	fe := NewFastExact()
+	if err := fe.login(); err != nil {
+		t.Fatal(err)
+	}
+	jobs, err := fe.searchStates(states)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("%d", len(jobs))
+	for i := range jobs {
+		t.Logf("[%d] %+v", i, jobs[i])
+	}
+}
+
+func TestFastExactLogin(t *testing.T) {
+	logtr.SetLevel(logtr.SOS)
+	limiter = rate.NewLimiter(rate.Limit(60.0), 1)
+	fe := NewFastExact().WithMock()
+	_ = fe
+	db := storage.NewMap()
+	if err := fe._login("u", "p", db); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestFastExactSearchStates(t *testing.T) {
+	logtr.SetLevel(logtr.SOS)
+	limiter = rate.NewLimiter(rate.Limit(60.0), 1)
+	fe := NewFastExact().WithMock()
+	_ = fe
+	db := storage.NewMap()
+	_ = db
+	if jobs, err := fe.searchStates([]config.State{config.State("NC"), config.State("SC")}); err != nil {
+		t.Fatal(err)
+	} else if len(jobs) != 9 {
+		t.Fatal(len(jobs))
+	} else {
+		for _, job := range jobs {
+			if job.ID == "" {
+				t.Error(job)
+			} else if job.Miles == 0 {
+				t.Error(job)
+			} else if job.URI == "" {
+				t.Error(job)
+			} else if job.Meta == "" {
+				t.Error(job)
+			} else if job.Weight == 0 {
+				t.Error(job)
+			} else if job.Pickup.Date.IsZero() {
+				t.Error(job)
+			} else if job.Pickup.Date != job.Dropoff.Date {
+				t.Error(job)
+			} else if job.Dropoff.State == "" {
+				t.Error(job)
+			} else if job.Dropoff.City == "" {
+				t.Error(job)
+			} else if job.Pickup.State == "" {
+				t.Error(job)
+			} else if job.Pickup.City == "" {
+				t.Error(job)
+			}
+		}
+	}
+}

broker/job.go

@@ -1,9 +1,10 @@
 package broker
 
 import (
+	"encoding/base64"
 	"fmt"
-	"local/truckstop/config"
-	"log"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/zip"
 	"strings"
 	"time"
 )
@@ -16,6 +17,8 @@ type Job struct {
 	Weight  int
 	Miles   int
 	Meta    string
+	Pays    string
+	secrets func(j *Job) `json:"-"`
 }
 
 type JobLocation struct {
@@ -24,6 +27,25 @@ type JobLocation struct {
 	State string
 }
 
+func (j Job) UID() string {
+	return fmt.Sprintf(
+		"%v-%s-%s-%s-%s-%v",
+		j.ID,
+		j.Pickup.State,
+		base64.StdEncoding.EncodeToString([]byte(j.Pickup.City)),
+		j.Dropoff.State,
+		base64.StdEncoding.EncodeToString([]byte(j.Dropoff.City)),
+		j.Pickup.Date.Unix(),
+	)
+}
+
+func (j *Job) Secrets() {
+	if j.secrets == nil {
+		return
+	}
+	j.secrets(j)
+}
+
 func (j Job) String() string {
 	return fmt.Sprintf(
 		`%s => %s (%d miles), Weight:%d, Notes:%s Link:%s`,
@@ -40,13 +62,51 @@ func (j JobLocation) String() string {
 	return fmt.Sprintf("%s, %s @ %s", j.City, j.State, j.Date.Format("Monday Jan 02"))
 }
 
+func (j Job) FormatMultilineTextDead() string {
+	return fmt.Sprintf(
+		"no longer available: %s,%s => %s,%s for $%v @%s",
+		j.Pickup.City,
+		j.Pickup.State,
+		j.Dropoff.City,
+		j.Dropoff.State,
+		j.Pays,
+		j.URI,
+	)
+
+}
+
 func (j Job) FormatMultilineText() string {
 	foo := func(client string) string {
 		return fmt.Sprintf(
-			"--- %s: %s => %s ---\nPickup: %s\nDropoff: %s\nNotes: %d lbs, %d miles, %s\n%s",
+			"--- %s: %s => %s ---",
 			client,
 			j.Pickup.State,
 			j.Dropoff.State,
+		)
+	}
+	out := ""
+	clients := config.Clients(j.Pickup.Date)
+	useZip := config.Get().Brokers.UseZips
+	zipRadius := config.Get().Brokers.RadiusMiles
+	jobZip := zip.FromCityState(j.Pickup.City, j.Pickup.State)
+	for k := range clients {
+		should := strings.Contains(fmt.Sprint(clients[k].States), j.Pickup.State)
+		if useZip {
+			for _, z := range clients[k].Zips {
+				should = should || zip.Get(z).MilesTo(jobZip) <= zipRadius
+			}
+		}
+		if should {
+			if len(out) > 0 {
+				out += "\n"
+			}
+			out += foo(k)
+		}
+	}
+	if len(out) > 0 {
+		out = fmt.Sprintf(
+			"%s\nPickup: %s\nDropoff: %s\nNotes: %d lbs, %d miles, %s\n%s",
+			out,
 			j.Pickup.String(),
 			j.Dropoff.String(),
 			j.Weight,
@@ -55,16 +115,5 @@ func (j Job) FormatMultilineText() string {
 			j.URI,
 		)
 	}
-	out := ""
-	clients := config.Clients(j.Pickup.Date)
-	for k := range clients {
-		log.Printf("job multiline: %+v contains %s then use %v", clients[k].States, j.Pickup.State, k)
-		if strings.Contains(fmt.Sprint(clients[k].States), j.Pickup.State) {
-			if len(out) > 0 {
-				out += "\n\n"
-			}
-			out += foo(k)
-		}
-	}
 	return out
 }

broker/ntgvision.go

@@ -7,15 +7,17 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"local/truckstop/config"
-	"log"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/logtr"
+	"gogs.inhome.blapointe.com/local/truckstop/zip"
 	"net/http"
 	"time"
 )
 
 type NTGVision struct {
 	searcher interface {
-		search(states []config.State) (io.ReadCloser, error)
+		searchStates(states []config.State) (io.ReadCloser, error)
+		searchJobReadCloser(id int64) (io.ReadCloser, error)
 	}
 }
 
@@ -33,9 +35,97 @@ type ntgVisionJob struct {
 	Weight       int    `json:"weight"`
 	Equipment    string `json:"equip"`
 	Temp         string `json:"temp"`
+	jobinfo      ntgVisionJobInfo
 }
 
-func (ntgJob ntgVisionJob) Job() Job {
+type ntgVisionJobInfo struct {
+	StopsInfo []struct {
+		StopHours       string `json:"stopHours"`
+		AppointmentTime string `json:"appointmentTime"`
+		Instructions    string `json:"instructions"`
+		IsDropTrailer   bool   `json:"isDropTrailer"`
+	} `json:"stopinfos"`
+	PayUpTo          float32 `json:"payUpTo"`
+	TotalCarrierRate float32 `json:"totalCarrierRate"`
+	LoadState        string  `json:"loadStatus"`
+	//CanBidNow bool    `json:"canBidNow"`
+}
+
+func (ji ntgVisionJobInfo) IsZero() bool {
+	return len(ji.StopsInfo) == 0 && ji.TotalCarrierRate == 0 && ji.PayUpTo == 0 && ji.LoadState == ""
+}
+
+func (ji ntgVisionJobInfo) String() string {
+	if ji.IsZero() {
+		return ""
+	}
+	out := ""
+	if ji.PayUpTo != 0 {
+		out = fmt.Sprintf("\nPAYS:%v\n%s", ji.PayUpTo, out)
+	}
+	if ji.TotalCarrierRate != 0 {
+		out = fmt.Sprintf("%s Total_Carrier_Rate:%v", out, ji.TotalCarrierRate)
+	}
+	if ji.LoadState != "" {
+		out = fmt.Sprintf("%s Auction:%s", out, ji.LoadState)
+	}
+	if len(ji.StopsInfo) != 2 {
+		return out
+	}
+	return fmt.Sprintf(
+		"%s Pickup:{Hours:%s Notes:%s, DropTrailer:%v} Dropoff:{Appointment:%s Notes:%s}",
+		out,
+		ji.StopsInfo[0].StopHours,
+		ji.StopsInfo[0].Instructions,
+		ji.StopsInfo[0].IsDropTrailer,
+		ji.StopsInfo[1].AppointmentTime,
+		ji.StopsInfo[1].Instructions,
+	)
+}
+
+func (ntgJob *ntgVisionJob) JobInfo() (ntgVisionJobInfo, error) {
+	if !config.Get().Brokers.NTG.JobInfo {
+		return ntgJob.jobinfo, nil
+	}
+	if !ntgJob.jobinfo.IsZero() {
+		return ntgJob.jobinfo, nil
+	}
+	db := config.Get().DB()
+	key := fmt.Sprintf("ntg_job_info_%v", ntgJob.ID)
+	if b, err := db.Get(key); err != nil {
+	} else if err := json.Unmarshal(b, &ntgJob.jobinfo); err == nil {
+		return ntgJob.jobinfo, nil
+	}
+	ntg := NewNTGVision()
+	ji, err := ntg.searchJob(ntgJob.ID)
+	if err == nil {
+		ntgJob.jobinfo = ji
+		b, err := json.Marshal(ntgJob.jobinfo)
+		if err == nil {
+			db.Set(key, b)
+		}
+	}
+	return ji, err
+}
+
+func (ntg NTGVision) searchJobReadCloser(id int64) (io.ReadCloser, error) {
+	time.Sleep(config.Get().Interval.JobInfo.Get())
+	request, err := http.NewRequest(http.MethodGet, fmt.Sprintf(config.Get().Brokers.NTG.LoadPageAPIURIFormat, id), nil)
+	if err != nil {
+		return nil, err
+	}
+	setNTGHeaders(request)
+	resp, err := do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	b, _ := ioutil.ReadAll(resp.Body)
+	logtr.Debugf("fetch ntg job info %+v: %d: %s", request, resp.StatusCode, b)
+	return io.NopCloser(bytes.NewReader(b)), nil
+}
+
+func (ntgJob *ntgVisionJob) Job() Job {
 	pickup, _ := time.ParseInLocation("01/02/06", ntgJob.PickupDate, time.Local)
 	dropoff, _ := time.ParseInLocation("01/02/06", ntgJob.DropoffDate, time.Local)
 	return Job{
@@ -54,12 +144,24 @@ func (ntgJob ntgVisionJob) Job() Job {
 		Miles:  ntgJob.Miles,
 		Weight: ntgJob.Weight,
 		Meta:   fmt.Sprintf("equipment:%s", ntgJob.Equipment),
+		secrets: func(j *Job) {
+			jobInfo, err := ntgJob.JobInfo()
+			if err != nil {
+				logtr.Errorf("failed to get jobinfo: %v", err)
+				return
+			}
+			j.Meta = jobInfo.String()
+			j.Pays = fmt.Sprint(jobInfo.PayUpTo)
+		},
 	}
 }
 
 func NewNTGVision() NTGVision {
 	ntgv := NTGVision{}
 	ntgv.searcher = ntgv
+	if config.Get().Brokers.NTG.Mock {
+		ntgv = ntgv.WithMock()
+	}
 	return ntgv
 }
 
@@ -68,8 +170,95 @@ func (ntg NTGVision) WithMock() NTGVision {
 	return ntg
 }
 
-func (ntg NTGVision) Search(states []config.State) ([]Job, error) {
-	rc, err := ntg.searcher.search(states)
+func (ntg NTGVision) searchJob(id int64) (ntgVisionJobInfo, error) {
+	rc, err := ntg.searcher.searchJobReadCloser(id)
+	if err != nil {
+		return ntgVisionJobInfo{}, err
+	}
+	defer rc.Close()
+	b, err := ioutil.ReadAll(rc)
+	if err != nil {
+		return ntgVisionJobInfo{}, fmt.Errorf("failed to readall search job result: %w", err)
+	}
+	var result ntgVisionJobInfo
+	err = json.Unmarshal(b, &result)
+	if err != nil {
+		err = fmt.Errorf("failed to parse job info: %w: %s", err, b)
+	}
+	return result, err
+}
+
+func (ntg NTGVision) SearchZips(zips []string) ([]Job, error) {
+	radius := config.Get().Brokers.RadiusMiles
+	statesm := map[string]struct{}{}
+	for _, z := range zips {
+		somestates := zip.GetStatesWithin(zip.Get(z), radius)
+		for _, state := range somestates {
+			statesm[state] = struct{}{}
+		}
+	}
+	states := make([]config.State, 0, len(statesm))
+	for state := range statesm {
+		states = append(states, config.State(state))
+	}
+	if len(states) == 0 {
+		return nil, fmt.Errorf("failed to map zipcodes %+v to any states", zips)
+	}
+	jobs, err := ntg.SearchStates(states)
+	if err != nil {
+		return nil, err
+	}
+	for i := len(jobs) - 1; i >= 0; i-- {
+		ok := false
+		for _, z := range zips {
+			ok = ok || zip.Get(z).MilesTo(zip.FromCityState(jobs[i].Pickup.City, jobs[i].Pickup.State)) <= radius
+		}
+		if !ok {
+			jobs[i], jobs[len(jobs)-1] = jobs[len(jobs)-1], jobs[i]
+			jobs = jobs[:len(jobs)-1]
+		}
+	}
+	return jobs, nil
+}
+
+func (ntg NTGVision) workingHours(now time.Time) bool {
+	// TODO assert M-F 9-4 EST
+	now = now.In(time.FixedZone("EST", -5*60*60))
+	working := config.Get().Brokers.NTG.Working
+	logtr.Debugf("ntg.workingHours: now=%s, weekday=%v, hour=%v (ok=%+v)", now.String(), now.Weekday(), now.Hour(), working)
+	if ok := func() bool {
+		for _, hr := range working.Hours {
+			if now.Hour() == hr {
+				return true
+			}
+		}
+		return false
+	}(); !ok {
+		return false
+	}
+	if ok := func() bool {
+		for _, weekday := range working.Weekdays {
+			if now.Weekday() == time.Weekday(weekday) {
+				return true
+			}
+		}
+		return false
+	}(); !ok {
+		return false
+	}
+	return true
+}
+
+func (ntg NTGVision) SearchStates(states []config.State) ([]Job, error) {
+	if !ntg.workingHours(time.Now()) {
+		lastNtgB, _ := config.Get().DB().Get("ntg_last_search_states")
+		var jobs []Job
+		json.Unmarshal(lastNtgB, &jobs)
+		logtr.Verbosef("ntg.SearchStates: outside of working hours so returning ntg_last_search_states: %+v", jobs)
+		return jobs, nil
+	}
+
+	rc, err := ntg.searcher.searchStates(states)
 	if err != nil {
 		return nil, err
 	}
@@ -80,35 +269,71 @@ func (ntg NTGVision) Search(states []config.State) ([]Job, error) {
 		return nil, err
 	}
 
-	log.Printf("ntg search for %+v: %s", states, b)
+	logtr.Debugf("ntg search for %+v: %s", states, b)
 
 	var ntgjobs []ntgVisionJob
 	err = json.Unmarshal(b, &ntgjobs)
+	if err != nil {
+		return nil, err
+	}
 
 	jobs := make([]Job, len(ntgjobs))
 	for i := range jobs {
 		jobs[i] = ntgjobs[i].Job()
 	}
-	return jobs, err
+
+	jobsB, err := json.Marshal(jobs)
+	if err == nil {
+		config.Get().DB().Set("ntg_last_search_states", jobsB)
+		logtr.Verbosef("ntg.SearchStates: in working hours so setting ntg_last_search_states: %+v", jobs)
+	}
+
+	return jobs, nil
 }
 
-func (ntg NTGVision) search(states []config.State) (io.ReadCloser, error) {
-	if config.Get().Brokers.NTG.Token == "" {
+func getNTGTokenKey() string {
+	return "brokers_ntg_token"
+}
+
+func getNTGToken() string {
+	db := config.Get().DB()
+	b, _ := db.Get(getNTGTokenKey())
+	return string(b)
+}
+
+func setNTGToken(token string) {
+	db := config.Get().DB()
+	db.Set(getNTGTokenKey(), []byte(token))
+}
+
+func (ntg NTGVision) searchStates(states []config.State) (io.ReadCloser, error) {
+	if getNTGToken() == "" {
+		logtr.Debugf("NTG token is empty, refreshing ntg auth")
 		if err := ntg.refreshAuth(); err != nil {
 			return nil, err
 		}
 	}
-	rc, err := ntg._search(states)
+	rc, err := ntg._searchStates(states)
 	if err == ErrNoAuth {
+		logtr.Debugf("err no auth on search, refreshing ntg auth")
 		if err := ntg.refreshAuth(); err != nil {
 			return nil, err
 		}
-		rc, err = ntg._search(states)
+		rc, err = ntg._searchStates(states)
 	}
 	return rc, err
 }
 
 func (ntg NTGVision) refreshAuth() error {
+	err := ntg._refreshAuth()
+	if err != nil {
+		logtr.SOSf("failed to refresh ntg auth: %v", err)
+	}
+	return err
+}
+
+func (ntg NTGVision) _refreshAuth() error {
+	logtr.Infof("refreshing ntg auth...")
 	b, _ := json.Marshal(map[string]string{
 		"username": config.Get().Brokers.NTG.Username,
 		"password": config.Get().Brokers.NTG.Password,
@@ -136,13 +361,11 @@ func (ntg NTGVision) refreshAuth() error {
 	if len(v.Token) == 0 {
 		return errors.New("failed to get token from login call")
 	}
-	conf := config.Get()
-	conf.Brokers.NTG.Token = v.Token
-	config.Set(*conf)
+	setNTGToken(v.Token)
 	return nil
 }
 
-func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
+func (ntg NTGVision) _searchStates(states []config.State) (io.ReadCloser, error) {
 	request, err := ntg.newRequest(states)
 	if err != nil {
 		return nil, err
@@ -151,10 +374,17 @@ func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
 	if err != nil {
 		return nil, err
 	}
+	defer resp.Body.Close()
+	b, _ := ioutil.ReadAll(resp.Body)
+	resp.Body = io.NopCloser(bytes.NewReader(b))
 	if resp.StatusCode != http.StatusOK {
 		b, _ := ioutil.ReadAll(resp.Body)
 		resp.Body.Close()
-		if resp.StatusCode > 400 && resp.StatusCode < 500 && resp.StatusCode != 404 && resp.StatusCode != 410 {
+		request2, _ := ntg.newRequest(states)
+		requestb, _ := ioutil.ReadAll(request2.Body)
+		logtr.Debugf("ntg auth bad status: url=%s, status=%v, body=%s, headers=%+v, request=%+v, requestb=%s", request.URL.String(), resp.StatusCode, b, resp.Header, request2, requestb)
+		if resp.StatusCode > 400 && resp.StatusCode < 404 || resp.StatusCode == 417 { // TODO wtf is 417 for
+			logtr.Debugf("ntg auth bad status: err no auth")
 			return nil, ErrNoAuth
 		}
 		return nil, fmt.Errorf("bad status searching ntg: %d: %s", resp.StatusCode, b)
@@ -164,7 +394,7 @@ func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
 
 func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) {
 	body, err := json.Marshal(map[string]interface{}{
-		"OriginFromDate":        time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
+		"OriginFromDate":        time.Now().Add(time.Hour * -24).UTC().Format("2006-01-02T15:04:05.000Z"),
 		"OriginToDate":          time.Now().UTC().Add(time.Hour * 24 * 30).Format("2006-01-02T15:04:05.000Z"),
 		"DestinationFromDate":   nil,
 		"DestinationToDate":     nil,
@@ -187,7 +417,8 @@ func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) {
 		return nil, err
 	}
 	setNTGHeaders(request)
-	request.Header.Set("Authorization", "Bearer "+config.Get().Brokers.NTG.Token)
+	request.Header.Set("Authorization", "Bearer "+getNTGToken())
+	request.Header.Set("Cookie", "NTGAuthToken="+getNTGToken())
 
 	return request, nil
 }
@@ -198,7 +429,7 @@ func setNTGHeaders(request *http.Request) {
 	request.Header.Set("Accept-Language", "en-US,en;q=0.5")
 	request.Header.Set("Accept-Encoding", "gzip, deflate, br")
 	request.Header.Set("Content-Type", "application/json;charset=utf-8")
-	//request.Header.Set("Authorization", "Bearer "+config.Get().Brokers.NTG.Token)
+	//request.Header.Set("Authorization", "Bearer "+getNTGToken())
 	request.Header.Set("Origin", "https://ntgvision.com")
 	request.Header.Set("DNT", "1")
 	request.Header.Set("Connection", "keep-alive")

broker/ntgvision_mock.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"io"
 	"io/ioutil"
-	"local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
 	"os"
 	"path"
 )
@@ -15,8 +15,14 @@ func NewNTGVisionMock() NTGVisionMock {
 	return NTGVisionMock{}
 }
 
-func (ntgm NTGVisionMock) search(states []config.State) (io.ReadCloser, error) {
+func (ntgm NTGVisionMock) searchStates(states []config.State) (io.ReadCloser, error) {
 	path := path.Join(os.Getenv("GOPATH"), "src", "local", "truckstop", "broker", "testdata", "ntgvision_response.json")
 	b, err := ioutil.ReadFile(path)
 	return io.NopCloser(bytes.NewReader(b)), err
 }
+
+func (ntgm NTGVisionMock) searchJobReadCloser(id int64) (io.ReadCloser, error) {
+	path := path.Join(os.Getenv("GOPATH"), "src", "local", "truckstop", "broker", "testdata", "ntgvision_jobinfo_response.json")
+	b, err := ioutil.ReadFile(path)
+	return io.NopCloser(bytes.NewReader(b)), err
+}

broker/ntgvision_test.go

@@ -0,0 +1,22 @@
+package broker
+
+import (
+	"gogs.inhome.blapointe.com/local/truckstop/config"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestWorkingHoursNTG(t *testing.T) {
+	os.Setenv("CONFIG", "../config.json")
+	if err := config.Refresh(nil); err != nil {
+		t.Fatal(err)
+	}
+	ntg := NTGVision{}
+	if !ntg.workingHours(time.Date(2022, 1, 27, 12, 0, 0, 0, time.FixedZone("MST", -7*60*60))) {
+		t.Fatal("noon MST not ok")
+	}
+	if ntg.workingHours(time.Date(2022, 1, 27, 23, 0, 0, 0, time.FixedZone("MST", -7*60*60))) {
+		t.Fatal("midnight MST ok")
+	}
+}

broker/testdata/fastexact_login.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+
+curl -i -Ss \
+   'https://www.fastexact.com/secure/index.php?page=userLogin' \
+   -X POST \
+   -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' \
+   -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \
+   -H 'Accept-Language: en-US,en;q=0.5' \
+   -H 'Accept-Encoding: gzip, deflate, br' \
+   -H 'Content-Type: application/x-www-form-urlencoded' \
+   --data-raw 'user_name=birdman&user_password=166647&buttonSubmit=Login' \
+

broker/testdata/fastexact_search.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+curl -sS \
+   'https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode=27006&records_per_page=10&distance=300&st_loc_zip=8' \
+   -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' \
+   -H 'Accept: */*' \
+   -H 'Accept-Language: en-US,en;q=0.5' \
+   -H 'Accept-Encoding: gzip, deflate, br' \
+   -H 'Content-Type: application/x-www-form-urlencoded' \
+   -H 'Cookie: PHPSESSID=vq4ss3lbfg0viluaau7e0d32q6'  \
+| gzip -d

broker/testdata/fastexact_search.xml

@@ -0,0 +1,218 @@
+<form name="frm_search" action="index.php?page=manageJobs" method="post">
+<a href="https://play.google.com/store/apps/details?id=app.FastExact" target="_blank" title="Android app, Available in the App Store, Download and get all more..." class="android_app"></a>
+<a href="http://itunes.apple.com/us/app/fastexact/id497933887?mt=8" target="_blank" title="iPhone app, Available in the App Store, Download and get all more..." class="iphone_app"></a>
+  <div class="search_box w67p">
+    <div class="header"><strong>Search</strong> <span class="usZipLink"><a href="http://zip4.usps.com/zip4/citytown.jsp" title="US Zip Code Finder" target="_blank">US Zip Code Finder &raquo;</a></span> </div>
+    <div class="content">
+      <table border="0" cellspacing="0" cellpadding="0">
+        <tr>
+          <td valign="top" width="342"><strong>Enter your Zip Code to show radius
+            <!--of 100 miles-->
+            from your location</strong></td>
+          <td valign="top" width="68"><input type="text" name="zipcode" id="zipcode" value="27006" onKeyPress="return disableEnterKey(event)" onkeydown="if(event.keyCode == 13){radiusSearch('10')}" /></td>
+          <td valign="top" width="59"><select name="distance" id="distance" class="select" onkeydown="if(event.keyCode == 13){radiusSearch('10')}">
+              <option value="100" >100</option>
+              <option value="200" >200</option>
+              <option value="300" selected=selected>300</option>
+            </select></td>
+          <td valign="top" width="128"><!--<input type="checkbox" name="st_loc_zip" id="st_loc_zip" value="1" class="ownBox" checked=checked onKeyPress="return disableEnterKey(event)" onkeydown="if(event.keyCode == 13){radiusSearch('10')}"/>--></td>
+          <td valign="top" class="padr12" width="337">Make this your current zip code<br />
+            for the next
+            <select name="st_loc_zip" id="st_loc_zip" class="dropdown">
+                            <option value="1">1</option>
+                            <option value="2">2</option>
+                            <option value="3">3</option>
+                            <option value="4">4</option>
+                            <option value="5">5</option>
+                            <option value="6">6</option>
+                            <option value="7">7</option>
+                            <option value="8" selected="selected">8</option>
+                            <option value="9">9</option>
+                            <option value="10">10</option>
+                            <option value="11">11</option>
+                            <option value="12">12</option>
+                            <option value="13">13</option>
+                            <option value="14">14</option>
+                            <option value="15">15</option>
+                            <option value="16">16</option>
+                            <option value="17">17</option>
+                            <option value="18">18</option>
+                            <option value="19">19</option>
+                            <option value="20">20</option>
+                            <option value="21">21</option>
+                            <option value="22">22</option>
+                            <option value="23">23</option>
+                            <option value="24">24</option>
+                          </select>
+            hours.<a href="javascript:radiusSearch('10','save');" ><img src="images/buttons/save.png"  /></a></td>
+          <td valign="top"><a href="javascript:radiusSearch('10','search');"><img src="images/buttons/search.gif" alt="Search" align="middle" title="Search" /></a></td>
+        </tr>
+      </table>
+    </div>
+  </div>
+  <div class="br"></div>
+</form>
+<br clear="all" />
+<div id="list" align="center">
+  <table width="100%" border="0" cellspacing="0" cellpadding="0" summary="Order List">
+    <tr>
+      <th width="14%" height="38" align="center">ID&nbsp;&nbsp;<span id="sort8"><a href="javascript:contentSort('ordernum=8&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',8)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
+      <th width="14%" height="38" align="center">Origin&nbsp;&nbsp;<span id="sort1"><a href="javascript:contentSort('ordernum=1&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',1)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
+      <th  width="14%" height="38" align="center">Destination&nbsp;&nbsp;<span id="sort2"><a href="javascript:contentSort('ordernum=2&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',2)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
+      <th width="14%" height="38" align="center">Mile&nbsp;&nbsp;<span id="sort3"><a href="javascript:contentSort('ordernum=3&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',3)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
+      <th height="38" align="center">Pieces / Weight&nbsp;&nbsp;<span id="sort4"><a href="javascript:contentSort('ordernum=4&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',4)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0"/></a></span></th>
+      <th width="14%" height="38" align="center">Dimension <span id="sort5"><a href="javascript:contentSort('ordernum=5&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',5)"><img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0"/></a></span></th>
+	  <th width="14%" height="38" align="center">Vehicle Size <span id="sort5"><a href="javascript:contentSort('ordernum=5&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',5)"><img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0"/></a></span></th>
+      <th width="20%" height="38" align="center">Pickup (EST)&nbsp;&nbsp;<span id="sort6"><a href="javascript:contentSort('ordernum=6&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',6)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
+      <th width="6%" height="38" align="center">View</th>
+    </tr>
+        <tr >
+      <td align="center" height="38">Z-5344222</td>
+      <td align="center" height="38">Circleville, OH 43113</td>
+      <td align="center">Willard, OH 44890</td>
+      <td align="center">128</td>
+      <td align="center">4 / 4800</td>
+      <td align="center">96x48x88in.</td>
+      <td align="center">SMALL STRAIGHT</td>
+      <td align="center">31-Jan-2022  17:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344222"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr class="alternate">
+      <td align="center" height="38">Z-5344003</td>
+      <td align="center" height="38">WASHINGTON, DC</td>
+      <td align="center">WINSTON SALEM, NC</td>
+      <td align="center">332</td>
+      <td align="center">2 / 3000</td>
+      <td align="center">78x35x99 in.</td>
+      <td align="center">LARGE STRAIGHT</td>
+      <td align="center">31-Jan-2022  08:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344003"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr >
+      <td align="center" height="38">Z-5344092</td>
+      <td align="center" height="38">Winchester, VA 22603</td>
+      <td align="center">Milwaukee, WI 53221</td>
+      <td align="center">728</td>
+      <td align="center">4 / 2195</td>
+      <td align="center">NO DIMENSIONS SPECIFIED</td>
+      <td align="center">VAN</td>
+      <td align="center">30-Jan-2022  15:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344092"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr class="alternate">
+      <td align="center" height="38">Z-5344035</td>
+      <td align="center" height="38">ATLANTA, GA 30301</td>
+      <td align="center">BURNSVILLE, MN 55337</td>
+      <td align="center">1098</td>
+      <td align="center">10 / 6000</td>
+      <td align="center">48x40x40 in.</td>
+      <td align="center">LARGE STRAIGHT</td>
+      <td align="center">29-Jan-2022  08:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344035"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr >
+      <td align="center" height="38">Z-5344518</td>
+      <td align="center" height="38">Hamlet, NC 28345</td>
+      <td align="center">East Windsor, CT 06088</td>
+      <td align="center">720</td>
+      <td align="center">1 / 500</td>
+      <td align="center">54x36x24in.</td>
+      <td align="center">CARGO VAN</td>
+      <td align="center">28-Jan-2022  13:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344518"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr class="alternate">
+      <td align="center" height="38">X-5344304</td>
+      <td align="center" height="38">MOUNT PLEASANT, PA 15666</td>
+      <td align="center">MIAMI, FL 33178-1056</td>
+      <td align="center">1156</td>
+      <td align="center">0 / 5000.00</td>
+      <td align="center">48Inches                                                             x                                                             48Inches                                                             x                                                      </td>
+      <td align="center">Straight Truck 20-24 ft</td>
+      <td align="center">28-Jan-2022  13:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344304"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr >
+      <td align="center" height="38">Z-5344061</td>
+      <td align="center" height="38">Chesapeake, VA 23321</td>
+      <td align="center">Fort Wayne, IN 46818</td>
+      <td align="center">727</td>
+      <td align="center">0 / 5000</td>
+      <td align="center">NO DIMENSIONS SPECIFIED</td>
+      <td align="center">LARGE STRAIGHT</td>
+      <td align="center">28-Jan-2022  13:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344061"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr class="alternate">
+      <td align="center" height="38">B-5344514</td>
+      <td align="center" height="38">MAXTON, NC 28364</td>
+      <td align="center">PEMBINA, ND 58271</td>
+      <td align="center">1637</td>
+      <td align="center">7 / 28000</td>
+      <td align="center">7@54x96x40</td>
+      <td align="center"></td>
+      <td align="center">28-Jan-2022  12:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344514"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr >
+      <td align="center" height="38">Z-5344114</td>
+      <td align="center" height="38">Durham, NC 27703</td>
+      <td align="center">Cumming, GA 30040</td>
+      <td align="center">368</td>
+      <td align="center">6 / 3200</td>
+      <td align="center">48x40x48in.</td>
+      <td align="center">LARGE STRAIGHT</td>
+      <td align="center">28-Jan-2022  12:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344114"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr class="alternate">
+      <td align="center" height="38">Z-5344231</td>
+      <td align="center" height="38">Durham, NC 27702</td>
+      <td align="center">Cumming, GA 30040</td>
+      <td align="center">366</td>
+      <td align="center">6 / 3200</td>
+      <td align="center">48x40x48in.</td>
+      <td align="center">LARGE STRAIGHT</td>
+      <td align="center">28-Jan-2022  12:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344231"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+          </table>
+</div>
+<br clear="all" />
+<div class="paging">
+  <table cellspacing="0" cellpadding="0" border="0" width="100%">
+    <tbody>
+      <tr>
+        <td>Details Found: 81<img height="1" width="14" alt="" src="images/tspacer.gif">Page 1 of 9</td>
+        <td align="center"><div id="Page">
+                        <img height="1" width="8" alt="" src="images/tspacer.gif">
+                        <a href="javascript:fnPaging('pageNum=2&records_per_page=10&zipcode=27006&distance=300&st_loc_zip=8')">Next</a> <strong>&raquo;</strong>
+                      </div></td>
+        <td align="right"><table cellspacing="0" cellpadding="0" border="0">
+            <tbody>
+              <tr>
+                <td valign="top" class="record">Records Per Page:&nbsp;</td>
+                <td valign="top"><div id="recNum">
+                    <select style="font-size: 11px;" class="dropdown" id="pagei" name="pagei" onchange="changeRecordCount('27006','300','8')">
+                      <option value="10" selected="selected" >10</option>
+                      <option value="20" >20</option>
+                      <option value="50" >50</option>
+                      <option value="100" >100</option>
+                    </select>
+                  </div></td>
+              </tr>
+            </tbody>
+          </table></td>
+      </tr>
+    </tbody>
+  </table>

broker/testdata/ntgvision_jobinfo_response.json

@@ -0,0 +1,84 @@
+{
+  "headerSummary": "A to B",
+  "loadId": 123,
+  "miles": 123,
+  "weight": 123,
+  "temp": null,
+  "equipment": "equipment",
+  "categoryName": "category",
+  "categoryLabel": "category generic",
+  "cargoInformation": [
+    "carto info"
+  ],
+  "loadRequirements": [
+    "load requirements"
+  ],
+  "stopInfos": [
+    {
+      "loadInfoIds": [
+        123
+      ],
+      "stopDateTime": "0001-01-01T00:00:00",
+      "isPickup": true,
+      "location": "A",
+      "stopDate": "Friday, 01/14/22",
+      "stopHours": "10:00 - 12:00",
+      "appointmentTime": "",
+      "appointmentType": "FCFS",
+      "instructions": "instruction",
+      "isDropTrailer": false,
+      "shipperName": "client",
+      "processedStopDate": ""
+    },
+    {
+      "loadInfoIds": [
+        123
+      ],
+      "stopDateTime": "0001-01-01T00:00:00",
+      "isPickup": false,
+      "location": "B",
+      "stopDate": "Monday, 01/17/22",
+      "stopHours": "09:00 - 10:00",
+      "appointmentTime": "09:00",
+      "appointmentType": "APPT",
+      "instructions": "instructions",
+      "isDropTrailer": false,
+      "shipperName": "client",
+      "processedStopDate": ""
+    }
+  ],
+  "drayStopInfos": null,
+  "rateInfo": null,
+  "isDray": false,
+  "equipmentGroupId": 8,
+  "totalCarrierRate": 2600,
+  "payUpTo": 2700,
+  "firstLoadInfoId": 123,
+  "loadStatus": "ACTIVE",
+  "hasBlockingAlert": false,
+  "customerLoadBlocked": false,
+  "canSeeRate": false,
+  "canBookNow": false,
+  "canBidNow": true,
+  "buttonData": {
+    "useBidDialog": false,
+    "lastBidAmount": null,
+    "remainingBids": 1,
+    "carrierPhoneNumber": null,
+    "carrierPhoneExtension": null
+  },
+  "stopData": [
+    {
+      "addr": "IDX NORTH CAROLINA",
+      "city": "Washington",
+      "state": "NC",
+      "zip": "27889"
+    },
+    {
+      "addr": "hibbett sports",
+      "city": "Abilene",
+      "state": "TX",
+      "zip": "79606"
+    }
+  ]
+}

broker/testdata/ntgvision_response.json

@@ -1,21 +1,4 @@
 [
-  {
-    "id": 4650337,
-    "sDate": "01/12/22",
-    "sCity": "Columbus",
-    "sState": "OH",
-    "sdh": null,
-    "cDate": "01/13/22",
-    "cCity": "Jamaica",
-    "cState": "NY",
-    "cdh": null,
-    "stopCnt": 2,
-    "miles": 578,
-    "weight": 5000,
-    "equip": "Str Truck W/ Lift Gate",
-    "temp": "",
-    "alertReasons": []
-  },
   {
     "id": 4650338,
     "sDate": "01/12/22",
@@ -23,8 +6,8 @@
     "sState": "NC",
     "sdh": null,
     "cDate": "01/13/22",
-    "cCity": "Atlanta",
-    "cState": "GA",
+    "cCity": "Winston-Salem",
+    "cState": "NC",
     "cdh": null,
     "stopCnt": 2,
     "miles": 378,

config.json

@@ -1,8 +1,22 @@
 {
+   "Log": {
+      "Path": "/tmp/truckstop.log",
+      "Level": "DEB",
+      "SOSMatrix": {
+         "ReceiveEnabled": true,
+         "Mock": false,
+         "Homeserver": "https://m.bltrucks.top",
+         "Username": "@bot.m.bltrucks.top",
+         "Token": "mvDWB96KXMF8XhOam8EC5XVdQvSEw0CDeClcSWocBcYkwZX3FPNWZ5uOnQk2EmT1cjpzfeuD7gDYPPjOuyZlI3bE9TE35UjNOlZgi0Tugm25s91iVsbIF6kMZsCIhVMSmEf6w3jxX6wQYOWvmDZ4mu6f5c8wr221EMDcOpEzQV09d1zuBSWgKLBgjqAkYHJZ5dTRIWpEDpPgujhOFZa2ld1HiAOxrJKlIrlfDBN0CUsTlGOGplujDAr4VtpFzNRS",
+         "Device": "TGNIOGKATZ",
+         "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
+      }
+   },
    "Interval": {
       "Input": "5s..10s",
       "OK": "6h0m0s..6h0m0s",
-      "Error": "6h0m0s..6h0m0s"
+      "Error": "6h0m0s..6h0m0s",
+      "JobInfo": "10s..20s"
    },
    "Images": {
       "ClientID": "d9ac7cabe813d10",
@@ -16,41 +30,33 @@
       "UploadMethod": "POST"
    },
    "Maps": {
-      "DirectionsURIFormat": "https://maps.googleapis.com/maps/api/directions/json?origin=%s\u0026destination=%s\u0026mode=driving\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
-      "PathedURIFormat": "https://maps.googleapis.com/maps/api/staticmap?size=250x250\u0026path=%s\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg\u0026size=250x250\u0026markers=%s|%s\u0026zoom=5",
       "URIFormat": "https://maps.googleapis.com/maps/api/staticmap?center=%s\u0026markers=label=A|%s\u0026zoom=5\u0026size=250x250\u0026scale=1\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
-      "Pathed": true,
       "Pickup": false,
-      "Dropoff": false
+      "Dropoff": false,
+      "Pathed": {
+         "Enabled": false,
+         "DirectionsURIFormat": "https://maps.googleapis.com/maps/api/directions/json?origin=%s\u0026destination=%s\u0026mode=driving\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
+         "PathedURIFormat": "https://maps.googleapis.com/maps/api/staticmap?size=250x250\u0026path=%s\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg\u0026size=250x250\u0026markers=%s|%s",
+         "Zoom": {
+            "AcceptableLatLngDelta": 2,
+            "Override": 5
+         }
+      }
    },
    "Clients": {
       "bel": {
          "States": [
             "NC"
          ],
+         "Zips": [
+            "27006",
+            "73044",
+            "84058"
+         ],
          "IDs": {
             "Matrix": "@bel:m.bltrucks.top"
          },
          "Available": 1512328400
-      },
-      "broc": {
-         "States": [
-            "FL",
-            "NC"
-         ],
-         "IDs": {
-            "Matrix": "@broc:m.bltrucks.top"
-         },
-         "Available": 5642452800
-      },
-      "pa": {
-         "States": [
-            "NC"
-         ],
-         "IDs": {
-            "Matrix": "@ron:m.bltrucks.top"
-         },
-         "Available": -62135596800
       }
    },
    "Storage": [
@@ -61,7 +67,6 @@
       "Matrix": {
          "ReceiveEnabled": true,
          "Mock": false,
-         "Continuation": "1314",
          "Homeserver": "https://m.bltrucks.top",
          "Username": "@bot.m.bltrucks.top",
          "Token": "mvDWB96KXMF8XhOam8EC5XVdQvSEw0CDeClcSWocBcYkwZX3FPNWZ5uOnQk2EmT1cjpzfeuD7gDYPPjOuyZlI3bE9TE35UjNOlZgi0Tugm25s91iVsbIF6kMZsCIhVMSmEf6w3jxX6wQYOWvmDZ4mu6f5c8wr221EMDcOpEzQV09d1zuBSWgKLBgjqAkYHJZ5dTRIWpEDpPgujhOFZa2ld1HiAOxrJKlIrlfDBN0CUsTlGOGplujDAr4VtpFzNRS",
@@ -69,14 +74,28 @@
          "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
       }
    },
-   "Once": true,
+   "Once": false,
    "Brokers": {
+      "UseZips": true,
+      "RadiusMiles": 200,
       "NTG": {
+         "Working": {
+            "Hours": [6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
+            "Weekdays": [1, 2, 3, 4, 5]
+         },
+         "Enabled": false,
+         "JobInfo": true,
          "Mock": true,
          "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",
-         "Token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIzMTAyOSIsInVuaXF1ZV9uYW1lIjoibm9lYXN5cnVuc3RydWNraW5nQGdtYWlsLmNvbSIsImp0aSI6IjFmYzhmNjk1LTQzNTYtNGYzZS05NWY1LWZkNWVjMDJlMDkxNyIsImlhdCI6IjEvMTAvMjAyMiAxMTo1OTozNiBQTSIsIm50Z3ZSb2xlIjoiQ2FycmllckFwcHJvdmVkIiwidXNlckNhcnJpZXJzIjoiMTUzNDIzIiwib3RyVXNlciI6IkZhbHNlIiwibmJmIjoxNjQxODU5MTc2LCJleHAiOjE2NDE5NDE5NzYsImlzcyI6Ik5URyBTZWN1cml0eSBUb2tlbiBTZXJ2aWNlIiwiYXVkIjoiTlRHIn0.kpHOBTtcQhbdloAw7xEjnkzfxf4ToMgidrLCMomZmnmKQHlD_7OQuI8nQiCTHc_ntuGtt8Ui92kwWWUiLwN_2tT2vC7Jy6m9IjwqgbAzsgTLi4jAbIwITD-awiDh4FUKDwGq3XpEjs-i7XM3rI7tTk7jg9QSDId8EF3Pt5fJq6QhztC6y7-JaSFQZLMtkSCAWmOQx_TgKgVoVbgMeiqhHbZ2hhoA7TtpEIIL5Gnfq46t3E18ExdrsO96ZCGQGcBw5x8J1ustq2cwdlFKeg4ULNWAAd1ay1hojRa7jCHs98AcoJ3Nts9-o7yEMuN2rrfpK_nm68nciwFtF-ke1KoiBg",
+         "LoadPageAPIURIFormat": "https://ntgvision.com/api/v1/load/LoadDetails?loadId=%d",
          "Username": "noeasyrunstrucking@gmail.com",
-         "Password": "thumper123"
+         "Password": "thumper12345"
+      },
+      "FastExact": {
+         "Enabled": true,
+         "Mock": true,
+         "Username": "birdman",
+         "Password": "166647"
       }
    }
 }

config.main_test.json

@@ -0,0 +1,99 @@
+{
+   "Log": {
+      "Path": "/tmp/truckstop.log",
+      "Level": "sos",
+      "SOSMatrix": {
+         "ReceiveEnabled": true,
+         "Mock": true,
+         "Homeserver": "https://m.bltrucks.top",
+         "Username": "@bot.m.bltrucks.top",
+         "Token": "mvDWB96KXMF8XhOam8EC5XVdQvSEw0CDeClcSWocBcYkwZX3FPNWZ5uOnQk2EmT1cjpzfeuD7gDYPPjOuyZlI3bE9TE35UjNOlZgi0Tugm25s91iVsbIF6kMZsCIhVMSmEf6w3jxX6wQYOWvmDZ4mu6f5c8wr221EMDcOpEzQV09d1zuBSWgKLBgjqAkYHJZ5dTRIWpEDpPgujhOFZa2ld1HiAOxrJKlIrlfDBN0CUsTlGOGplujDAr4VtpFzNRS",
+         "Device": "TGNIOGKATZ",
+         "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
+      }
+   },
+   "Interval": {
+      "Input": "5s..10s",
+      "OK": "6h0m0s..6h0m0s",
+      "Error": "6h0m0s..6h0m0s",
+      "JobInfo": "10s..20s"
+   },
+   "Images": {
+      "ClientID": "d9ac7cabe813d10",
+      "ClientSecret": "9d0b3d82800b30ca88f595d3bcd6985f627d7d82",
+      "RefreshToken": "171417741bf762b99b0b9f9137491b7a69874a77",
+      "AccessToken": "e63db98f92d2db7ac7f56914a2030c889b378e9b",
+      "RefreshURI": "https://api.imgur.com/oauth2/token",
+      "RefreshFormat": "refresh_token=%s\u0026client_id=%s\u0026client_secret=%s\u0026grant_type=refresh_token",
+      "RefreshMethod": "POST",
+      "UploadURI": "https://api.imgur.com/3/image?name=something.jpeg",
+      "UploadMethod": "POST"
+   },
+   "Maps": {
+      "URIFormat": "https://maps.googleapis.com/maps/api/staticmap?center=%s\u0026markers=label=A|%s\u0026zoom=5\u0026size=250x250\u0026scale=1\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
+      "Pickup": false,
+      "Dropoff": false,
+      "Pathed": {
+         "Enabled": true,
+         "DirectionsURIFormat": "https://maps.googleapis.com/maps/api/directions/json?origin=%s\u0026destination=%s\u0026mode=driving\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
+         "PathedURIFormat": "https://maps.googleapis.com/maps/api/staticmap?size=250x250\u0026path=%s\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg\u0026size=250x250\u0026markers=%s|%s",
+         "Zoom": {
+            "AcceptableLatLngDelta": 2,
+            "Override": 5
+         }
+      }
+   },
+   "Clients": {
+      "bel": {
+         "States": [
+            "NC"
+         ],
+         "Zips": [
+            "27006"
+         ],
+         "IDs": {
+            "Matrix": "@bel:m.bltrucks.top"
+         },
+         "Available": 1512328400
+      }
+   },
+   "Storage": [
+      "files",
+      "/tmp/play"
+   ],
+   "Message": {
+      "Matrix": {
+         "ReceiveEnabled": true,
+         "Mock": true,
+         "Homeserver": "https://m.bltrucks.top",
+         "Username": "@bot.m.bltrucks.top",
+         "Token": "mvDWB96KXMF8XhOam8EC5XVdQvSEw0CDeClcSWocBcYkwZX3FPNWZ5uOnQk2EmT1cjpzfeuD7gDYPPjOuyZlI3bE9TE35UjNOlZgi0Tugm25s91iVsbIF6kMZsCIhVMSmEf6w3jxX6wQYOWvmDZ4mu6f5c8wr221EMDcOpEzQV09d1zuBSWgKLBgjqAkYHJZ5dTRIWpEDpPgujhOFZa2ld1HiAOxrJKlIrlfDBN0CUsTlGOGplujDAr4VtpFzNRS",
+         "Device": "TGNIOGKATZ",
+         "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
+      }
+   },
+   "Once": true,
+   "Brokers": {
+      "UseZips": true,
+		"RadiusMiles": 100,
+      "FastExact": {
+         "Enabled": true,
+         "Mock": true,
+         "Username": "u",
+         "Password": "p"
+      },
+      "NTG": {
+         "Working": {
+            "Hours": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23],
+            "Weekdays": [0, 1, 2, 3, 4, 5, 6]
+         },
+         "Enabled": true,
+         "JobInfo": true,
+         "Mock": true,
+         "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",
+         "LoadPageAPIURIFormat": "https://ntgvision.com/api/v1/load/LoadDetails?loadId=%d",
+         "Username": "noeasyrunstrucking@gmail.com",
+         "Password": "thumper1234"
+      }
+   }
+}

config/config.go

@@ -3,17 +3,34 @@ package config
 import (
 	"encoding/json"
 	"io/ioutil"
-	"local/storage"
+	"gogs.inhome.blapointe.com/local/storage"
+	"gogs.inhome.blapointe.com/local/truckstop/logtr"
 	"os"
 	"sync"
 	"time"
 )
 
+type Matrix struct {
+	ReceiveEnabled bool
+	Mock           bool
+	Homeserver     string
+	Username       string
+	Token          string
+	Device         string
+	Room           string
+}
+
 type Config struct {
+	Log struct {
+		Path      string
+		Level     logtr.Level
+		SOSMatrix Matrix
+	}
 	Interval struct {
-		Input Duration
-		OK    Duration
-		Error Duration
+		Input   Duration
+		OK      Duration
+		Error   Duration
+		JobInfo Duration
 	}
 	Images struct {
 		ClientID      string
@@ -27,35 +44,46 @@ type Config struct {
 		UploadMethod  string
 	}
 	Maps struct {
-		DirectionsURIFormat string
-		PathedURIFormat     string
-		URIFormat           string
-		Pathed              bool
-		Pickup              bool
-		Dropoff             bool
+		URIFormat string
+		Pickup    bool
+		Dropoff   bool
+		Pathed    struct {
+			Enabled             bool
+			DirectionsURIFormat string
+			PathedURIFormat     string
+			Zoom                struct {
+				AcceptableLatLngDelta float32
+				Override              int
+			}
+		}
 	}
 	Clients map[string]Client
 	Storage []string
 	Message struct {
-		Matrix struct {
-			ReceiveEnabled bool
-			Mock           bool
-			Continuation   string
-			Homeserver     string
-			Username       string
-			Token          string
-			Device         string
-			Room           string
-		}
+		Matrix Matrix
 	}
 	Once    bool
 	Brokers struct {
-		NTG struct {
-			Mock              bool
-			LoadPageURIFormat string
-			Token             string
-			Username          string
-			Password          string
+		UseZips     bool
+		RadiusMiles int
+		NTG         struct {
+			Working struct {
+				Hours    []int
+				Weekdays []int
+			}
+			Enabled              bool
+			JobInfo              bool
+			Mock                 bool
+			LoadPageURIFormat    string
+			LoadPageAPIURIFormat string
+			Username             string
+			Password             string
+		}
+		FastExact struct {
+			Enabled  bool
+			Mock     bool
+			Username string
+			Password string
 		}
 	}
 
@@ -65,6 +93,7 @@ type Config struct {
 
 type Client struct {
 	States []State
+	Zips   []string
 	IDs    struct {
 		Matrix string
 	}
@@ -92,6 +121,20 @@ func Clients(t time.Time) map[string]Client {
 	return result
 }
 
+func AllZips() []string {
+	zipm := map[string]struct{}{}
+	for _, v := range Clients(time.Now().Add(time.Hour * 24 * 365)) {
+		for _, state := range v.Zips {
+			zipm[state] = struct{}{}
+		}
+	}
+	zips := make([]string, 0, len(zipm)+1)
+	for k := range zipm {
+		zips = append(zips, k)
+	}
+	return zips
+}
+
 func AllStates() []State {
 	statem := map[State]struct{}{}
 	for _, v := range Clients(time.Now().Add(time.Hour * 24 * 365)) {
@@ -106,7 +149,7 @@ func AllStates() []State {
 	return states
 }
 
-func Refresh() error {
+func Refresh(soser func() logtr.SOSer) error {
 	b, err := ioutil.ReadFile(configPath())
 	if err != nil {
 		return err
@@ -115,10 +158,15 @@ func Refresh() error {
 	if err := json.Unmarshal(b, &c); err != nil {
 		return err
 	}
+	logtr.SetLogpath(c.Log.Path)
+	logtr.SetLevel(c.Log.Level)
 	if live.db != nil {
 		live.db.Close()
 	}
 	live = c
+	if soser != nil {
+		logtr.SetSOSer(soser())
+	}
 	return nil
 }
 
@@ -132,7 +180,7 @@ func Set(other Config) {
 		return
 	}
 	ioutil.WriteFile(configPath(), b, os.ModePerm)
-	Refresh()
+	Refresh(nil)
 }
 
 func (c *Config) DB() storage.DB {

config/state.go

@@ -7,56 +7,56 @@ import (
 
 type State string
 
-var States = map[string]State{
-	"99654": "AL",
-	"72401": "AR",
-	"85364": "AZ",
-	"90011": "CA",
-	"80013": "CO",
-	"06902": "CT",
-	"19720": "DE",
-	"33024": "FL",
-	"30043": "GA",
-	"96706": "HI",
-	"50613": "IA",
-	"83646": "ID",
-	"60629": "IL",
-	"47906": "IN",
-	"66062": "KS",
-	"40475": "KY",
-	"70726": "LA",
-	"02301": "MA",
-	"20906": "MD",
-	"04401": "ME",
-	"48197": "MI",
-	"55106": "MN",
-	"63376": "MO",
-	"39503": "MS",
-	"59901": "MT",
-	"27006": "NC",
-	"58103": "ND",
-	"68516": "NE",
-	"03103": "NH",
-	"08701": "NJ",
-	"87121": "NM",
-	"89108": "NV",
-	"11368": "NY",
-	"45011": "OH",
-	"73099": "OK",
-	"97006": "OR",
-	"19120": "PA",
-	"02860": "RI",
-	"29483": "SC",
-	"57106": "SD",
-	"37013": "TN",
-	"77449": "TX",
-	"84058": "UT",
-	"22193": "VA",
-	"05401": "VT",
-	"99301": "WA",
-	"53215": "WI",
-	"26554": "WV",
-	"82001": "WY",
+var States = map[State]string{
+	"AL": "99654",
+	"AR": "72401",
+	"AZ": "85364",
+	"CA": "90011",
+	"CO": "80013",
+	"CT": "06902",
+	"DE": "19720",
+	"FL": "33024",
+	"GA": "30043",
+	"HI": "96706",
+	"IA": "50613",
+	"ID": "83646",
+	"IL": "60629",
+	"IN": "47906",
+	"KS": "66062",
+	"KY": "40475",
+	"LA": "70726",
+	"MA": "02301",
+	"MD": "20906",
+	"ME": "04401",
+	"MI": "48197",
+	"MN": "55106",
+	"MO": "63376",
+	"MS": "39503",
+	"MT": "59901",
+	"NC": "27006",
+	"ND": "58103",
+	"NE": "68516",
+	"NH": "03103",
+	"NJ": "08701",
+	"NM": "87121",
+	"NV": "89108",
+	"NY": "11368",
+	"OH": "45011",
+	"OK": "73099",
+	"OR": "97006",
+	"PA": "19120",
+	"RI": "02860",
+	"SC": "29483",
+	"SD": "57106",
+	"TN": "37013",
+	"TX": "77449",
+	"UT": "84058",
+	"VA": "22193",
+	"VT": "05401",
+	"WA": "99301",
+	"WI": "53215",
+	"WV": "26554",
+	"WY": "82001",
 }
 
 func (state *State) UnmarshalJSON(b []byte) error {
@@ -65,8 +65,8 @@ func (state *State) UnmarshalJSON(b []byte) error {
 		return err
 	}
 	for k := range States {
-		if string(States[k]) == s {
-			*state = States[k]
+		if string(k) == s {
+			*state = k
 			return nil
 		}
 	}

go.mod

@@ -1,31 +1,22 @@
-module local/truckstop
+module gogs.inhome.blapointe.com/local/truckstop
 
 go 1.17
 
-replace local/storage => ../storage
-
-replace local/logb => ../logb
-
-replace local/sandbox/contact/contact => ../sandbox/contact/contact
-
 require (
+	github.com/PuerkitoBio/goquery v1.8.0
+	github.com/google/uuid v1.3.0
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	local/sandbox/contact/contact v0.0.0-00010101000000-000000000000
-	local/storage v0.0.0-00010101000000-000000000000
+	gogs.inhome.blapointe.com/local/storage v0.0.0-20230410162102-db39d7b02e29
+	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c
 )
 
 require (
 	cloud.google.com/go v0.33.1 // indirect
 	github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/aws/aws-sdk-go v1.15.81 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
-	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect
-	github.com/buraksezer/consistent v0.9.0 // indirect
-	github.com/bytbox/go-pop3 v0.0.0-20120201222208-3046caf0763e // indirect
-	github.com/cespare/xxhash v1.1.0 // indirect
-	github.com/emersion/go-imap v1.2.0 // indirect
-	github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/golang/protobuf v1.2.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
@@ -54,17 +45,16 @@ require (
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	go.mongodb.org/mongo-driver v1.7.2 // indirect
+	gogs.inhome.blapointe.com/local/logb v0.0.0-20230410154319-880efa39d871 // indirect
 	golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect
 	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
 	golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
-	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
+	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
 	google.golang.org/api v0.0.0-20181120235003-faade3cbb06a // indirect
 	google.golang.org/appengine v1.3.0 // indirect
 	gopkg.in/ini.v1 v1.42.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	local/logb v0.0.0-00010101000000-000000000000 // indirect
 )

go.sum

@@ -5,27 +5,21 @@ github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9a
 github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 h1:6X8iB881g299aNEv6KXrcjL31iLOH7yA6NXoQX+MbDg=
 github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
 github.com/a8m/tree v0.0.0-20180321023834-3cf936ce15d6/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=
 github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
 github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
 github.com/anacrolix/dms v0.0.0-20180117034613-8af4925bffb5/go.mod h1:DGqLjaZ3ziKKNRt+U5Q9PLWJ52Q/4rxfaaH/b3QYKaE=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/aws/aws-sdk-go v1.15.81 h1:va7uoFaV9uKAtZ6BTmp1u7paoMsizYRRLvRuoC07nQ8=
 github.com/aws/aws-sdk-go v1.15.81/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
 github.com/billziss-gh/cgofuse v1.1.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0=
-github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
-github.com/buraksezer/consistent v0.9.0 h1:Zfs6bX62wbP3QlbPGKUhqDw7SmNkOzY5bHZIYXYpR5g=
-github.com/buraksezer/consistent v0.9.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw=
-github.com/bytbox/go-pop3 v0.0.0-20120201222208-3046caf0763e h1:mQTN05gz0rDZSABqKMzAPMb5ATWcvvdMljRzEh0LjBo=
-github.com/bytbox/go-pop3 v0.0.0-20120201222208-3046caf0763e/go.mod h1:alXX+s7a4cKaIprgjeEboqi4Tm7XR/HXEwUTxUV/ywU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/coreos/bbolt v0.0.0-20180318001526-af9db2027c98/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -34,12 +28,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/djherbis/times v1.1.0/go.mod h1:CGMZlo255K5r4Yw0b9RRfFQpM2y7uOmxg4jm9HsaVf8=
 github.com/dropbox/dropbox-sdk-go-unofficial v5.4.0+incompatible/go.mod h1:lr+LhMM3F6Y3lW1T9j2U5l7QeuWm87N9+PPXo3yH4qY=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/emersion/go-imap v1.2.0 h1:lyUQ3+EVM21/qbWE/4Ya5UG9r5+usDxlg4yfp3TgHFA=
-github.com/emersion/go-imap v1.2.0/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY=
-github.com/emersion/go-message v0.15.0/go.mod h1:wQUEfE+38+7EW8p8aZ96ptg6bAb1iwdgej19uXASlE4=
-github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ=
-github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
-github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -174,8 +162,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -205,6 +191,10 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul
 github.com/yunify/qingstor-sdk-go v2.2.15+incompatible/go.mod h1:w6wqLDQ5bBTzxGJ55581UrSwLrsTAsdo9N6yX/8d9RY=
 go.mongodb.org/mongo-driver v1.7.2 h1:pFttQyIiJUHEn50YfZgC9ECjITMT44oiN36uArf/OFg=
 go.mongodb.org/mongo-driver v1.7.2/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
+gogs.inhome.blapointe.com/local/logb v0.0.0-20230410154319-880efa39d871 h1:cMGPiwvK/QGg4TfW8VasO6SsS/O7UQmwyKDErV/ozoA=
+gogs.inhome.blapointe.com/local/logb v0.0.0-20230410154319-880efa39d871/go.mod h1:E0pLNvMLzY0Kth1W078y+06z1AUyVMWnChMpRFf4w2Q=
+gogs.inhome.blapointe.com/local/storage v0.0.0-20230410162102-db39d7b02e29 h1:SPSz7yQsEfScqyLlBS5uNSOGeT203BkIkFCL8jrm/FA=
+gogs.inhome.blapointe.com/local/storage v0.0.0-20230410162102-db39d7b02e29/go.mod h1:zk8Fe2Ezc2f6oOe2yllsbEhXqssUU1K2faoS0eQ9alY=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -219,6 +209,7 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -242,8 +233,9 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

logtr/log.go

@@ -0,0 +1,139 @@
+package logtr
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+type Level int
+
+const (
+	VERBOSE = Level(7)
+	DEBUG   = Level(8)
+	INFO    = Level(9)
+	ERROR   = Level(10)
+	SOS     = Level(15)
+)
+
+type SOSer interface {
+	Send(string) error
+}
+
+func (l Level) MarshalJSON() ([]byte, error) {
+	return json.Marshal(l.String())
+}
+
+func (l *Level) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+	s = strings.TrimSpace(s)
+	s = strings.ToUpper(s)
+	if len(s) > 3 {
+		s = s[:3]
+	}
+	for i := 0; i < int(SOS)+5; i++ {
+		l2 := Level(i)
+		if l2.String() == s {
+			*l = l2
+			return nil
+		}
+	}
+	return errors.New("unknown log level: " + s)
+}
+
+var logger io.Writer = os.Stderr
+var loggerPath string = ""
+var lock = &sync.Mutex{}
+var level Level = INFO
+var ansoser SOSer = nil
+
+func SetLogpath(p string) {
+	lock.Lock()
+	defer lock.Unlock()
+	if p == loggerPath {
+		return
+	}
+	f, err := os.OpenFile(p, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		panic(err)
+	}
+	logger = f
+	loggerPath = p
+}
+
+func SetSOSer(another SOSer) {
+	lock.Lock()
+	defer lock.Unlock()
+	ansoser = another
+}
+
+func SetLevel(l Level) {
+	lock.Lock()
+	defer lock.Unlock()
+	level = l
+}
+
+func logf(l Level, format string, args []interface{}) {
+	format = fmt.Sprintf("%v: %v: %s\n", time.Now().Format("01-02T15:04:05"), l.String(), strings.TrimSpace(format))
+	logContent := strings.ReplaceAll(fmt.Sprintf(format, args...), "\n", "") + "\n"
+	cLevel := level
+	cAnsoser := ansoser
+	if l >= cLevel {
+		fmt.Fprint(os.Stderr, logContent)
+	}
+	fmt.Fprint(logger, logContent)
+	if l == SOS && cAnsoser != nil {
+		if err := cAnsoser.Send(logContent); err != nil {
+			Errorf("failed to SOS: %v", err)
+		}
+	}
+}
+
+func SOSf(format string, args ...interface{}) {
+	logf(SOS, format, args)
+}
+
+func Infof(format string, args ...interface{}) {
+	logf(INFO, format, args)
+}
+
+func Printf(format string, args ...interface{}) {
+	Infof(format, args...)
+}
+
+func Debugf(format string, args ...interface{}) {
+	logf(DEBUG, format, args)
+}
+
+func Verbosef(format string, args ...interface{}) {
+	logf(VERBOSE, format, args)
+}
+
+func Errorf(format string, args ...interface{}) {
+	logf(ERROR, format, args)
+}
+
+func (l Level) String() string {
+	switch l {
+	case ERROR:
+		return "ERR"
+	case INFO:
+		return "INF"
+	case DEBUG:
+		return "DEB"
+	case VERBOSE:
+		return "VER"
+	case SOS:
+		return "SOS"
+	default:
+		return "?"
+	}
+}

main.go

@@ -4,64 +4,75 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"local/storage"
-	"local/truckstop/broker"
-	"local/truckstop/config"
-	"local/truckstop/message"
-	"log"
+	"gogs.inhome.blapointe.com/local/storage"
+	"gogs.inhome.blapointe.com/local/truckstop/broker"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/logtr"
+	"gogs.inhome.blapointe.com/local/truckstop/message"
 	"net/http"
 	"net/url"
 	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 )
 
 var stateFinder = regexp.MustCompile(`[A-Za-z]+`)
+var zipFinder = regexp.MustCompile(`[0-9]{5}[0-9]*`)
 
 func main() {
-	if err := config.Refresh(); err != nil {
+	if err := _main(); err != nil {
 		panic(err)
 	}
+}
+
+func _main() error {
+	if err := config.Refresh(message.NewSOSMatrix); err != nil {
+		return err
+	}
 	if config.Get().Message.Matrix.ReceiveEnabled {
 		if err := matrixrecv(); err != nil {
-			panic(err)
+			logtr.SOSf("failed to recv matrix on boot: %v", err)
+			return err
 		}
 	}
 	lock := &sync.Mutex{}
 	go func() {
 		for {
 			time.Sleep(config.Get().Interval.Input.Get())
-			if err := config.Refresh(); err != nil {
-				log.Println(err)
+			if err := config.Refresh(message.NewSOSMatrix); err != nil {
+				logtr.Errorf("failed parsing config: %v", err)
 			} else {
 				if config.Get().Message.Matrix.ReceiveEnabled {
 					lock.Lock()
 					if err := matrixrecv(); err != nil {
-						log.Print(err)
+						logtr.Errorf("failed receiving and parsing matrix: %v", err)
 					}
 					lock.Unlock()
 				}
 			}
 		}
 	}()
-	if err := _main(); err != nil {
-		panic(err)
+	if err := __main(); err != nil {
+		logtr.SOSf("failed __main: %v", err)
+		return err
 	}
 	lock.Lock()
+	return nil
 }
 
 func matrixrecv() error {
-	log.Printf("checking matrix...")
-	defer log.Printf("/checking matrix...")
+	logtr.Verbosef("checking matrix...")
+	defer logtr.Verbosef("/checking matrix...")
 	sender := message.NewMatrix()
 	messages, err := sender.Receive()
 	if err != nil {
 		return err
 	}
 	func() {
-		log.Printf("looking for help")
+		logtr.Verbosef("looking for help")
 		printed := false
 		for _, msg := range messages {
 			if !strings.HasPrefix(msg.Content, "!help") {
@@ -71,26 +82,56 @@ func matrixrecv() error {
 			db := config.Get().DB()
 			if !printed {
 				if _, err := db.Get(key); err == storage.ErrNotFound {
-					log.Printf("sending help")
-					help := fmt.Sprintf("commands:\n   `!help`   print this help\n   `!state nc NC nC Nc`   set states for self\n   `!available 2022-12-31`   set date self is available for work\n\nrun a command for someone else: `!state ga @caleb`")
+					logtr.Debugf("sending help")
+					locationHelp := "`!state nc NC nC Nc`   set states for self"
+					if config.Get().Brokers.UseZips {
+						locationHelp = "...`!zip 27006 84058` to set zip codes for self\n...`!radius 100` to set miles radius to 100, 200, or 300"
+					}
+					help := fmt.Sprintf("commands:\n...`!help`   print this help\n%s\n\nrun a command for someone else: `!zip 2022-12-31 @caleb`", locationHelp)
 					if err := sender.Send(help); err != nil {
-						log.Printf("failed to send help: %v", err)
+						logtr.Errorf("failed to send help: %v", err)
 					} else {
 						printed = true
 						if err := db.Set(key, []byte{'k'}); err != nil {
-							log.Printf("failed to mark help given @%s: %v", key, err)
+							logtr.Errorf("failed to mark help given @%s: %v", key, err)
 						}
 					}
 				}
 			} else {
 				if err := db.Set(key, []byte{'k'}); err != nil {
-					log.Printf("failed to mark help given @%s: %v", key, err)
+					logtr.Errorf("failed to mark help given @%s: %v", key, err)
 				}
 			}
 		}
 	}()
 	func() {
-		log.Printf("looking for states")
+		logtr.Verbosef("looking for zips")
+		db := config.Get().DB()
+		zips := map[string]map[string]struct{}{}
+		for _, msg := range messages {
+			key := fmt.Sprintf("zips_%d", msg.Timestamp.Unix())
+			if !strings.HasPrefix(msg.Content, "!zip") {
+				continue
+			}
+			if _, ok := zips[msg.Sender]; ok {
+				continue
+			}
+			if _, err := db.Get(key); err == storage.ErrNotFound {
+				zips[msg.Sender] = map[string]struct{}{}
+				for _, zip := range parseOutZips([]byte(msg.Content)) {
+					zips[msg.Sender][zip] = struct{}{}
+				}
+			}
+			if err := db.Set(key, []byte{'k'}); err != nil {
+				logtr.Errorf("failed to mark state gathered @%s: %v", key, err)
+			}
+		}
+		if config.Get().Brokers.UseZips {
+			setNewZips(zips)
+		}
+	}()
+	func() {
+		logtr.Verbosef("looking for states")
 		db := config.Get().DB()
 		states := map[string]map[config.State]struct{}{}
 		for _, msg := range messages {
@@ -108,13 +149,40 @@ func matrixrecv() error {
 				}
 			}
 			if err := db.Set(key, []byte{'k'}); err != nil {
-				log.Printf("failed to mark state gathered @%s: %v", key, err)
+				logtr.Errorf("failed to mark state gathered @%s: %v", key, err)
 			}
 		}
-		setNewStates(states)
+		if !config.Get().Brokers.UseZips {
+			setNewStates(states)
+		}
 	}()
 	func() {
-		log.Printf("looking for pauses")
+		logtr.Verbosef("looking for radius")
+		db := config.Get().DB()
+		var radius *int
+		for _, msg := range messages {
+			key := fmt.Sprintf("radius_%d", msg.Timestamp.Unix())
+			if !strings.HasPrefix(msg.Content, "!radius ") {
+				continue
+			}
+			if _, err := db.Get(key); err == storage.ErrNotFound {
+				cmd := msg.Content
+				for strings.HasPrefix(cmd, "!radius ") {
+					cmd = strings.TrimPrefix(cmd, "!radius ")
+				}
+				n, err := strconv.Atoi(strings.TrimSpace(cmd))
+				if err == nil {
+					radius = &n
+				}
+			}
+			if err := db.Set(key, []byte{'k'}); err != nil {
+				logtr.Errorf("failed to mark radius gathered @%s: %v", key, err)
+			}
+		}
+		setNewRadius(radius)
+	}()
+	func() {
+		logtr.Verbosef("looking for pauses")
 		db := config.Get().DB()
 		pauses := map[string]time.Time{}
 		for _, msg := range messages {
@@ -136,24 +204,47 @@ func matrixrecv() error {
 				}
 			}
 			if err := db.Set(key, []byte{'k'}); err != nil {
-				log.Printf("failed to mark state gathered @%s: %v", key, err)
+				logtr.Errorf("failed to mark pause gathered @%s: %v", key, err)
 			}
 		}
 		setNewPauses(pauses)
 	}()
-	conf := *config.Get()
-	if conf.Message.Matrix.Continuation != sender.Continuation() {
-		conf.Message.Matrix.Continuation = sender.Continuation()
-		config.Set(conf)
-	}
+	message.SetMatrixContinuation(sender.Continuation())
 	return nil
 }
 
+func setNewRadius(radius *int) {
+	if radius == nil {
+		return
+	}
+	sender := message.NewMatrix()
+	logtr.Debugf("set new radius: %d", *radius)
+	acceptable := map[int]struct{}{
+		100: struct{}{},
+		200: struct{}{},
+		300: struct{}{},
+	}
+	if _, ok := acceptable[*radius]; !ok {
+		sender.Send(fmt.Sprintf("radius of %v is not among acceptable %+v", *radius, acceptable))
+		logtr.Debugf("bad radius, not setting: %d", *radius)
+		return
+	}
+	conf := *config.Get()
+	if conf.Brokers.RadiusMiles == *radius {
+		logtr.Debugf("noop radius, not setting: %d", *radius)
+		return
+	}
+	conf.Brokers.RadiusMiles = *radius
+	logtr.Infof("updating config new pauses: %+v", conf)
+	config.Set(conf)
+	sender.Send(fmt.Sprintf("now using radius of %d mi for zip code search", *radius))
+}
+
 func setNewPauses(pauses map[string]time.Time) {
 	if len(pauses) == 0 {
 		return
 	}
-	log.Printf("set new pauses: %+v", pauses)
+	logtr.Debugf("set new pauses: %+v", pauses)
 	conf := *config.Get()
 	changed := map[string]time.Time{}
 	for client, pause := range pauses {
@@ -168,11 +259,46 @@ func setNewPauses(pauses map[string]time.Time) {
 	if len(changed) == 0 {
 		return
 	}
-	log.Printf("updating config new pauses: %+v", conf)
+	logtr.Infof("updating config new pauses: %+v", conf)
 	config.Set(conf)
 	for client, pause := range changed {
 		if err := sendNewPause(client, pause); err != nil {
-			log.Printf("failed to send new pause %s/%+v: %v", client, pause, err)
+			logtr.Errorf("failed to send new pause %s/%+v: %v", client, pause, err)
+		}
+	}
+}
+
+func setNewZips(zips map[string]map[string]struct{}) {
+	if len(zips) == 0 {
+		return
+	}
+	conf := *config.Get()
+	changed := map[string][]string{}
+	for client, clientZips := range zips {
+		newzips := []string{}
+		for k := range clientZips {
+			newzips = append(newzips, k)
+		}
+		sort.Slice(newzips, func(i, j int) bool {
+			return newzips[i] < newzips[j]
+		})
+		clientconf := conf.Clients[client]
+		if fmt.Sprint(newzips) == fmt.Sprint(clientconf.Zips) {
+			message.NewMatrix().Send(fmt.Sprintf("%s: still searching for %+v", client, newzips))
+			continue
+		}
+		clientconf.Zips = newzips
+		conf.Clients[client] = clientconf
+		changed[client] = newzips
+	}
+	if len(changed) == 0 {
+		return
+	}
+	logtr.Infof("updating config new zips: %+v", conf)
+	config.Set(conf)
+	for client, zips := range changed {
+		if err := sendNewZips(client, zips); err != nil {
+			logtr.Errorf("failed to send new zips %s/%+v: %v", client, zips, err)
 		}
 	}
 }
@@ -203,15 +329,27 @@ func setNewStates(states map[string]map[config.State]struct{}) {
 	if len(changed) == 0 {
 		return
 	}
-	log.Printf("updating config new states: %+v", conf)
+	logtr.Infof("updating config new states: %+v", conf)
 	config.Set(conf)
 	for client, states := range changed {
 		if err := sendNewStates(client, states); err != nil {
-			log.Printf("failed to send new states %s/%+v: %v", client, states, err)
+			logtr.Errorf("failed to send new states %s/%+v: %v", client, states, err)
 		}
 	}
 }
 
+func parseOutZips(b []byte) []string {
+	var zips []string
+	candidates := zipFinder.FindAll(b, -1)
+	for _, candidate := range candidates {
+		if len(candidate) != 5 {
+			continue
+		}
+		zips = append(zips, string(candidate))
+	}
+	return zips
+}
+
 func parseOutStates(b []byte) []config.State {
 	var states []config.State
 	var state config.State
@@ -230,14 +368,14 @@ func parseOutStates(b []byte) []config.State {
 	return states
 }
 
-func _main() error {
+func __main() error {
 	for {
-		err := _mainOne()
+		err := __mainOne()
 		if err != nil {
-			log.Println(err)
+			logtr.Errorf("failed _main: %v", err)
 		}
 		if config.Get().Once {
-			time.Sleep(time.Second)
+			time.Sleep(10 * time.Second)
 			return err
 		}
 		if err != nil {
@@ -249,16 +387,18 @@ func _main() error {
 	return nil
 }
 
-func _mainOne() error {
-	log.Println("config.refreshing...")
-	if err := config.Refresh(); err != nil {
+func __mainOne() error {
+	logtr.Debugf("config.refreshing...")
+	if err := config.Refresh(message.NewSOSMatrix); err != nil {
+		logtr.SOSf("bad config: %v", err)
 		return err
 	}
-	log.Println("once...")
+	logtr.Debugf("once...")
 	if err := once(); err != nil {
+		logtr.SOSf("failed once(): %v", err)
 		return err
 	}
-	log.Println("/_mainOne")
+	logtr.Debugf("/__mainOne")
 	return nil
 }
 
@@ -267,23 +407,37 @@ func once() error {
 	if err != nil {
 		return err
 	}
-	log.Printf("once: all jobs: %+v", alljobs)
+	logtr.Debugf("once: update dead jobs: %+v", alljobs)
+	err = updateDeadJobs(alljobs)
+	if err != nil {
+		return err
+	}
+	logtr.Debugf("once: all jobs: %+v", alljobs)
 	newjobs, err := dropStaleJobs(alljobs)
 	if err != nil {
 		return err
 	}
-	log.Printf("once: new jobs: %+v", newjobs)
+	logtr.Debugf("once: new jobs: %+v", newjobs)
 	jobs, err := dropBanlistJobs(newjobs)
 	if err != nil {
 		return err
 	}
-	log.Printf("once: sending jobs: %+v", jobs)
+	logtr.Debugf("found jobs: %+v", jobs)
+	if len(jobs) == 0 {
+		return nil
+	}
+	logtr.Debugf("once: loading job secrets: %+v", jobs)
+	for i := range jobs {
+		jobs[i].Secrets()
+	}
+	logtr.Infof("once: sending jobs: %+v", jobs)
+	db := config.Get().DB()
 	for i := range jobs {
 		if ok, err := sendJob(jobs[i]); err != nil {
 			return err
 		} else if ok {
-			log.Println("sent job", jobs[i])
-			if err := config.Get().DB().Set(jobs[i].ID, []byte(`sent`)); err != nil {
+			logtr.Debugf("sent job %+v", jobs[i])
+			if err := db.Set(jobs[i].UID(), []byte(`sent`)); err != nil {
 				return err
 			}
 		}
@@ -292,15 +446,17 @@ func once() error {
 }
 
 func getJobs() ([]broker.Job, error) {
-	states := config.AllStates()
-	ntg := broker.NewNTGVision()
-	if config.Get().Brokers.NTG.Mock {
-		ntg = ntg.WithMock()
+	if config.Get().Brokers.UseZips {
+		return getJobsZips()
 	}
-	brokers := []broker.Broker{ntg}
+	return getJobsStates()
+}
+
+func getJobsZips() ([]broker.Job, error) {
+	zips := config.AllZips()
 	jobs := []broker.Job{}
-	for _, broker := range brokers {
-		somejobs, err := broker.Search(states)
+	for _, broker := range getBrokers() {
+		somejobs, err := broker.SearchZips(zips)
 		if err != nil {
 			return nil, err
 		}
@@ -309,10 +465,84 @@ func getJobs() ([]broker.Job, error) {
 	return jobs, nil
 }
 
+func getJobsStates() ([]broker.Job, error) {
+	states := config.AllStates()
+	jobs := []broker.Job{}
+	for _, broker := range getBrokers() {
+		somejobs, err := broker.SearchStates(states)
+		if err != nil {
+			return nil, err
+		}
+		jobs = append(jobs, somejobs...)
+	}
+	return jobs, nil
+}
+
+func getBrokers() []broker.Broker {
+	brokers := []broker.Broker{}
+	if config.Get().Brokers.NTG.Enabled {
+		logtr.Debugf("NTG enabled")
+		brokers = append(brokers, broker.NewNTGVision())
+	}
+	if config.Get().Brokers.FastExact.Enabled {
+		logtr.Debugf("FastExact enabled")
+		brokers = append(brokers, broker.NewFastExact())
+	}
+	return brokers
+}
+
+type recordedJob struct {
+	Job            broker.Job
+	SentNS         int64
+	MatrixID       string
+	MatrixImageIDs []string
+}
+
+func updateDeadJobs(jobs []broker.Job) error {
+	db := config.Get().DB()
+	list, err := db.List([]string{}, "sent_job_", "sent_job_}}")
+	if err != nil {
+		return err
+	}
+	for _, listEntry := range list {
+		wouldBe := strings.TrimPrefix(listEntry, "sent_job_")
+		found := false
+		for i := range jobs {
+			found = found || jobs[i].UID() == wouldBe || jobs[i].ID == wouldBe
+		}
+		logtr.Debugf("found job %s to be still alive==%v", wouldBe, found)
+		if !found {
+			logtr.Debugf("updating dead job %+v", listEntry)
+			b, err := db.Get(listEntry)
+			if err != nil {
+				return err
+			}
+			var recorded recordedJob
+			if err := json.Unmarshal(b, &recorded); err != nil {
+				return err
+			}
+			/* // TODO this beeps on fluffychat
+			if err := message.NewMatrix().Update(recorded.MatrixID, recorded.Job.FormatMultilineTextDead()); err != nil {
+				return err
+			}
+			*/
+			if err := db.Set(listEntry, nil); err != nil {
+				logtr.Debugf("failed to remove db: %v: %v", listEntry, err)
+			}
+			for _, imageid := range recorded.MatrixImageIDs {
+				if err := message.NewMatrix().Update(imageid, "<job no longer available>"); err != nil {
+					logtr.Debugf("failed to update matrix image: %v: %v", imageid, err)
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func dropStaleJobs(jobs []broker.Job) ([]broker.Job, error) {
 	db := config.Get().DB()
 	for i := len(jobs) - 1; i >= 0; i-- {
-		if _, err := db.Get(jobs[i].ID); err == storage.ErrNotFound {
+		if _, err := db.Get(jobs[i].UID()); err == storage.ErrNotFound {
 		} else if err != nil {
 			return nil, err
 		} else {
@@ -330,18 +560,36 @@ func dropBanlistJobs(jobs []broker.Job) ([]broker.Job, error) {
 func sendJob(job broker.Job) (bool, error) {
 	sender := message.NewMatrix()
 	payload := job.FormatMultilineText()
-	log.Printf("once: send job %s if nonzero: %s", job.String(), payload)
+	logtr.Debugf("once: send job %s if nonzero: %q", job.String(), payload)
 	if len(payload) == 0 {
 		return false, nil
 	}
-	if err := sender.Send(payload); err != nil {
+	id, err := sender.SendTracked(payload)
+	if err != nil {
 		return false, err
 	}
+	recordedJob := recordedJob{
+		Job:      job,
+		SentNS:   time.Now().UnixNano(),
+		MatrixID: id,
+	}
+	defer func() {
+		db := config.Get().DB()
+		b, err := json.Marshal(recordedJob)
+		if err != nil {
+			logtr.Errorf("failed to marshal recorded job: %v", err)
+			return
+		}
+		if err := db.Set("sent_job_"+job.UID(), b); err != nil {
+			logtr.Errorf("failed to set recorded job: %v", err)
+			return
+		}
+	}()
 	maps := config.Get().Maps
 	pickup := fmt.Sprintf("%s,%s", url.QueryEscape(job.Pickup.City), job.Pickup.State)
 	dropoff := fmt.Sprintf("%s,%s", url.QueryEscape(job.Dropoff.City), job.Dropoff.State)
-	if maps.Pathed {
-		directionsURI := fmt.Sprintf(maps.DirectionsURIFormat, pickup, dropoff)
+	if maps.Pathed.Enabled {
+		directionsURI := fmt.Sprintf(maps.Pathed.DirectionsURIFormat, pickup, dropoff)
 		resp, err := http.Get(directionsURI)
 		if err != nil {
 			return true, err
@@ -368,37 +616,68 @@ func sendJob(job broker.Job) (bool, error) {
 		if len(directionsResp.Routes) < 1 || len(directionsResp.Routes[0].Legs) < 1 || len(directionsResp.Routes[0].Legs[0].Steps) < 1 {
 		} else {
 			latLngPath := make([]string, 0)
+			minLat := directionsResp.Routes[0].Legs[0].Steps[0].StartLocation.Lat
+			maxLat := directionsResp.Routes[0].Legs[0].Steps[0].StartLocation.Lat
+			minLng := directionsResp.Routes[0].Legs[0].Steps[0].StartLocation.Lng
+			maxLng := directionsResp.Routes[0].Legs[0].Steps[0].StartLocation.Lng
 			for _, v := range directionsResp.Routes[0].Legs[0].Steps {
+				if v.StartLocation.Lng < minLng {
+					minLng = v.StartLocation.Lng
+				}
+				if v.StartLocation.Lng > maxLng {
+					maxLng = v.StartLocation.Lng
+				}
+				if v.StartLocation.Lat < minLat {
+					minLat = v.StartLocation.Lat
+				}
+				if v.StartLocation.Lat > maxLat {
+					maxLat = v.StartLocation.Lat
+				}
 				latLngPath = append(latLngPath, fmt.Sprintf("%.9f,%.9f", v.StartLocation.Lat, v.StartLocation.Lng))
 			}
 			pathQuery := strings.Join(latLngPath, "|")
-			uri := fmt.Sprintf(maps.PathedURIFormat, pathQuery, pickup, dropoff)
-			log.Printf("sending pathed image: %s", uri)
-			if err := sender.SendImage(uri); err != nil {
+			uri := fmt.Sprintf(maps.Pathed.PathedURIFormat, pathQuery, pickup, dropoff)
+			// if the bigger delta is <acceptable, override zoom
+			if maxLat-minLat <= maps.Pathed.Zoom.AcceptableLatLngDelta && maxLng-minLng <= maps.Pathed.Zoom.AcceptableLatLngDelta {
+				uri = fmt.Sprintf("%s&zoom=%d", uri, maps.Pathed.Zoom.Override)
+			}
+			logtr.Debugf("sending pathed image: %s", uri)
+			pathedid, err := sender.SendImageTracked(uri)
+			if err != nil {
 				return true, err
 			}
+			recordedJob.MatrixImageIDs = append(recordedJob.MatrixImageIDs, pathedid)
 		}
 	}
 	if maps.Pickup {
 		uri := fmt.Sprintf(maps.URIFormat, pickup, pickup)
-		log.Printf("sending pickup image: %s", uri)
-		if err := sender.SendImage(uri); err != nil {
+		logtr.Debugf("sending pickup image: %s", uri)
+		pickupid, err := sender.SendImageTracked(uri)
+		if err != nil {
 			return true, err
 		}
+		recordedJob.MatrixImageIDs = append(recordedJob.MatrixImageIDs, pickupid)
 	}
 	if maps.Dropoff {
 		uri := fmt.Sprintf(maps.URIFormat, dropoff, dropoff)
-		log.Printf("sending dropoff image: %s", uri)
-		if err := sender.SendImage(uri); err != nil {
+		logtr.Debugf("sending dropoff image: %s", uri)
+		dropid, err := sender.SendImageTracked(uri)
+		if err != nil {
 			return true, err
 		}
+		recordedJob.MatrixImageIDs = append(recordedJob.MatrixImageIDs, dropid)
 	}
 	return true, nil
 }
 
+func sendNewZips(client string, zips []string) error {
+	sender := message.NewMatrix()
+	return sender.Send(fmt.Sprintf("%s: now searching for loads from zip codes: %+v", client, zips))
+}
+
 func sendNewStates(client string, states []config.State) error {
 	sender := message.NewMatrix()
-	return sender.Send(fmt.Sprintf("%s: now searching for loads from: %+v", client, states))
+	return sender.Send(fmt.Sprintf("%s: now searching for loads from states: %+v", client, states))
 }
 
 func sendNewPause(client string, pause time.Time) error {

main_test.go

@@ -2,7 +2,8 @@ package main
 
 import (
 	"fmt"
-	"local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
+	"os"
 	"testing"
 )
 
@@ -33,3 +34,10 @@ Content-Type: text/html; charset="UTF-8"
 		t.Fatal(want, got)
 	}
 }
+
+func TestMain(t *testing.T) {
+	os.Setenv("CONFIG", "./config.main_test.json")
+	if err := _main(); err != nil {
+		t.Fatal(err)
+	}
+}

message/images.go

@@ -6,8 +6,8 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"local/truckstop/config"
-	"log"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/logtr"
 	"mime/multipart"
 	"net/http"
 	"net/url"
@@ -40,7 +40,7 @@ func uploadImage(b []byte) (string, error) {
 	} else if s, ok := u.Query()["name"]; !ok {
 	} else {
 		name = s[0]
-		log.Printf("found name in upload uri: %s", name)
+		logtr.Debugf("found name in upload uri: %s", name)
 	}
 	part, err := writer.CreateFormFile("image", name)
 	if err != nil {
@@ -64,7 +64,7 @@ func uploadImage(b []byte) (string, error) {
 	request.Header.Set("Authorization", "Bearer "+images.AccessToken)
 	request.Header.Set("Content-Type", writer.FormDataContentType())
 
-	c := &http.Client{Timeout: time.Minute}
+	c := &http.Client{Timeout: time.Minute * 5}
 	response, err := c.Do(request)
 	if err != nil {
 		return "", err

message/images_test.go

@@ -2,7 +2,7 @@ package message
 
 import (
 	"io/ioutil"
-	"local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
 	"os"
 	"testing"
 )
@@ -16,7 +16,7 @@ func TestImageUpload(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := config.Refresh(); err != nil {
+	if err := config.Refresh(nil); err != nil {
 		t.Fatal(err)
 	}
 	got, err := UploadImage(b)

message/matrix.go

@@ -2,10 +2,11 @@ package message
 
 import (
 	"bytes"
+	"crypto/tls"
 	"fmt"
 	"io/ioutil"
-	"local/truckstop/config"
-	"log"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/logtr"
 	"net/http"
 	"regexp"
 	"strings"
@@ -23,20 +24,49 @@ type Matrix struct {
 	continuation string
 }
 
+func NewSOSMatrix() logtr.SOSer {
+	conf := config.Get().Log.SOSMatrix
+	return newMatrix(conf, "0")
+}
+
 func NewMatrix() Matrix {
 	conf := config.Get().Message.Matrix
+	return newMatrix(conf, GetMatrixContinuation())
+}
+
+func newMatrix(conf config.Matrix, cont string) Matrix {
 	return Matrix{
 		homeserver:   conf.Homeserver,
 		username:     conf.Username,
 		token:        conf.Token,
 		room:         conf.Room,
 		mock:         conf.Mock,
-		continuation: conf.Continuation,
+		continuation: cont,
 	}
 }
 
+func (m Matrix) closeclient(client *gomatrix.Client) {
+	go func() {
+		client.Client.CloseIdleConnections()
+		client.StopSync()
+		client.Logout()
+	}()
+}
+
 func (m Matrix) getclient() (*gomatrix.Client, error) {
-	return gomatrix.NewClient(m.homeserver, m.username, m.token)
+	client, err := gomatrix.NewClient(m.homeserver, m.username, m.token)
+	if err != nil {
+		return nil, err
+	}
+	client.Client = &http.Client{
+		Timeout: time.Minute * 5,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+	}
+	return client, nil
 }
 
 func (m Matrix) Continuation() string {
@@ -45,10 +75,10 @@ func (m Matrix) Continuation() string {
 
 func (m *Matrix) Receive() ([]Message, error) {
 	if m.mock {
-		log.Printf("matrix.Receive()")
+		logtr.Infof("matrix.Receive()")
 		messages := make([]Message, 0)
 		for k := range config.Get().Clients {
-			messages = append(messages, Message{Timestamp: time.Now(), Sender: k, Content: "!state OH"})
+			messages = append(messages, Message{Timestamp: time.Now(), Sender: k, Content: "!state nc"})
 			if k == "bel" {
 				messages = append(messages, Message{Timestamp: time.Now(), Sender: k, Content: "!help"})
 			}
@@ -70,15 +100,16 @@ func (m *Matrix) Receive() ([]Message, error) {
 	if err != nil {
 		return nil, err
 	}
+	defer m.closeclient(c)
 	messages := make([]Message, 0)
-	result, err := c.Messages(m.room, "999999999999999999", m.continuation, 'b', 50)
+	result, err := c.Messages(m.room, "999999999999999999", m.Continuation(), 'b', 50)
 	if err != nil {
 		return nil, err
 	}
-	log.Printf("%s => {Start:%s End:%v};; %v, (%d)", m.continuation, result.Start, result.End, err, len(result.Chunk))
+	logtr.Verbosef("%s => {Start:%s End:%v};; %v, (%d)", m.Continuation(), result.Start, result.End, err, len(result.Chunk))
 	m.continuation = result.End
 	for _, event := range result.Chunk {
-		//log.Printf("%+v", event)
+		logtr.Verbosef("matrix event: %+v", event)
 		if _, ok := matrixIDs[event.Sender]; !ok {
 			continue
 		}
@@ -91,7 +122,7 @@ func (m *Matrix) Receive() ([]Message, error) {
 		}
 	}
 	clientChange := regexp.MustCompile("@[a-z]+$")
-	log.Printf("rewriting messages based on @abc")
+	logtr.Verbosef("rewriting messages based on @abc")
 	for i := range messages {
 		if found := clientChange.FindString(messages[i].Content); found != "" {
 			messages[i].Content = strings.TrimSpace(strings.ReplaceAll(messages[i].Content, found, ""))
@@ -105,7 +136,7 @@ func (m *Matrix) Receive() ([]Message, error) {
 		}
 		messages[i].Content = strings.TrimSpace(messages[i].Content)
 	}
-	log.Printf("rewriting messages based on ! CoMmAnD ...")
+	logtr.Verbosef("rewriting messages based on ! CoMmAnD ...")
 	for i := range messages {
 		if strings.HasPrefix(messages[i].Content, "!") {
 			messages[i].Content = "!" + strings.TrimSpace(messages[i].Content[1:])
@@ -116,48 +147,134 @@ func (m *Matrix) Receive() ([]Message, error) {
 	return messages, nil
 }
 
-func (m Matrix) Send(text string) error {
+func (m Matrix) Remove(id string) error {
 	if m.mock {
-		log.Printf("matrix.Send(%s)", text)
+		logtr.Infof("matrix.Remove(%s)", id)
 		return nil
 	}
 	c, err := m.getclient()
 	if err != nil {
 		return err
 	}
-	_, err = c.SendText(m.room, text)
+	defer m.closeclient(c)
+	_, err = c.RedactEvent(m.room, id, &gomatrix.ReqRedact{Reason: "stale"})
 	return err
 }
 
-func (m Matrix) SendImage(uri string) error {
+func (m Matrix) Update(id, text string) error {
 	if m.mock {
-		log.Printf("matrix.SendImage(%s)", uri)
+		logtr.Infof("matrix.Update(%s)", text)
 		return nil
 	}
+	c, err := m.getclient()
+	if err != nil {
+		return err
+	}
+	defer m.closeclient(c)
+	type MRelatesTo struct {
+		EventID string `json:"event_id"`
+		RelType string `json:"rel_type"`
+	}
+	type NewContent struct {
+		Body    string `json:"body"`
+		MsgType string `json:"msgtype"`
+	}
+	type RelatesToRoomMessage struct {
+		Body        string     `json:"body"`
+		MsgType     string     `json:"msgtype"`
+		MRelatesTo  MRelatesTo `json:"m.relates_to"`
+		MNewContent NewContent `json:"m.new_content"`
+	}
+	_, err = c.SendMessageEvent(m.room, "m.room.message", RelatesToRoomMessage{
+		Body:    "",
+		MsgType: "m.text",
+		MNewContent: NewContent{
+			Body:    text,
+			MsgType: "m.text",
+		},
+		MRelatesTo: MRelatesTo{
+			EventID: id,
+			RelType: "m.replace",
+		},
+	})
+	return err
+}
+
+func (m Matrix) Send(text string) error {
+	_, err := m.SendTracked(text)
+	return err
+}
+
+func (m Matrix) SendTracked(text string) (string, error) {
+	if m.mock {
+		logtr.Infof("matrix.SendTracked(%s)", text)
+		return "", nil
+	}
+	c, err := m.getclient()
+	if err != nil {
+		return "", err
+	}
+	defer m.closeclient(c)
+	resp, err := c.SendText(m.room, text)
+	if err != nil {
+		return "", err
+	}
+	return resp.EventID, nil
+}
+
+func (m Matrix) SendImage(uri string) error {
+	_, err := m.SendImageTracked(uri)
+	return err
+}
+
+func (m Matrix) SendImageTracked(uri string) (string, error) {
+	if m.mock {
+		logtr.Infof("matrix.SendImage(%s)", uri)
+		return "", nil
+	}
 	response, err := http.Get(uri)
 	if err != nil {
-		return err
+		return "", err
 	}
 	if response.StatusCode != http.StatusOK {
 		b, _ := ioutil.ReadAll(response.Body)
 		response.Body.Close()
-		return fmt.Errorf("failed to get %s: (%d) %s", uri, response.StatusCode, b)
+		return "", fmt.Errorf("failed to get %s: (%d) %s", uri, response.StatusCode, b)
 	}
 	b, err := ioutil.ReadAll(response.Body)
 	response.Body.Close()
 	if err != nil {
-		return err
+		return "", err
 	}
 	c, err := m.getclient()
 	if err != nil {
-		return err
+		return "", err
 	}
+	defer m.closeclient(c)
 	mediaUpload, err := c.UploadToContentRepo(bytes.NewReader(b), "image/jpeg", int64(len(b)))
 	if err != nil {
-		return err
+		return "", err
 	}
 	publicURI := mediaUpload.ContentURI
 	resp, err := c.SendImage(m.room, "img", publicURI)
-	log.Printf("sent image %s => %s: %+v", uri, publicURI, resp)
-	return err
+	logtr.Verbosef("sent image %s => %s: %+v", uri, publicURI, resp)
+	return resp.EventID, err
+}
+
+func SetMatrixContinuation(continuation string) {
+	db := config.Get().DB()
+	db.Set(getMatrixContinuationKey(), []byte(continuation))
+}
+
+func GetMatrixContinuation() string {
+	db := config.Get().DB()
+	b, _ := db.Get(getMatrixContinuationKey())
+	if b == nil {
+		return "0"
+	}
+	return string(b)
+}
+
+func getMatrixContinuationKey() string {
+	return "matrix_continuation"
 }

message/matrix_test.go

@@ -1,57 +1,65 @@
 package message
 
 import (
-	"encoding/json"
 	"io/ioutil"
-	"local/truckstop/config"
+	"gogs.inhome.blapointe.com/local/truckstop/config"
 	"os"
+	"path"
 	"testing"
+
+	"github.com/google/uuid"
 )
 
-func TestMatrixSend(t *testing.T) {
-	if len(os.Getenv("INTEGRATION")) == 0 {
-		t.Skip("$INTEGRATION not set")
-	}
-	var c config.Config
-	b, err := ioutil.ReadFile("../config.json")
-	if err != nil {
+func TestMatrixSendDel(t *testing.T) {
+	sender := testMatrix(t)
+	if id, err := sender.SendTracked("hello world from unittest"); err != nil {
 		t.Fatal(err)
-	}
-	if err := json.Unmarshal(b, &c); err != nil {
-		t.Fatal(err)
-	}
-	var sender Sender = &Matrix{
-		homeserver: c.Message.Matrix.Homeserver,
-		username:   c.Message.Matrix.Username,
-		token:      c.Message.Matrix.Token,
-		room:       c.Message.Matrix.Room,
-	}
-	if err := sender.Send("hello world from unittest"); err != nil {
+	} else if err := sender.Remove(id); err != nil {
 		t.Fatal(err)
 	}
 }
 
-func TestMatrixReceive(t *testing.T) {
-	if len(os.Getenv("INTEGRATION")) == 0 {
-		t.Skip("$INTEGRATION not set")
-	}
-	var c config.Config
-	b, err := ioutil.ReadFile("../config.json")
+func TestMatrixUpdate(t *testing.T) {
+	// 19:32:34: VER: matrix event: {StateKey:<nil> Sender:@bot:m.bltrucks.top Type:m.room.message Timestamp:1642471594729 ID:$n0ln9TFZrko_cBNFXeh8YyICZ3fFm17Jhz3bmZcqig4 RoomID:!OYZqtInrBCn1cyz90D:m.bltrucks.top Redacts: Unsigned:map[transaction_id:go1642471594571852423] Content:map[body:hello world from unittest format: formatted_body: msgtype:m.text] PrevContent:map[]}
+	// func (m Matrix) Update(id, text string) error {
+	m := testMatrix(t)
+	uid := uuid.New().String()[:3]
+	id, err := m.SendTracked("hello, heheheh from test matrix update " + uid)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := json.Unmarshal(b, &c); err != nil {
+	err = m.Update(id, "heheheh i updated from test matrix update "+uid)
+	if err != nil {
 		t.Fatal(err)
 	}
-	var sender Sender = &Matrix{
-		homeserver: c.Message.Matrix.Homeserver,
-		username:   c.Message.Matrix.Username,
-		token:      c.Message.Matrix.Token,
-		room:       c.Message.Matrix.Room,
-	}
+	m.Receive()
+}
+
+func TestMatrixReceive(t *testing.T) {
+	sender := testMatrix(t)
 	if msgs, err := sender.Receive(); err != nil {
 		t.Fatal(err)
 	} else {
 		t.Logf("%+v", msgs)
 	}
 }
+
+func testMatrix(t *testing.T) Matrix {
+	if len(os.Getenv("INTEGRATION")) == 0 {
+		t.Skip("$INTEGRATION not set")
+	}
+	d := t.TempDir()
+	f := path.Join(d, "config.test.json")
+	b, err := ioutil.ReadFile("../config.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(f, b, os.ModePerm); err != nil {
+		t.Fatal(err)
+	}
+	os.Setenv("CONFIG", f)
+	if err := config.Refresh(nil); err != nil {
+		t.Fatal(err)
+	}
+	return NewMatrix()
+}

message/message.go

@@ -4,6 +4,8 @@ import "time"
 
 type Sender interface {
 	Send(string) error
+	SendTracked(string) (string, error)
+	Update(string, string) (string, error)
 	Receive() ([]Message, error)
 }
 

todo.yaml

@@ -1,26 +1,39 @@
 todo:
-- write-as for clients so ma can write to pa by default
+- more than NTG; blue one
+- !states emits current state
+- test each !command callbacks to matrix
+- recv-as for clients so pa receives mas commands as writes
 - continuation is garbo, but I can still do better client side to avoid get-set high level
+- no hard code jpeg or have it in multiple places
+- change matrix so I test my custom logic even if I dont fetch remote
+- warn/err/etc. on clobbering ids.matrix since clients can mess with one another
+- todo: filter out jobs like CA
+  subtasks:
+  - banlist criteria like vendors, brokers, metadata
+- set up copy for caleb, broc
+- move from main() and make more functions
+done:
+- modify old items once no longer available; drop stale jobs good candidate but requires new matrix interaction
+- from states to zip codes w/ range
+- fast exact does not use ID in UID because they spammy
+- fast exact is dumb or...?
+- try search ntg by autoinc?
+- TEST. Just like, refactor and test to shit.
+- mark jobs no longer avail by modifying in matrix;; save matrix ID over dummy payload
+- TEST its falling apart
+- help() log on truckstop for stuff like perma 403
+- figure out zoom on maps;; is there like an auto-zoom I can leverage?
+- mock is nigh useless
+- mark consumed;; save scroll id for matrix
+- todo: switch house to selfhosted for rate limit
+  tags: now
 - todo: details from ntg; stophours for pickup, appointmenttime/facitlyhours for dest
   details: |
     curl 'https://ntgvision.com/api/v1/load/LoadDetails?loadId=5088453' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' -H 'Accept: application/json, text/plain, */*' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'Authorization: Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMzgyMSIsInVuaXF1ZV9uYW1lIjoiYmlyZGNvbXBhbnlsb2dpc3RpY3NAZ21haWwuY29tIiwianRpIjoiYTFiMjkxM2YtN2RjZS00ZDQ0LWEzMDYtMTRjMTBlYjQ2MmE1IiwiaWF0IjoiMS8xMy8yMDIyIDI6MTE6NTUgUE0iLCJudGd2Um9sZSI6IkNhcnJpZXJBcHByb3ZlZCIsImxvY2tVc2VyIjoiRmFsc2UiLCJvdmVycmlkZUJsYWNrbGlzdCI6IkZhbHNlIiwic2hvd1JhdGVzIjoiRmFsc2UiLCJ1c2VyQ2FycmllcnMiOiIxMTQxOTMiLCJvdHJVc2VyIjoiRmFsc2UiLCJuYmYiOjE2NDIwODMxMTUsImV4cCI6MTY0MjE2NTkxNSwiaXNzIjoiTlRHIFNlY3VyaXR5IFRva2VuIFNlcnZpY2UiLCJhdWQiOiJOVEcifQ.eDE2Jfok79b7oD1deP_00h9g8zZ3AnY60McnJ1hfS-8YJ12L3Bl-CJeN0eBos0aj2FXHv5MrqOJ6MFCaatqjIddPJl2MvtSByGUCLBzih67O20mkCowCmZfGjSnKvUu7DHFxjzo_y4_a3cjjGPYNc2LSCVJxV9NYzSXIuXG3WvXeE3dv8ml23bJFFhMWZzhSAfWBOW4kR61sibS6BpRD8dpkKoqUfXInq_7o8jz9_PsEhPBdJydqbXwg8ifQHkaPNAojsIr2rnJ3Tf_iQpom-6oAEUAOd2D3fK1XuWtRrysnD8s41YDthmqKpni2WOk72L-sKzRxD4KX2t_gyEb3Fw' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Cookie: cookiesession1=678A3E1398901234BCDEFGHIJKLMA492; _uiq_id.711119701.3d83=516d3e744ebe2d9a.1641792415.0.1642083653..; NTGAuthToken=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMzgyMSIsInVuaXF1ZV9uYW1lIjoiYmlyZGNvbXBhbnlsb2dpc3RpY3NAZ21haWwuY29tIiwianRpIjoiYTFiMjkxM2YtN2RjZS00ZDQ0LWEzMDYtMTRjMTBlYjQ2MmE1IiwiaWF0IjoiMS8xMy8yMDIyIDI6MTE6NTUgUE0iLCJudGd2Um9sZSI6IkNhcnJpZXJBcHByb3ZlZCIsImxvY2tVc2VyIjoiRmFsc2UiLCJvdmVycmlkZUJsYWNrbGlzdCI6IkZhbHNlIiwic2hvd1JhdGVzIjoiRmFsc2UiLCJ1c2VyQ2FycmllcnMiOiIxMTQxOTMiLCJvdHJVc2VyIjoiRmFsc2UiLCJuYmYiOjE2NDIwODMxMTUsImV4cCI6MTY0MjE2NTkxNSwiaXNzIjoiTlRHIFNlY3VyaXR5IFRva2VuIFNlcnZpY2UiLCJhdWQiOiJOVEcifQ.eDE2Jfok79b7oD1deP_00h9g8zZ3AnY60McnJ1hfS-8YJ12L3Bl-CJeN0eBos0aj2FXHv5MrqOJ6MFCaatqjIddPJl2MvtSByGUCLBzih67O20mkCowCmZfGjSnKvUu7DHFxjzo_y4_a3cjjGPYNc2LSCVJxV9NYzSXIuXG3WvXeE3dv8ml23bJFFhMWZzhSAfWBOW4kR61sibS6BpRD8dpkKoqUfXInq_7o8jz9_PsEhPBdJydqbXwg8ifQHkaPNAojsIr2rnJ3Tf_iQpom-6oAEUAOd2D3fK1XuWtRrysnD8s41YDthmqKpni2WOk72L-sKzRxD4KX2t_gyEb3Fw' -H 'Sec-Fetch-Dest: empty' -H 'Sec-Fetch-Mode: cors' -H 'Sec-Fetch-Site: same-origin' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache'
     {"headerSummary":"Washington, NC to Abilene, TX","loadId":5088453,"miles":1481,"weight":6000,"temp":null,"equipment":"Str Truck W/ Lift Gate","categoryName":"Lighting / Lighting Fixtures","categoryLabel":"Product Category","cargoInformation":["Store Fixtures - 6000 lbs"],"loadRequirements":["Straight Truck with lift gate required"],"stopInfos":[{"loadInfoIds":[5431607],"stopDateTime":"0001-01-01T00:00:00","isPickup":true,"location":"Washington, NC","stopDate":"Friday, 01/14/22","stopHours":"10:00 - 12:00","appointmentTime":"","appointmentType":"FCFS","instructions":"***LIFTGATE, PALLET JACK, DRIVER ASSIST***--MUST DELIVER ONLY ON APPT TIME-- CANNOT DELIVER EARLY OR LATE \r\n","isDropTrailer":false,"shipperName":"IDX NORTH CAROLINA","processedStopDate":""},{"loadInfoIds":[5431607],"stopDateTime":"0001-01-01T00:00:00","isPickup":false,"location":"Abilene, TX","stopDate":"Monday, 01/17/22","stopHours":"09:00 - 10:00","appointmentTime":"09:00","appointmentType":"APPT","instructions":"***LIFTGATE, PALLET JACK, DRIVER ASSIST***--MUST DELIVER ONLY ON APPT TIME-- CANNOT DELIVER EARLY OR LATE \r\n","isDropTrailer":false,"shipperName":"hibbett sports","processedStopDate":""}],"drayStopInfos":null,"rateInfo":null,"isDray":false,"equipmentGroupId":8,"totalCarrierRate":2600.00,"payUpTo":2700.00,"firstLoadInfoId":5431607,"loadStatus":"ACTIVE","hasBlockingAlert":false,"customerLoadBlocked":false,"canSeeRate":false,"canBookNow":false,"canBidNow":true,"buttonData":{"useBidDialog":false,"lastBidAmount":null,"remainingBids":1,"carrierPhoneNumber":null,"carrierPhoneExtension":null},"stopData":[{"addr":"IDX NORTH CAROLINA","city":"Washington","state":"NC","zip":"27889"},{"addr":"hibbett sports","city":"Abilene","state":"TX","zip":"79606"}]}
 
-- no hard code jpeg or have it in multiple places
-- todo: switch house to selfhosted for rate limit
-  tags: now
-- mark consumed;; save scroll id for matrix
-- TEST its falling apart
-- test each !command callbacks to matrix
-- change matrix so I test my custom logic even if I dont fetch remote
-- warn/err/etc. on clobbering ids.matrix since clients can mess with one another
-- modify old items once no longer available; drop stale jobs good candidate but requires new matrix interaction
-- more than NTG
-- todo: filter out jobs like CA
-  subtasks:
-  - banlist criteria like vendors, brokers, metadata
-- set up copy for caleb, broc
-done:
+- tokens in db, not in config
+- cache on disk jobinfo
 - link to view more
 - map with to-from line
 - TO CONFLUENT.RS OR W/E
@@ -32,7 +45,7 @@ done:
 - pause until => !busy until
 - todo: maps of to+from to get location within state via api
   details: |
-    curl  'https://maps.googleapis.com/maps/api/staticmap?center=Advance,NC&markers=label=A|Advance,NC&zoom=5&size=250x250&scale=2&format=jpeg&maptype=roadmap&key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' -H 'Accept: image/avif,image/webp,*/*' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'DNT: 1' -H 'Alt-Used: maps.googleapis.com' -H 'Connection: keep-alive' -H 'Sec-Fetch-Dest: image' -H 'Sec-Fetch-Mode: no-cors' -H 'Sec-Fetch-Site: cross-site' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'TE: trailers'  > whatever.jpg; open whatever.jpg~/Go/src/local/truckstop
+    curl  'https://maps.googleapis.com/maps/api/staticmap?center=Advance,NC&markers=label=A|Advance,NC&zoom=5&size=250x250&scale=2&format=jpeg&maptype=roadmap&key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' -H 'Accept: image/avif,image/webp,*/*' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'DNT: 1' -H 'Alt-Used: maps.googleapis.com' -H 'Connection: keep-alive' -H 'Sec-Fetch-Dest: image' -H 'Sec-Fetch-Mode: no-cors' -H 'Sec-Fetch-Site: cross-site' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'TE: trailers'  > whatever.jpg; open whatever.jpg~/Go/src/gogs.inhome.blapointe.com/local/truckstop
   subtasks:
   - DONE; yandex; key 9baa3e42-c6e5-4eb5-a891-05ffcede6a25 yandex maps
   - google; key AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg
@@ -62,4 +75,4 @@ done:
 - parse jobs
 - gather jobs
 - read states
-- read email to state file from local/sandbox/contact/contact
+- read email to state file from gogs.inhome.blapointe.com/local/sandbox/contact/contact

zip/zip.go

@@ -0,0 +1,112 @@
+package zip
+
+import (
+	_ "embed"
+	"gogs.inhome.blapointe.com/local/truckstop/logtr"
+	"math"
+	"strconv"
+	"strings"
+)
+
+//go:embed testdata/simplemaps_uszips_basicv1.79/uszips.csv
+var csv string
+
+type Zip struct {
+	Lat   float64
+	Lng   float64
+	City  string
+	State string
+}
+
+var zips map[string]Zip
+
+func init() {
+	if len(zips) > 0 {
+		return
+	}
+	zips = map[string]Zip{}
+	trim := func(s string) string {
+		return strings.Trim(s, `"`)
+	}
+	for _, line := range strings.Split(csv, "\n")[1:] {
+		strs := strings.Split(line, ",")
+		if len(strs) < 5 {
+			continue
+		}
+		zip := trim(strs[0])
+		lat, _ := strconv.ParseFloat(trim(strs[1]), 32)
+		lng, _ := strconv.ParseFloat(trim(strs[2]), 32)
+		city := trim(alphafy(strs[3]))
+		state := strings.ToUpper(trim(strs[4]))
+		zips[zip] = Zip{
+			Lat:   lat,
+			Lng:   lng,
+			City:  city,
+			State: state,
+		}
+	}
+	logtr.Infof("loaded %d zip codes", len(zips))
+}
+
+func alphafy(s string) string {
+	bs := make([]byte, 0, len(s)+1)
+	for i := range s {
+		if (s[i] < 'a' || s[i] > 'z') && (s[i] < 'A' || s[i] > 'Z') {
+		} else {
+			bs = append(bs, s[i])
+		}
+	}
+	return strings.ToLower(string(bs))
+}
+
+func (zip Zip) MilesTo(other Zip) int {
+	// c**2 = a**2 + b**2
+	// 69.2 mi per lat
+	// 60.0 mi per lng
+	return int(math.Sqrt(
+		math.Pow(69.2*(math.Abs(zip.Lat)-math.Abs(other.Lat)), 2) +
+			math.Pow(60.0*(math.Abs(zip.Lng)-math.Abs(other.Lng)), 2),
+	))
+}
+
+func Get(zip string) Zip {
+	if z, ok := zips[zip]; ok {
+		return z
+	}
+	zipAsI, err := strconv.Atoi(strings.Split(zip, "-")[0])
+	if err != nil {
+		return Zip{}
+	}
+	for i := 0; i < 5; i++ {
+		j := i - 2
+		if z2, ok := zips[strconv.Itoa(zipAsI+j)]; ok {
+			return z2
+		}
+	}
+	return Zip{}
+}
+
+func GetStatesWithin(zip Zip, miles int) []string {
+	states := map[string]struct{}{}
+	for _, other := range zips {
+		if zip.MilesTo(other) <= miles {
+			states[other.State] = struct{}{}
+		}
+	}
+	result := []string{}
+	for state := range states {
+		result = append(result, state)
+	}
+	return result
+}
+
+func FromCityState(city, state string) Zip {
+	city = alphafy(city)
+	state = strings.ToUpper(state)
+	for _, z := range zips {
+		if z.City == city && z.State == state {
+			return z
+		}
+	}
+	return Zip{}
+}

zip/zip_test.go

@@ -0,0 +1,28 @@
+package zip
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+)
+
+func TestZip(t *testing.T) {
+	_ = true
+	zip27006 := Get("27006")
+	zip84059 := Get("84059")
+	t.Logf("27006 = %+v", zip27006)
+	t.Logf("84059 = %+v", zip84059)
+	if zip27006.MilesTo(zip84059) < 1800 {
+		t.Error("failed to compute miles from advance to ut")
+	}
+	sorted := func(s []string) []string {
+		sort.Strings(s)
+		return s
+	}
+	if got := GetStatesWithin(Get("27006"), 100); fmt.Sprint(sorted(got)) != fmt.Sprint(sorted([]string{"NC", "SC", "TN", "VA"})) {
+		t.Error(got)
+	}
+	if got := FromCityState("ADVANCE", "nc"); got != zip27006 {
+		t.Error(got)
+	}
+}