return errnoauth on 401-403 for fe

broker/broker.go

@@ -2,9 +2,13 @@ package broker
 
 import (
 	"context"
+	"encoding/json"
+	"local/storage"
 	"local/truckstop/config"
+	"local/truckstop/logtr"
 	"net/http"
 	"strings"
+	"time"
 
 	"golang.org/x/time/rate"
 )
@@ -20,9 +24,86 @@ type Broker interface {
 }
 
 func do(r *http.Request) (*http.Response, error) {
+	return _do(config.Get().DB(), r)
+}
+
+func _do(db storage.DB, r *http.Request) (*http.Response, error) {
 	limiter.Wait(context.Background())
 	if strings.Contains(strings.ToLower(r.URL.Path), "login") {
 		authlimiter.Wait(context.Background())
 	}
-	return http.DefaultClient.Do(r)
+	client := &http.Client{
+		Timeout: time.Hour,
+	}
+
+	cookieJarKey := "cookies_" + r.URL.Host
+	cookies, err := getCookies(db, cookieJarKey)
+	if err != nil {
+		logtr.Errorf("failed to get cookies: %v", err)
+	} else {
+		logtr.Verbosef("got cookies for %s: %+v", cookieJarKey, cookies)
+		for _, cookie := range cookies {
+			r.Header.Add("Cookie", cookie)
+		}
+	}
+	logtr.Verbosef("_do: %+v", r)
+	resp, err := client.Do(r)
+	if err != nil {
+		return nil, err
+	}
+	if err := setCookies(db, cookieJarKey, resp); err != nil {
+		logtr.Errorf("failed to set cookies: %v", err)
+	}
+	return resp, err
+}
+
+func getCookies(db storage.DB, host string) ([]string, error) {
+	b, err := db.Get(host)
+	if err != nil {
+		return nil, err
+	}
+	var result []string
+	err = json.Unmarshal(b, &result)
+	return result, err
+}
+
+func setCookies(db storage.DB, host string, resp *http.Response) error {
+	nameValues := [][2]string{}
+	old, _ := getCookies(db, host)
+	for _, value := range old {
+		if len(value) == 0 {
+			continue
+		}
+		name := strings.Split(value, "=")[0]
+		nameValues = append(nameValues, [2]string{name, value})
+	}
+	for _, value := range resp.Header["Set-Cookie"] {
+		if len(value) == 0 {
+			continue
+		}
+		name := strings.Split(value, "=")[0]
+		found := false
+		for i := range nameValues {
+			if nameValues[i][0] == name {
+				found = true
+				nameValues[i][1] = value
+			}
+		}
+		if !found {
+			nameValues = append(nameValues, [2]string{name, value})
+		}
+	}
+	result := []string{}
+	for i := range nameValues {
+		if len(nameValues[i][1]) == 0 {
+			continue
+		}
+		result = append(result, nameValues[i][1])
+	}
+	logtr.Verbosef("setting cookies for %s: %+v", host, result)
+	b, err := json.Marshal(result)
+	if err != nil {
+		return err
+	}
+	return db.Set(host, b)
 }

broker/broker_test.go

@@ -0,0 +1,73 @@
+package broker
+
+import (
+	"fmt"
+	"local/storage"
+	"local/truckstop/logtr"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+func TestBrokerInterface(t *testing.T) {
+	logtr.SetLevel(logtr.SOS)
+	var b Broker
+	b = FastExact{}
+	b = NTGVision{}
+	_ = b
+}
+
+func TestDoCookies(t *testing.T) {
+	logtr.SetLevel(logtr.SOS)
+	limiter = rate.NewLimiter(rate.Limit(20.0), 1)
+	calls := 0
+	db := storage.NewMap()
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls += 1
+		if calls != 1 {
+			if !strings.Contains(fmt.Sprint(r.Header["Cookie"]), "name=value"+strconv.Itoa(calls-1)) {
+				w.WriteHeader(http.StatusBadRequest)
+				t.Error("cookie not set as latest")
+			}
+			if !strings.Contains(fmt.Sprint(r.Header["Cookie"]), "Expires") {
+				w.WriteHeader(http.StatusBadRequest)
+				t.Error("cookie not expiration: ", r.Header["Cookie"])
+			}
+		}
+		http.SetCookie(w, &http.Cookie{
+			Name:    "name",
+			Value:   "value" + strconv.Itoa(calls),
+			Expires: time.Now().Add(time.Hour),
+		})
+	}))
+	defer s.Close()
+	req, _ := http.NewRequest(http.MethodGet, s.URL, nil)
+
+	if cookies, err := getCookies(db, "cookies_"+req.URL.Host); err == nil {
+		t.Fatal(err)
+	} else if len(cookies) != 0 {
+		t.Fatal(cookies)
+	}
+
+	for i := 0; i < 3; i++ {
+		resp, err := _do(db, req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if resp.StatusCode != http.StatusOK {
+			t.Fatal(resp.StatusCode)
+		}
+		resp.Body.Close()
+
+		if cookies, err := getCookies(db, "cookies_"+req.URL.Host); err != nil {
+			t.Fatal(err)
+		} else if len(cookies) == 0 {
+			t.Fatal(cookies)
+		}
+	}
+}

broker/fastexact.go

@@ -0,0 +1,257 @@
+package broker
+
+import (
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"local/storage"
+	"local/truckstop/config"
+	"local/truckstop/logtr"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type FastExact struct {
+	doer interface {
+		doRequest(*http.Request) (*http.Response, error)
+	}
+}
+
+func NewFastExact() FastExact {
+	fe := FastExact{}
+	fe.doer = fe
+	if config.Get().Brokers.FastExact.Mock {
+		fe = fe.WithMock()
+	}
+	return fe
+}
+
+func (fe FastExact) WithMock() FastExact {
+	fe.doer = mockFastExactDoer{}
+	return fe
+}
+
+func (fe FastExact) Search(states []config.State) ([]Job, error) {
+	jobs, err := fe.search(states)
+	if err == ErrNoAuth {
+		if err := fe.login(); err != nil {
+			return nil, err
+		}
+		jobs, err = fe.search(states)
+	}
+	return jobs, err
+}
+
+func (fe FastExact) login() error {
+	conf := config.Get()
+	return fe._login(conf.Brokers.FastExact.Username, conf.Brokers.FastExact.Password, conf.DB())
+}
+
+func (fe FastExact) _login(username, password string, db storage.DB) error {
+	req, err := http.NewRequest(
+		http.MethodPost,
+		`https://www.fastexact.com/secure/index.php?page=userLogin`,
+		strings.NewReader(fmt.Sprintf(
+			`user_name=%s&user_password=%s&buttonSubmit=Login`,
+			username,
+			password,
+		)),
+	)
+	if err != nil {
+		return err
+	}
+	fe.setHeaders(req)
+
+	db.Set("cookies_"+req.URL.Host, nil)
+	resp, err := fe.doer.doRequest(req)
+	if err != nil {
+		return err
+	}
+	b, _ := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("bad status logging into fast exact: %d: %s", resp.StatusCode, b)
+	}
+	return nil
+}
+
+func (fe FastExact) setHeaders(req *http.Request) {
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.5")
+	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+}
+
+func (fe FastExact) search(states []config.State) ([]Job, error) {
+	var result []Job
+	for _, state := range states {
+		jobs, err := fe.searchOne(state)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, jobs...)
+	}
+	return result, nil
+}
+
+func (fe FastExact) searchOne(state config.State) ([]Job, error) {
+	req, err := fe.newRequest(state)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := fe.doer.doRequest(req)
+	logtr.Verbosef("req: %+v => resp: %+v", req, resp)
+	if err != nil {
+		return nil, err
+	}
+	return fe.parse(resp)
+}
+
+func (fe FastExact) newRequest(state config.State) (*http.Request, error) {
+	zip, ok := config.States[state]
+	if !ok {
+		return nil, fmt.Errorf("no configured zip for %s", state)
+	}
+	req, err := http.NewRequest(
+		http.MethodGet,
+		"https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode="+zip+"&records_per_page=50&distance=300&st_loc_zip=8",
+		nil,
+	)
+	if err != nil {
+		return nil, err
+	}
+	fe.setHeaders(req)
+	return req, nil
+}
+
+func (fe FastExact) doRequest(req *http.Request) (*http.Response, error) {
+	resp, err := do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		b, _ := ioutil.ReadAll(resp.Body)
+		logtr.Errorf("fastExact bad status: %d: %s", resp.StatusCode, b)
+		if resp.StatusCode > 400 && resp.StatusCode < 404 {
+			return nil, ErrNoAuth
+		}
+		return nil, fmt.Errorf("bad status from FastExact: %d: %s", resp.StatusCode, b)
+	}
+	return resp, nil
+}
+
+func (fe FastExact) parse(resp *http.Response) ([]Job, error) {
+	b, _ := ioutil.ReadAll(resp.Body)
+	if !bytes.HasPrefix(bytes.TrimSpace(b), []byte("<")) {
+		gzip, err := gzip.NewReader(bytes.NewReader(b))
+		if err != nil {
+			return nil, err
+		}
+		b, _ = ioutil.ReadAll(gzip)
+	}
+	logtr.Verbosef("fe.parse %s", b)
+	resp.Body = io.NopCloser(bytes.NewReader(b))
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	result := make([]Job, 0)
+	doc.Find("#list table tr").Each(func(i int, s *goquery.Selection) {
+		columns := []string{}
+		s.Find("td").Each(func(i int, s *goquery.Selection) {
+			if s.Nodes[0].LastChild != nil && len(s.Nodes[0].LastChild.Attr) > 0 {
+				attrs := s.Nodes[0].LastChild.Attr
+				columns = append(columns, attrs[len(attrs)-1].Val)
+			} else {
+				columns = append(columns, s.Text())
+			}
+		})
+		if len(columns) < 9 {
+			return
+		}
+		job := Job{
+			ID:  columns[0],
+			URI: columns[8],
+		}
+		job.Pickup.Date, _ = time.ParseInLocation("02-Jan-2006  15:04:05", columns[7], time.Local)
+		job.Pickup.City = strings.Title(strings.ToLower(strings.Split(columns[1], ",")[0]))
+		if strings.Contains(columns[1], ",") {
+			job.Pickup.State = strings.Title(strings.Split(strings.Split(columns[1], ",")[1], " ")[1])
+		}
+
+		job.Dropoff.Date = job.Pickup.Date
+		job.Dropoff.City = strings.Title(strings.ToLower(strings.Split(columns[2], ",")[0]))
+		if strings.Contains(columns[2], ",") {
+			job.Dropoff.State = strings.Title(strings.Split(strings.Split(columns[2], ",")[1], " ")[1])
+		}
+
+		job.Miles, _ = strconv.Atoi(columns[3])
+		if strings.Contains(columns[4], "/") {
+			weight, _ := strconv.ParseFloat(strings.TrimSpace(strings.Split(columns[4], "/")[1]), 32)
+			job.Weight = int(weight)
+		}
+
+		job.Meta = fmt.Sprintf(`dimensions:%s`, strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(columns[5], " ", ""), "\n", "")))
+
+		logtr.Verbosef("fe.parse %+v => %+v", columns, job)
+
+		result = append(result, job)
+	})
+	return result, nil
+}
+
+type mockFastExactDoer struct{}
+
+func (mock mockFastExactDoer) doRequest(req *http.Request) (*http.Response, error) {
+	if req.URL.Path != "/secure/index.php" {
+		return nil, errors.New("bad path")
+	}
+	switch req.URL.Query().Get("page") {
+	case "userLogin":
+		if b, _ := ioutil.ReadAll(req.Body); !bytes.Equal(b, []byte(`user_name=u&user_password=p&buttonSubmit=Login`)) {
+			return nil, errors.New("bad req body")
+		}
+		return &http.Response{
+			Status:     http.StatusText(http.StatusOK),
+			StatusCode: http.StatusOK,
+			Header:     http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromLogin; path=/"}},
+			Body:       io.NopCloser(bytes.NewReader([]byte{})),
+		}, nil
+	case "ajaxListJobs":
+		if req.URL.Query().Get("action") != "ajax" {
+			return nil, errors.New("bad query: action should be ajax")
+		}
+		if req.URL.Query().Get("records_per_page") != "50" {
+			return nil, errors.New("bad query: records_per_page should be 50")
+		}
+		if req.URL.Query().Get("distance") != "300" {
+			return nil, errors.New("bad query: distance should be 300")
+		}
+		if req.URL.Query().Get("zipcode") == "" {
+			return nil, errors.New("bad query: zip code empty")
+		}
+		b, err := ioutil.ReadFile("./testdata/fastexact_search.xml")
+		if err != nil {
+			b, err = ioutil.ReadFile("./broker/testdata/fastexact_search.xml")
+		}
+		if err != nil {
+			return nil, err
+		}
+		return &http.Response{
+			Status:     http.StatusText(http.StatusOK),
+			StatusCode: http.StatusOK,
+			Header:     http.Header{"Set-Cookie": []string{"PHPSESSID=SessionFromSearch; path=/"}},
+			Body:       io.NopCloser(bytes.NewReader(b)),
+		}, nil
+	}
+	return nil, errors.New("bad query")
+}

broker/fastexact_test.go

@@ -0,0 +1,97 @@
+package broker
+
+import (
+	"local/storage"
+	"local/truckstop/config"
+	"local/truckstop/logtr"
+	"os"
+	"path"
+	"testing"
+
+	"golang.org/x/time/rate"
+)
+
+func TestFastExactReal(t *testing.T) {
+	if os.Getenv("INTEGRATION_FAST_EXACT") == "" {
+		t.Skip("$INTEGRATION_FAST_EXACT not set")
+	}
+	os.Setenv("CONFIG", "../config.json")
+	if err := config.Refresh(nil); err != nil {
+		t.Fatal(err)
+	}
+	conf := config.Get()
+	conf.Storage = []string{"files", path.Join(t.TempDir(), "storage")}
+	os.Setenv("CONFIG", path.Join(t.TempDir(), "config.json"))
+	config.Set(*conf)
+	if err := config.Refresh(nil); err != nil {
+		t.Fatal(err)
+	}
+	states := []config.State{
+		config.State("NC"),
+	}
+
+	fe := NewFastExact()
+	if err := fe.login(); err != nil {
+		t.Fatal(err)
+	}
+	jobs, err := fe.search(states)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("%d", len(jobs))
+	for i := range jobs {
+		t.Logf("[%d] %+v", i, jobs[i])
+	}
+}
+
+func TestFastExactLogin(t *testing.T) {
+	logtr.SetLevel(logtr.SOS)
+	limiter = rate.NewLimiter(rate.Limit(60.0), 1)
+	fe := NewFastExact().WithMock()
+	_ = fe
+	db := storage.NewMap()
+	if err := fe._login("u", "p", db); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestFastExactSearch(t *testing.T) {
+	logtr.SetLevel(logtr.SOS)
+	limiter = rate.NewLimiter(rate.Limit(60.0), 1)
+	fe := NewFastExact().WithMock()
+	_ = fe
+	db := storage.NewMap()
+	_ = db
+	if jobs, err := fe.search([]config.State{config.State("NC"), config.State("SC")}); err != nil {
+		t.Fatal(err)
+	} else if len(jobs) != 20 {
+		t.Fatal(jobs)
+	} else {
+		for _, job := range jobs {
+			if job.ID == "" {
+				t.Error(job)
+			} else if job.Miles == 0 {
+				t.Error(job)
+			} else if job.URI == "" {
+				t.Error(job)
+			} else if job.Meta == "" {
+				t.Error(job)
+			} else if job.Weight == 0 {
+				t.Error(job)
+			} else if job.Pickup.Date.IsZero() {
+				t.Error(job)
+			} else if job.Pickup.Date != job.Dropoff.Date {
+				t.Error(job)
+			} else if job.Dropoff.State == "" {
+				t.Error(job)
+			} else if job.Dropoff.City == "" {
+				t.Error(job)
+			} else if job.Pickup.State == "" {
+				t.Error(job)
+			} else if job.Pickup.City == "" {
+				t.Error(job)
+			}
+		}
+	}
+}

broker/ntgvision.go

@@ -16,7 +16,7 @@ import (
 type NTGVision struct {
 	searcher interface {
 		search(states []config.State) (io.ReadCloser, error)
-		searchJob(id int64) (io.ReadCloser, error)
+		searchJobReadCloser(id int64) (io.ReadCloser, error)
 	}
 }
 
@@ -93,10 +93,10 @@ func (ntgJob *ntgVisionJob) JobInfo() (ntgVisionJobInfo, error) {
 	key := fmt.Sprintf("ntg_job_info_%v", ntgJob.ID)
 	if b, err := db.Get(key); err != nil {
 	} else if err := json.Unmarshal(b, &ntgJob.jobinfo); err == nil {
-		return ntgJob.jobinfo, fmt.Errorf("failed to parse ntg job info from db: %w: %s", err, b)
+		return ntgJob.jobinfo, nil
 	}
 	ntg := NewNTGVision()
-	ji, err := ntg.SearchJob(ntgJob.ID)
+	ji, err := ntg.searchJob(ntgJob.ID)
 	if err == nil {
 		ntgJob.jobinfo = ji
 		b, err := json.Marshal(ntgJob.jobinfo)
@@ -107,7 +107,7 @@ func (ntgJob *ntgVisionJob) JobInfo() (ntgVisionJobInfo, error) {
 	return ji, err
 }
 
-func (ntg NTGVision) searchJob(id int64) (io.ReadCloser, error) {
+func (ntg NTGVision) searchJobReadCloser(id int64) (io.ReadCloser, error) {
 	time.Sleep(config.Get().Interval.JobInfo.Get())
 	request, err := http.NewRequest(http.MethodGet, fmt.Sprintf(config.Get().Brokers.NTG.LoadPageAPIURIFormat, id), nil)
 	if err != nil {
@@ -169,8 +169,8 @@ func (ntg NTGVision) WithMock() NTGVision {
 	return ntg
 }
 
-func (ntg NTGVision) SearchJob(id int64) (ntgVisionJobInfo, error) {
+func (ntg NTGVision) searchJob(id int64) (ntgVisionJobInfo, error) {
-	rc, err := ntg.searcher.searchJob(id)
+	rc, err := ntg.searcher.searchJobReadCloser(id)
 	if err != nil {
 		return ntgVisionJobInfo{}, err
 	}
@@ -228,12 +228,14 @@ func setNTGToken(token string) {
 
 func (ntg NTGVision) search(states []config.State) (io.ReadCloser, error) {
 	if getNTGToken() == "" {
+		logtr.Debugf("NTG token is empty, refreshing ntg auth")
 		if err := ntg.refreshAuth(); err != nil {
 			return nil, err
 		}
 	}
 	rc, err := ntg._search(states)
 	if err == ErrNoAuth {
+		logtr.Debugf("err no auth on search, refreshing ntg auth")
 		if err := ntg.refreshAuth(); err != nil {
 			return nil, err
 		}
@@ -295,7 +297,11 @@ func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
 	if resp.StatusCode != http.StatusOK {
 		b, _ := ioutil.ReadAll(resp.Body)
 		resp.Body.Close()
-		if resp.StatusCode > 400 && resp.StatusCode < 500 && resp.StatusCode != 404 && resp.StatusCode != 410 {
+		request2, _ := ntg.newRequest(states)
+		requestb, _ := ioutil.ReadAll(request2.Body)
+		logtr.Debugf("ntg auth bad status: url=%s, status=%v, body=%s, headers=%+v, request=%+v, requestb=%s", request.URL.String(), resp.StatusCode, b, resp.Header, request2, requestb)
+		if resp.StatusCode > 400 && resp.StatusCode < 404 {
+			logtr.Debugf("ntg auth bad status: err no auth")
 			return nil, ErrNoAuth
 		}
 		return nil, fmt.Errorf("bad status searching ntg: %d: %s", resp.StatusCode, b)
@@ -305,7 +311,7 @@ func (ntg NTGVision) _search(states []config.State) (io.ReadCloser, error) {
 
 func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) {
 	body, err := json.Marshal(map[string]interface{}{
-		"OriginFromDate":        time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
+		"OriginFromDate":        time.Now().Add(time.Hour * -24).UTC().Format("2006-01-02T15:04:05.000Z"),
 		"OriginToDate":          time.Now().UTC().Add(time.Hour * 24 * 30).Format("2006-01-02T15:04:05.000Z"),
 		"DestinationFromDate":   nil,
 		"DestinationToDate":     nil,
@@ -329,6 +335,7 @@ func (ntg NTGVision) newRequest(states []config.State) (*http.Request, error) {
 	}
 	setNTGHeaders(request)
 	request.Header.Set("Authorization", "Bearer "+getNTGToken())
+	request.Header.Set("Cookie", "NTGAuthToken="+getNTGToken())
 
 	return request, nil
 }

broker/ntgvision_mock.go

@@ -21,7 +21,7 @@ func (ntgm NTGVisionMock) search(states []config.State) (io.ReadCloser, error) {
 	return io.NopCloser(bytes.NewReader(b)), err
 }
 
-func (ntgm NTGVisionMock) searchJob(id int64) (io.ReadCloser, error) {
+func (ntgm NTGVisionMock) searchJobReadCloser(id int64) (io.ReadCloser, error) {
 	path := path.Join(os.Getenv("GOPATH"), "src", "local", "truckstop", "broker", "testdata", "ntgvision_jobinfo_response.json")
 	b, err := ioutil.ReadFile(path)
 	return io.NopCloser(bytes.NewReader(b)), err

broker/testdata/fastexact_login.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+
+curl -i -Ss \
+   'https://www.fastexact.com/secure/index.php?page=userLogin' \
+   -X POST \
+   -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' \
+   -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \
+   -H 'Accept-Language: en-US,en;q=0.5' \
+   -H 'Accept-Encoding: gzip, deflate, br' \
+   -H 'Content-Type: application/x-www-form-urlencoded' \
+   --data-raw 'user_name=birdman&user_password=166647&buttonSubmit=Login' \
+

broker/testdata/fastexact_search.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+curl -sS \
+   'https://www.fastexact.com/secure/index.php?page=ajaxListJobs&action=ajax&zipcode=27006&records_per_page=10&distance=300&st_loc_zip=8' \
+   -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0' \
+   -H 'Accept: */*' \
+   -H 'Accept-Language: en-US,en;q=0.5' \
+   -H 'Accept-Encoding: gzip, deflate, br' \
+   -H 'Content-Type: application/x-www-form-urlencoded' \
+   -H 'Cookie: PHPSESSID=vq4ss3lbfg0viluaau7e0d32q6'  \
+| gzip -d

broker/testdata/fastexact_search.xml

@@ -0,0 +1,218 @@
+<form name="frm_search" action="index.php?page=manageJobs" method="post">
+<a href="https://play.google.com/store/apps/details?id=app.FastExact" target="_blank" title="Android app, Available in the App Store, Download and get all more..." class="android_app"></a>
+<a href="http://itunes.apple.com/us/app/fastexact/id497933887?mt=8" target="_blank" title="iPhone app, Available in the App Store, Download and get all more..." class="iphone_app"></a>
+  <div class="search_box w67p">
+    <div class="header"><strong>Search</strong> <span class="usZipLink"><a href="http://zip4.usps.com/zip4/citytown.jsp" title="US Zip Code Finder" target="_blank">US Zip Code Finder &raquo;</a></span> </div>
+    <div class="content">
+      <table border="0" cellspacing="0" cellpadding="0">
+        <tr>
+          <td valign="top" width="342"><strong>Enter your Zip Code to show radius
+            <!--of 100 miles-->
+            from your location</strong></td>
+          <td valign="top" width="68"><input type="text" name="zipcode" id="zipcode" value="27006" onKeyPress="return disableEnterKey(event)" onkeydown="if(event.keyCode == 13){radiusSearch('10')}" /></td>
+          <td valign="top" width="59"><select name="distance" id="distance" class="select" onkeydown="if(event.keyCode == 13){radiusSearch('10')}">
+              <option value="100" >100</option>
+              <option value="200" >200</option>
+              <option value="300" selected=selected>300</option>
+            </select></td>
+          <td valign="top" width="128"><!--<input type="checkbox" name="st_loc_zip" id="st_loc_zip" value="1" class="ownBox" checked=checked onKeyPress="return disableEnterKey(event)" onkeydown="if(event.keyCode == 13){radiusSearch('10')}"/>--></td>
+          <td valign="top" class="padr12" width="337">Make this your current zip code<br />
+            for the next
+            <select name="st_loc_zip" id="st_loc_zip" class="dropdown">
+                            <option value="1">1</option>
+                            <option value="2">2</option>
+                            <option value="3">3</option>
+                            <option value="4">4</option>
+                            <option value="5">5</option>
+                            <option value="6">6</option>
+                            <option value="7">7</option>
+                            <option value="8" selected="selected">8</option>
+                            <option value="9">9</option>
+                            <option value="10">10</option>
+                            <option value="11">11</option>
+                            <option value="12">12</option>
+                            <option value="13">13</option>
+                            <option value="14">14</option>
+                            <option value="15">15</option>
+                            <option value="16">16</option>
+                            <option value="17">17</option>
+                            <option value="18">18</option>
+                            <option value="19">19</option>
+                            <option value="20">20</option>
+                            <option value="21">21</option>
+                            <option value="22">22</option>
+                            <option value="23">23</option>
+                            <option value="24">24</option>
+                          </select>
+            hours.<a href="javascript:radiusSearch('10','save');" ><img src="images/buttons/save.png"  /></a></td>
+          <td valign="top"><a href="javascript:radiusSearch('10','search');"><img src="images/buttons/search.gif" alt="Search" align="middle" title="Search" /></a></td>
+        </tr>
+      </table>
+    </div>
+  </div>
+  <div class="br"></div>
+</form>
+<br clear="all" />
+<div id="list" align="center">
+  <table width="100%" border="0" cellspacing="0" cellpadding="0" summary="Order List">
+    <tr>
+      <th width="14%" height="38" align="center">ID&nbsp;&nbsp;<span id="sort8"><a href="javascript:contentSort('ordernum=8&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',8)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
+      <th width="14%" height="38" align="center">Origin&nbsp;&nbsp;<span id="sort1"><a href="javascript:contentSort('ordernum=1&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',1)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
+      <th  width="14%" height="38" align="center">Destination&nbsp;&nbsp;<span id="sort2"><a href="javascript:contentSort('ordernum=2&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',2)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
+      <th width="14%" height="38" align="center">Mile&nbsp;&nbsp;<span id="sort3"><a href="javascript:contentSort('ordernum=3&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',3)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
+      <th height="38" align="center">Pieces / Weight&nbsp;&nbsp;<span id="sort4"><a href="javascript:contentSort('ordernum=4&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',4)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0"/></a></span></th>
+      <th width="14%" height="38" align="center">Dimension <span id="sort5"><a href="javascript:contentSort('ordernum=5&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',5)"><img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0"/></a></span></th>
+	  <th width="14%" height="38" align="center">Vehicle Size <span id="sort5"><a href="javascript:contentSort('ordernum=5&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',5)"><img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0"/></a></span></th>
+      <th width="20%" height="38" align="center">Pickup (EST)&nbsp;&nbsp;<span id="sort6"><a href="javascript:contentSort('ordernum=6&dir=asc&pageNum=1&zipcode=27006&distance=300&st_loc_zip=8&records_per_page=10',6)"> <img src="https://www.fastexact.com/secure/images/sortarrow_down.gif" border="0" /></a></span></th>
+      <th width="6%" height="38" align="center">View</th>
+    </tr>
+        <tr >
+      <td align="center" height="38">Z-5344222</td>
+      <td align="center" height="38">Circleville, OH 43113</td>
+      <td align="center">Willard, OH 44890</td>
+      <td align="center">128</td>
+      <td align="center">4 / 4800</td>
+      <td align="center">96x48x88in.</td>
+      <td align="center">SMALL STRAIGHT</td>
+      <td align="center">31-Jan-2022  17:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344222"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr class="alternate">
+      <td align="center" height="38">Z-5344003</td>
+      <td align="center" height="38">WASHINGTON, DC</td>
+      <td align="center">WINSTON SALEM, NC</td>
+      <td align="center">332</td>
+      <td align="center">2 / 3000</td>
+      <td align="center">78x35x99 in.</td>
+      <td align="center">LARGE STRAIGHT</td>
+      <td align="center">31-Jan-2022  08:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344003"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr >
+      <td align="center" height="38">Z-5344092</td>
+      <td align="center" height="38">Winchester, VA 22603</td>
+      <td align="center">Milwaukee, WI 53221</td>
+      <td align="center">728</td>
+      <td align="center">4 / 2195</td>
+      <td align="center">NO DIMENSIONS SPECIFIED</td>
+      <td align="center">VAN</td>
+      <td align="center">30-Jan-2022  15:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344092"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr class="alternate">
+      <td align="center" height="38">Z-5344035</td>
+      <td align="center" height="38">ATLANTA, GA 30301</td>
+      <td align="center">BURNSVILLE, MN 55337</td>
+      <td align="center">1098</td>
+      <td align="center">10 / 6000</td>
+      <td align="center">48x40x40 in.</td>
+      <td align="center">LARGE STRAIGHT</td>
+      <td align="center">29-Jan-2022  08:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344035"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr >
+      <td align="center" height="38">Z-5344518</td>
+      <td align="center" height="38">Hamlet, NC 28345</td>
+      <td align="center">East Windsor, CT 06088</td>
+      <td align="center">720</td>
+      <td align="center">1 / 500</td>
+      <td align="center">54x36x24in.</td>
+      <td align="center">CARGO VAN</td>
+      <td align="center">28-Jan-2022  13:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344518"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr class="alternate">
+      <td align="center" height="38">X-5344304</td>
+      <td align="center" height="38">MOUNT PLEASANT, PA 15666</td>
+      <td align="center">MIAMI, FL 33178-1056</td>
+      <td align="center">1156</td>
+      <td align="center">0 / 5000.00</td>
+      <td align="center">48Inches                                                             x                                                             48Inches                                                             x                                                      </td>
+      <td align="center">Straight Truck 20-24 ft</td>
+      <td align="center">28-Jan-2022  13:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344304"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr >
+      <td align="center" height="38">Z-5344061</td>
+      <td align="center" height="38">Chesapeake, VA 23321</td>
+      <td align="center">Fort Wayne, IN 46818</td>
+      <td align="center">727</td>
+      <td align="center">0 / 5000</td>
+      <td align="center">NO DIMENSIONS SPECIFIED</td>
+      <td align="center">LARGE STRAIGHT</td>
+      <td align="center">28-Jan-2022  13:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344061"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr class="alternate">
+      <td align="center" height="38">B-5344514</td>
+      <td align="center" height="38">MAXTON, NC 28364</td>
+      <td align="center">PEMBINA, ND 58271</td>
+      <td align="center">1637</td>
+      <td align="center">7 / 28000</td>
+      <td align="center">7@54x96x40</td>
+      <td align="center"></td>
+      <td align="center">28-Jan-2022  12:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344514"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr >
+      <td align="center" height="38">Z-5344114</td>
+      <td align="center" height="38">Durham, NC 27703</td>
+      <td align="center">Cumming, GA 30040</td>
+      <td align="center">368</td>
+      <td align="center">6 / 3200</td>
+      <td align="center">48x40x48in.</td>
+      <td align="center">LARGE STRAIGHT</td>
+      <td align="center">28-Jan-2022  12:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344114"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+        <tr class="alternate">
+      <td align="center" height="38">Z-5344231</td>
+      <td align="center" height="38">Durham, NC 27702</td>
+      <td align="center">Cumming, GA 30040</td>
+      <td align="center">366</td>
+      <td align="center">6 / 3200</td>
+      <td align="center">48x40x48in.</td>
+      <td align="center">LARGE STRAIGHT</td>
+      <td align="center">28-Jan-2022  12:00:00</td>
+      <td align="center"><!--thickbox-->
+        <a title="Do Bidding" href="https://www.fastexact.com/secure/index.php?page=doBidding&job_id=5344231"><img src="https://www.fastexact.com/site_manager/images/icons/view.jpg" border="0" title="View this Record" /></a></td>
+    </tr>
+          </table>
+</div>
+<br clear="all" />
+<div class="paging">
+  <table cellspacing="0" cellpadding="0" border="0" width="100%">
+    <tbody>
+      <tr>
+        <td>Details Found: 81<img height="1" width="14" alt="" src="images/tspacer.gif">Page 1 of 9</td>
+        <td align="center"><div id="Page">
+                        <img height="1" width="8" alt="" src="images/tspacer.gif">
+                        <a href="javascript:fnPaging('pageNum=2&records_per_page=10&zipcode=27006&distance=300&st_loc_zip=8')">Next</a> <strong>&raquo;</strong>
+                      </div></td>
+        <td align="right"><table cellspacing="0" cellpadding="0" border="0">
+            <tbody>
+              <tr>
+                <td valign="top" class="record">Records Per Page:&nbsp;</td>
+                <td valign="top"><div id="recNum">
+                    <select style="font-size: 11px;" class="dropdown" id="pagei" name="pagei" onchange="changeRecordCount('27006','300','8')">
+                      <option value="10" selected="selected" >10</option>
+                      <option value="20" >20</option>
+                      <option value="50" >50</option>
+                      <option value="100" >100</option>
+                    </select>
+                  </div></td>
+              </tr>
+            </tbody>
+          </table></td>
+      </tr>
+    </tbody>
+  </table>

config.json

@@ -71,13 +71,18 @@
    },
    "Once": true,
    "Brokers": {
+      "FastExact": {
+         "Mock": false,
+         "Username": "birdman",
+         "Password": "166647"
+      },
       "NTG": {
          "JobInfo": true,
          "Mock": true,
          "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",
          "LoadPageAPIURIFormat": "https://ntgvision.com/api/v1/load/LoadDetails?loadId==%d",
          "Username": "noeasyrunstrucking@gmail.com",
-         "Password": "thumper1234"
+         "Password": "thumper12345"
       }
    }
 }

config.main_test.json

@@ -0,0 +1,88 @@
+{
+   "Log": {
+      "Path": "/tmp/truckstop.log",
+      "Level": "sos",
+      "SOSMatrix": {
+         "ReceiveEnabled": true,
+         "Mock": true,
+         "Homeserver": "https://m.bltrucks.top",
+         "Username": "@bot.m.bltrucks.top",
+         "Token": "mvDWB96KXMF8XhOam8EC5XVdQvSEw0CDeClcSWocBcYkwZX3FPNWZ5uOnQk2EmT1cjpzfeuD7gDYPPjOuyZlI3bE9TE35UjNOlZgi0Tugm25s91iVsbIF6kMZsCIhVMSmEf6w3jxX6wQYOWvmDZ4mu6f5c8wr221EMDcOpEzQV09d1zuBSWgKLBgjqAkYHJZ5dTRIWpEDpPgujhOFZa2ld1HiAOxrJKlIrlfDBN0CUsTlGOGplujDAr4VtpFzNRS",
+         "Device": "TGNIOGKATZ",
+         "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
+      }
+   },
+   "Interval": {
+      "Input": "5s..10s",
+      "OK": "6h0m0s..6h0m0s",
+      "Error": "6h0m0s..6h0m0s",
+      "JobInfo": "10s..20s"
+   },
+   "Images": {
+      "ClientID": "d9ac7cabe813d10",
+      "ClientSecret": "9d0b3d82800b30ca88f595d3bcd6985f627d7d82",
+      "RefreshToken": "171417741bf762b99b0b9f9137491b7a69874a77",
+      "AccessToken": "e63db98f92d2db7ac7f56914a2030c889b378e9b",
+      "RefreshURI": "https://api.imgur.com/oauth2/token",
+      "RefreshFormat": "refresh_token=%s\u0026client_id=%s\u0026client_secret=%s\u0026grant_type=refresh_token",
+      "RefreshMethod": "POST",
+      "UploadURI": "https://api.imgur.com/3/image?name=something.jpeg",
+      "UploadMethod": "POST"
+   },
+   "Maps": {
+      "URIFormat": "https://maps.googleapis.com/maps/api/staticmap?center=%s\u0026markers=label=A|%s\u0026zoom=5\u0026size=250x250\u0026scale=1\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
+      "Pickup": false,
+      "Dropoff": false,
+      "Pathed": {
+         "Enabled": true,
+         "DirectionsURIFormat": "https://maps.googleapis.com/maps/api/directions/json?origin=%s\u0026destination=%s\u0026mode=driving\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg",
+         "PathedURIFormat": "https://maps.googleapis.com/maps/api/staticmap?size=250x250\u0026path=%s\u0026format=jpeg\u0026maptype=roadmap\u0026key=AIzaSyBkACm-LQkoSfsTO5_XAzBVZE9-JQzcNkg\u0026size=250x250\u0026markers=%s|%s",
+         "Zoom": {
+            "AcceptableLatLngDelta": 2,
+            "Override": 5
+         }
+      }
+   },
+   "Clients": {
+      "bel": {
+         "States": [
+            "NC"
+         ],
+         "IDs": {
+            "Matrix": "@bel:m.bltrucks.top"
+         },
+         "Available": 1512328400
+      }
+   },
+   "Storage": [
+      "files",
+      "/tmp/play"
+   ],
+   "Message": {
+      "Matrix": {
+         "ReceiveEnabled": true,
+         "Mock": true,
+         "Homeserver": "https://m.bltrucks.top",
+         "Username": "@bot.m.bltrucks.top",
+         "Token": "mvDWB96KXMF8XhOam8EC5XVdQvSEw0CDeClcSWocBcYkwZX3FPNWZ5uOnQk2EmT1cjpzfeuD7gDYPPjOuyZlI3bE9TE35UjNOlZgi0Tugm25s91iVsbIF6kMZsCIhVMSmEf6w3jxX6wQYOWvmDZ4mu6f5c8wr221EMDcOpEzQV09d1zuBSWgKLBgjqAkYHJZ5dTRIWpEDpPgujhOFZa2ld1HiAOxrJKlIrlfDBN0CUsTlGOGplujDAr4VtpFzNRS",
+         "Device": "TGNIOGKATZ",
+         "Room": "!OYZqtInrBCn1cyz90D:m.bltrucks.top"
+      }
+   },
+   "Once": true,
+   "Brokers": {
+      "FastExact": {
+         "Mock": true,
+         "Username": "u",
+         "Password": "p"
+      },
+      "NTG": {
+         "JobInfo": true,
+         "Mock": true,
+         "LoadPageURIFormat": "https://ntgvision.com/LoadDetails?loadId=%d",
+         "LoadPageAPIURIFormat": "https://ntgvision.com/api/v1/load/LoadDetails?loadId==%d",
+         "Username": "noeasyrunstrucking@gmail.com",
+         "Password": "thumper1234"
+      }
+   }
+}

config/config.go

@@ -72,6 +72,11 @@ type Config struct {
 			Username             string
 			Password             string
 		}
+		FastExact struct {
+			Mock     bool
+			Username string
+			Password string
+		}
 	}
 
 	lock sync.Mutex

config/state.go

@@ -7,56 +7,56 @@ import (
 
 type State string
 
-var States = map[string]State{
+var States = map[State]string{
-	"99654": "AL",
+	"AL": "99654",
-	"72401": "AR",
+	"AR": "72401",
-	"85364": "AZ",
+	"AZ": "85364",
-	"90011": "CA",
+	"CA": "90011",
-	"80013": "CO",
+	"CO": "80013",
-	"06902": "CT",
+	"CT": "06902",
-	"19720": "DE",
+	"DE": "19720",
-	"33024": "FL",
+	"FL": "33024",
-	"30043": "GA",
+	"GA": "30043",
-	"96706": "HI",
+	"HI": "96706",
-	"50613": "IA",
+	"IA": "50613",
-	"83646": "ID",
+	"ID": "83646",
-	"60629": "IL",
+	"IL": "60629",
-	"47906": "IN",
+	"IN": "47906",
-	"66062": "KS",
+	"KS": "66062",
-	"40475": "KY",
+	"KY": "40475",
-	"70726": "LA",
+	"LA": "70726",
-	"02301": "MA",
+	"MA": "02301",
-	"20906": "MD",
+	"MD": "20906",
-	"04401": "ME",
+	"ME": "04401",
-	"48197": "MI",
+	"MI": "48197",
-	"55106": "MN",
+	"MN": "55106",
-	"63376": "MO",
+	"MO": "63376",
-	"39503": "MS",
+	"MS": "39503",
-	"59901": "MT",
+	"MT": "59901",
-	"27006": "NC",
+	"NC": "27006",
-	"58103": "ND",
+	"ND": "58103",
-	"68516": "NE",
+	"NE": "68516",
-	"03103": "NH",
+	"NH": "03103",
-	"08701": "NJ",
+	"NJ": "08701",
-	"87121": "NM",
+	"NM": "87121",
-	"89108": "NV",
+	"NV": "89108",
-	"11368": "NY",
+	"NY": "11368",
-	"45011": "OH",
+	"OH": "45011",
-	"73099": "OK",
+	"OK": "73099",
-	"97006": "OR",
+	"OR": "97006",
-	"19120": "PA",
+	"PA": "19120",
-	"02860": "RI",
+	"RI": "02860",
-	"29483": "SC",
+	"SC": "29483",
-	"57106": "SD",
+	"SD": "57106",
-	"37013": "TN",
+	"TN": "37013",
-	"77449": "TX",
+	"TX": "77449",
-	"84058": "UT",
+	"UT": "84058",
-	"22193": "VA",
+	"VA": "22193",
-	"05401": "VT",
+	"VT": "05401",
-	"99301": "WA",
+	"WA": "99301",
-	"53215": "WI",
+	"WI": "53215",
-	"26554": "WV",
+	"WV": "26554",
-	"82001": "WY",
+	"WY": "82001",
 }
 
 func (state *State) UnmarshalJSON(b []byte) error {
@@ -65,8 +65,8 @@ func (state *State) UnmarshalJSON(b []byte) error {
 		return err
 	}
 	for k := range States {
-		if string(States[k]) == s {
+		if string(k) == s {
-			*state = States[k]
+			*state = k
 			return nil
 		}
 	}

go.mod

@@ -16,8 +16,10 @@ require (
 
 require (
 	cloud.google.com/go v0.33.1 // indirect
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
 	github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/aws/aws-sdk-go v1.15.81 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect

go.sum

@@ -7,12 +7,16 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619 h1:6X8iB881g299aNEv6KXrcjL31iLOH7yA6NXoQX+MbDg=
 github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
 github.com/a8m/tree v0.0.0-20180321023834-3cf936ce15d6/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=
 github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
 github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
 github.com/anacrolix/dms v0.0.0-20180117034613-8af4925bffb5/go.mod h1:DGqLjaZ3ziKKNRt+U5Q9PLWJ52Q/4rxfaaH/b3QYKaE=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/aws/aws-sdk-go v1.15.81 h1:va7uoFaV9uKAtZ6BTmp1u7paoMsizYRRLvRuoC07nQ8=
 github.com/aws/aws-sdk-go v1.15.81/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
 github.com/billziss-gh/cgofuse v1.1.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
@@ -219,6 +223,7 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=

logtr/log.go

@@ -82,15 +82,16 @@ func SetLevel(l Level) {
 }
 
 func logf(l Level, format string, args []interface{}) {
-	format = fmt.Sprintf("%v: %v: %s\n", time.Now().Format("15:04:05"), l.String(), strings.TrimSpace(format))
+	format = fmt.Sprintf("%v: %v: %s\n", time.Now().Format("01-02T15:04:05"), l.String(), strings.TrimSpace(format))
+	logContent := strings.ReplaceAll(fmt.Sprintf(format, args...), "\n", "") + "\n"
 	cLevel := level
 	cAnsoser := ansoser
 	if l >= cLevel {
-		fmt.Fprintf(os.Stderr, format, args...)
+		fmt.Fprint(os.Stderr, logContent)
 	}
-	fmt.Fprintf(logger, format, args...)
+	fmt.Fprint(logger, logContent)
 	if l == SOS && cAnsoser != nil {
-		if err := cAnsoser.Send(fmt.Sprintf(format, args...)); err != nil {
+		if err := cAnsoser.Send(logContent); err != nil {
 			Errorf("failed to SOS: %v", err)
 		}
 	}

main.go

@@ -21,13 +21,19 @@ import (
 var stateFinder = regexp.MustCompile(`[A-Za-z]+`)
 
 func main() {
-	if err := config.Refresh(message.NewSOSMatrix); err != nil {
+	if err := _main(); err != nil {
 		panic(err)
 	}
+}
+
+func _main() error {
+	if err := config.Refresh(message.NewSOSMatrix); err != nil {
+		return err
+	}
 	if config.Get().Message.Matrix.ReceiveEnabled {
 		if err := matrixrecv(); err != nil {
 			logtr.SOSf("failed to recv matrix on boot: %v", err)
-			panic(err)
+			return err
 		}
 	}
 	lock := &sync.Mutex{}
@@ -47,11 +53,12 @@ func main() {
 			}
 		}
 	}()
-	if err := _main(); err != nil {
+	if err := __main(); err != nil {
-		logtr.SOSf("failed _main: %v", err)
+		logtr.SOSf("failed __main: %v", err)
-		panic(err)
+		return err
 	}
 	lock.Lock()
+	return nil
 }
 
 func matrixrecv() error {
@@ -228,9 +235,9 @@ func parseOutStates(b []byte) []config.State {
 	return states
 }
 
-func _main() error {
+func __main() error {
 	for {
-		err := _mainOne()
+		err := __mainOne()
 		if err != nil {
 			logtr.Errorf("failed _main: %v", err)
 		}
@@ -247,7 +254,7 @@ func _main() error {
 	return nil
 }
 
-func _mainOne() error {
+func __mainOne() error {
 	logtr.Debugf("config.refreshing...")
 	if err := config.Refresh(message.NewSOSMatrix); err != nil {
 		logtr.SOSf("bad config: %v", err)
@@ -258,7 +265,7 @@ func _mainOne() error {
 		logtr.SOSf("failed once(): %v", err)
 		return err
 	}
-	logtr.Debugf("/_mainOne")
+	logtr.Debugf("/__mainOne")
 	return nil
 }
 
@@ -308,7 +315,8 @@ func once() error {
 func getJobs() ([]broker.Job, error) {
 	states := config.AllStates()
 	ntg := broker.NewNTGVision()
-	brokers := []broker.Broker{ntg}
+	fe := broker.NewFastExact()
+	brokers := []broker.Broker{ntg, fe}
 	jobs := []broker.Job{}
 	for _, broker := range brokers {
 		somejobs, err := broker.Search(states)
@@ -350,9 +358,14 @@ func updateDeadJobs(jobs []broker.Job) error {
 			if err := json.Unmarshal(b, &recorded); err != nil {
 				return err
 			}
+			/* // TODO this beeps on fluffychat
 			if err := message.NewMatrix().Update(recorded.MatrixID, recorded.Job.FormatMultilineTextDead()); err != nil {
 				return err
 			}
+			*/
+			if err := message.NewMatrix().Remove(recorded.MatrixID); err != nil {
+				return err
+			}
 			if err := db.Set(listEntry, nil); err != nil {
 				return err
 			}

main_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"local/truckstop/config"
+	"os"
 	"testing"
 )
 
@@ -33,3 +34,10 @@ Content-Type: text/html; charset="UTF-8"
 		t.Fatal(want, got)
 	}
 }
+
+func TestMain(t *testing.T) {
+	os.Setenv("CONFIG", "./config.main_test.json")
+	if err := _main(); err != nil {
+		t.Fatal(err)
+	}
+}

todo.yaml

@@ -1,4 +1,5 @@
 todo:
+- !states emits current state 
 - TEST. Just like, refactor and test to shit.
 - try search ntg by autoinc?
 - test each !command callbacks to matrix